diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 11f0e83..a03b6c7 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -9,6 +9,7 @@ services: - JWT_SIGNING_SECRET=secret - SCRAPER=api # - API_KEY=... + # - FLARESOLVERR_URL=http://flaresolverr:8191 ports: - '80:8080' restart: 'always' diff --git a/go.mod b/go.mod index b5fb1e7..a20f4ae 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/go-resty/resty/v2 v2.12.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/joho/godotenv v1.5.1 + golang.org/x/sync v0.7.0 ) require ( @@ -58,7 +59,6 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect google.golang.org/protobuf v1.34.1 // indirect diff --git a/src/routes/image.go b/src/routes/image.go index a070985..931d660 100644 --- a/src/routes/image.go +++ b/src/routes/image.go @@ -2,12 +2,13 @@ package routes import ( "anonymousoverflow/src/types" + "anonymousoverflow/src/utils" "fmt" + "net/http" "os" "time" "github.com/gin-gonic/gin" - "github.com/go-resty/resty/v2" "github.com/golang-jwt/jwt/v4" ) @@ -51,12 +52,15 @@ func GetImage(c *gin.Context) { } // download the image - client := resty.New() - resp, err := client.R().Get(claims.ImageURL) + resp, err := utils.GetWithClearance(claims.ImageURL) if err != nil { c.AbortWithStatus(500) return } + if resp.StatusCode() != http.StatusOK { + c.AbortWithStatus(http.StatusBadGateway) + return + } // set the content type c.Header("Content-Type", resp.Header().Get("Content-Type")) diff --git a/src/utils/flaresolverr.go b/src/utils/flaresolverr.go new file mode 100644 index 0000000..183db3b --- /dev/null +++ b/src/utils/flaresolverr.go @@ -0,0 +1,161 @@ +package utils + +import ( + "fmt" + "net/http" + "net/url" + "os" + "sync" + "time" + + "github.com/go-resty/resty/v2" + "golang.org/x/sync/singleflight" +) + +// clearance is a solved Cloudflare challenge. cf_clearance is bound to the +// hostname and to the user agent that solved it, so both are kept per host. +type clearance struct { + url *url.URL + userAgent string + cookies []*http.Cookie +} + +type flaresolverrResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Solution struct { + URL string `json:"url"` + UserAgent string `json:"userAgent"` + Cookies []struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Secure bool `json:"secure"` + } `json:"cookies"` + } `json:"solution"` +} + +var ( + clearancesMu sync.Mutex + clearances = map[string]*clearance{} + + // Concurrent requests for a host share the solve result, including errors. + solves singleflight.Group + + solveMu sync.Mutex +) + +const solveTimeout = 60 * time.Second + +// GetWithClearance fetches target, solving a Cloudflare challenge through +// FlareSolverr (FLARESOLVERR_URL) when the host answers with 403. +func GetWithClearance(target string) (*resty.Response, error) { + u, err := url.Parse(target) + if err != nil { + return nil, err + } + host := u.Hostname() + + clearancesMu.Lock() + cl := clearances[host] + clearancesMu.Unlock() + + res, err := getWith(target, cl) + if err != nil || res.StatusCode() != http.StatusForbidden || os.Getenv("FLARESOLVERR_URL") == "" { + return res, err + } + + finalURL := res.RawResponse.Request.URL + target = finalURL.String() + if finalURL.Hostname() != host { + host = finalURL.Hostname() + clearancesMu.Lock() + cl = clearances[host] + clearancesMu.Unlock() + if cl != nil { + res, err = getWith(target, cl) + if err != nil || res.StatusCode() != http.StatusForbidden { + return res, err + } + } + } + + cl, err = solve(host, target, cl) + if err != nil { + return nil, err + } + + return getWith(target, cl) +} + +func getWith(target string, cl *clearance) (*resty.Response, error) { + client := resty.New() + if cl != nil { + client.GetClient().Jar.SetCookies(cl.url, cl.cookies) + client.SetHeader("User-Agent", cl.userAgent) + } + return client.R().Get(target) +} + +func solve(host, target string, stale *clearance) (*clearance, error) { + result, err, _ := solves.Do(host, func() (interface{}, error) { + solveMu.Lock() + defer solveMu.Unlock() + + clearancesMu.Lock() + current := clearances[host] + clearancesMu.Unlock() + // Another request refreshed the clearance while this one waited. + if current != stale { + return current, nil + } + + endpoint, err := url.JoinPath(os.Getenv("FLARESOLVERR_URL"), "v1") + if err != nil { + return nil, err + } + + var fsRes flaresolverrResponse + // maxTimeout only bounds the solve inside FlareSolverr; the HTTP timeout + // keeps a hung FlareSolverr from holding solveMu forever. + res, err := resty.New().SetTimeout(solveTimeout + 30*time.Second).R(). + SetBody(map[string]any{ + "cmd": "request.get", + "url": target, + "maxTimeout": solveTimeout.Milliseconds(), + "returnOnlyCookies": true, + }). + SetResult(&fsRes). + SetError(&fsRes). + Post(endpoint) + if err != nil { + return nil, fmt.Errorf("flaresolverr request failed: %w", err) + } + if res.StatusCode() != http.StatusOK || fsRes.Status != "ok" { + return nil, fmt.Errorf("flaresolverr failed: %d %s", res.StatusCode(), fsRes.Message) + } + + solutionURL, err := url.Parse(fsRes.Solution.URL) + if err != nil || solutionURL.Hostname() == "" || (solutionURL.Scheme != "http" && solutionURL.Scheme != "https") { + return nil, fmt.Errorf("flaresolverr returned an invalid solution URL") + } + + cl := &clearance{url: solutionURL, userAgent: fsRes.Solution.UserAgent} + for _, c := range fsRes.Solution.Cookies { + cl.cookies = append(cl.cookies, &http.Cookie{ + Name: c.Name, Value: c.Value, Domain: c.Domain, Path: c.Path, Secure: c.Secure, + }) + } + + clearancesMu.Lock() + clearances[host] = cl + clearancesMu.Unlock() + + return cl, nil + }) + if err != nil { + return nil, err + } + return result.(*clearance), nil +} diff --git a/src/utils/flaresolverr_test.go b/src/utils/flaresolverr_test.go new file mode 100644 index 0000000..2625bce --- /dev/null +++ b/src/utils/flaresolverr_test.go @@ -0,0 +1,215 @@ +package utils + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sync/singleflight" +) + +func TestSolveSharesConcurrentResult(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + failed bool + }{ + {"success", http.StatusOK, `{"status":"ok","solution":{"url":"https://images.example/image.png","userAgent":"test-agent","cookies":[{"name":"cf_clearance","value":"test-cookie","secure":true}]}}`, false}, + {"solver error", http.StatusOK, `{"status":"error","message":"challenge failed"}`, true}, + {"HTTP error", http.StatusInternalServerError, `{"status":"error","message":"solver unavailable"}`, true}, + } { + t.Run(tc.name, func(t *testing.T) { + var calls int32 + entered := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1" { + t.Errorf("unexpected solver request: %s %s", r.Method, r.URL.Path) + } + if atomic.AddInt32(&calls, 1) == 1 { + close(entered) + <-release + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.status) + fmt.Fprint(w, tc.body) + })) + defer server.Close() + defer unblock() + t.Setenv("FLARESOLVERR_URL", server.URL) + + host := t.Name() + stale := &clearance{userAgent: "old-agent"} + clearancesMu.Lock() + clearances[host] = stale + clearancesMu.Unlock() + defer func() { + clearancesMu.Lock() + delete(clearances, host) + clearancesMu.Unlock() + }() + + type result struct { + cl *clearance + err error + } + firstResult := make(chan result, 1) + var worker sync.WaitGroup + worker.Add(1) + defer func() { + unblock() + worker.Wait() + }() + go func() { + defer worker.Done() + cl, err := solve(host, "https://images.example/image.png", stale) + firstResult <- result{cl, err} + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("solver request did not start") + } + + // DoChan registers each waiter before returning, without scheduler delays. + var waiters []<-chan singleflight.Result + for i := 0; i < 7; i++ { + waiters = append(waiters, solves.DoChan(host, func() (interface{}, error) { + return nil, fmt.Errorf("waiting request started another solve") + })) + } + unblock() + + var first result + select { + case first = <-firstResult: + case <-time.After(5 * time.Second): + t.Fatal("solve did not finish") + } + if (first.err != nil) != tc.failed { + t.Fatalf("unexpected solve result: clearance=%v error=%v", first.cl, first.err) + } + for _, waiter := range waiters { + select { + case got := <-waiter: + if !got.Shared || got.Err != first.err || (got.Err == nil && got.Val != first.cl) { + t.Fatal("concurrent callers did not share the same result") + } + case <-time.After(5 * time.Second): + t.Fatal("waiting caller did not finish") + } + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("solver received %d requests, want 1", got) + } + if !tc.failed && (first.cl.userAgent != "test-agent" || len(first.cl.cookies) != 1 || first.cl.cookies[0].Value != "test-cookie" || !first.cl.cookies[0].Secure) { + t.Fatalf("unexpected clearance: %+v", first.cl) + } + + cl, err := solve(host, "https://images.example/another.png", stale) + if tc.failed { + if err == nil || atomic.LoadInt32(&calls) != 2 { + t.Fatal("a later request should retry after a failed solve") + } + } else if err != nil || cl != first.cl || atomic.LoadInt32(&calls) != 1 { + t.Fatal("a later request with stale clearance should reuse the refreshed clearance") + } + }) + } +} + +func TestGetWithClearanceAcrossHosts(t *testing.T) { + var generation int32 = 1 + var solverCalls int32 + image := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/images/image.png" { + t.Errorf("unexpected image path: %s", r.URL.Path) + } + expectedCookie := fmt.Sprintf("cf_clearance=v%d", atomic.LoadInt32(&generation)) + if r.Header.Get("Cookie") != expectedCookie || r.UserAgent() != "test-agent" { + w.WriteHeader(http.StatusForbidden) + return + } + w.Header().Set("Content-Type", "image/png") + fmt.Fprint(w, "image bytes") + })) + defer image.Close() + finalURL := strings.Replace(image.URL, "127.0.0.1", "localhost", 1) + "/images/image.png" + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if cookie := r.Header.Get("Cookie"); cookie != "" { + t.Errorf("clearance leaked to the original host: %s", cookie) + } + http.Redirect(w, r, finalURL, http.StatusFound) + })) + defer origin.Close() + + solver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&solverCalls, 1) + var request struct { + URL string `json:"url"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + } + if request.URL != finalURL { + t.Errorf("solver URL = %q, want %q", request.URL, finalURL) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"status":"ok","solution":{"url":%q,"userAgent":"test-agent","cookies":[ + {"name":"cf_clearance","value":"v%d","domain":"localhost","path":"/images","secure":false}, + {"name":"other_domain","value":"hidden","domain":"example.invalid","path":"/"}, + {"name":"other_path","value":"hidden","domain":"localhost","path":"/private"} + ]}}`, finalURL, atomic.LoadInt32(&generation)) + })) + defer solver.Close() + t.Setenv("FLARESOLVERR_URL", solver.URL) + defer func() { + clearancesMu.Lock() + delete(clearances, "localhost") + clearancesMu.Unlock() + }() + + for _, tc := range []struct { + name string + generation int32 + calls int32 + }{ + {"initial solve", 1, 1}, + {"cached clearance", 1, 1}, + {"expired clearance", 2, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + atomic.StoreInt32(&generation, tc.generation) + res, err := GetWithClearance(origin.URL + "/old.png") + if err != nil { + t.Fatal(err) + } + if res.StatusCode() != http.StatusOK || string(res.Body()) != "image bytes" { + t.Fatalf("image status=%d body=%q", res.StatusCode(), res.Body()) + } + if calls := atomic.LoadInt32(&solverCalls); calls != tc.calls { + t.Fatalf("solver calls=%d, want %d", calls, tc.calls) + } + }) + } + + clearancesMu.Lock() + cl := clearances["localhost"] + clearancesMu.Unlock() + res, err := getWith(origin.URL+"/old.png", cl) + if err != nil { + t.Fatal(err) + } + if res.StatusCode() != http.StatusOK { + t.Fatalf("cookie jar did not apply clearance after redirect: %d", res.StatusCode()) + } +}