feat: optional FlareSolverr support for proxied images

docker-compose.example.yml

@@ -9,6 +9,7 @@ services:
             - JWT_SIGNING_SECRET=secret
             - SCRAPER=api
             # - API_KEY=...
+            # - FLARESOLVERR_URL=http://flaresolverr:8191
         ports:
             - '80:8080'
         restart: 'always'

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/go-resty/resty/v2 v2.12.0
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/joho/godotenv v1.5.1
+	golang.org/x/sync v0.7.0
 )
 
 require (
@@ -58,7 +59,6 @@ require (
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/crypto v0.23.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect

src/routes/image.go

@@ -2,12 +2,13 @@ package routes
 
 import (
 	"anonymousoverflow/src/types"
+	"anonymousoverflow/src/utils"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/go-resty/resty/v2"
 	"github.com/golang-jwt/jwt/v4"
 )
 
@@ -51,12 +52,15 @@ func GetImage(c *gin.Context) {
 	}
 
 	// download the image
-	client := resty.New()
-	resp, err := client.R().Get(claims.ImageURL)
+	resp, err := utils.GetWithClearance(claims.ImageURL)
 	if err != nil {
 		c.AbortWithStatus(500)
 		return
 	}
+	if resp.StatusCode() != http.StatusOK {
+		c.AbortWithStatus(http.StatusBadGateway)
+		return
+	}
 
 	// set the content type
 	c.Header("Content-Type", resp.Header().Get("Content-Type"))

src/utils/flaresolverr.go

@@ -0,0 +1,161 @@
+package utils
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/go-resty/resty/v2"
+	"golang.org/x/sync/singleflight"
+)
+
+// clearance is a solved Cloudflare challenge. cf_clearance is bound to the
+// hostname and to the user agent that solved it, so both are kept per host.
+type clearance struct {
+	url       *url.URL
+	userAgent string
+	cookies   []*http.Cookie
+}
+
+type flaresolverrResponse struct {
+	Status   string `json:"status"`
+	Message  string `json:"message"`
+	Solution struct {
+		URL       string `json:"url"`
+		UserAgent string `json:"userAgent"`
+		Cookies   []struct {
+			Name   string `json:"name"`
+			Value  string `json:"value"`
+			Domain string `json:"domain"`
+			Path   string `json:"path"`
+			Secure bool   `json:"secure"`
+		} `json:"cookies"`
+	} `json:"solution"`
+}
+
+var (
+	clearancesMu sync.Mutex
+	clearances   = map[string]*clearance{}
+
+	// Concurrent requests for a host share the solve result, including errors.
+	solves singleflight.Group
+
+	solveMu sync.Mutex
+)
+
+const solveTimeout = 60 * time.Second
+
+// GetWithClearance fetches target, solving a Cloudflare challenge through
+// FlareSolverr (FLARESOLVERR_URL) when the host answers with 403.
+func GetWithClearance(target string) (*resty.Response, error) {
+	u, err := url.Parse(target)
+	if err != nil {
+		return nil, err
+	}
+	host := u.Hostname()
+
+	clearancesMu.Lock()
+	cl := clearances[host]
+	clearancesMu.Unlock()
+
+	res, err := getWith(target, cl)
+	if err != nil || res.StatusCode() != http.StatusForbidden || os.Getenv("FLARESOLVERR_URL") == "" {
+		return res, err
+	}
+
+	finalURL := res.RawResponse.Request.URL
+	target = finalURL.String()
+	if finalURL.Hostname() != host {
+		host = finalURL.Hostname()
+		clearancesMu.Lock()
+		cl = clearances[host]
+		clearancesMu.Unlock()
+		if cl != nil {
+			res, err = getWith(target, cl)
+			if err != nil || res.StatusCode() != http.StatusForbidden {
+				return res, err
+			}
+		}
+	}
+
+	cl, err = solve(host, target, cl)
+	if err != nil {
+		return nil, err
+	}
+
+	return getWith(target, cl)
+}
+
+func getWith(target string, cl *clearance) (*resty.Response, error) {
+	client := resty.New()
+	if cl != nil {
+		client.GetClient().Jar.SetCookies(cl.url, cl.cookies)
+		client.SetHeader("User-Agent", cl.userAgent)
+	}
+	return client.R().Get(target)
+}
+
+func solve(host, target string, stale *clearance) (*clearance, error) {
+	result, err, _ := solves.Do(host, func() (interface{}, error) {
+		solveMu.Lock()
+		defer solveMu.Unlock()
+
+		clearancesMu.Lock()
+		current := clearances[host]
+		clearancesMu.Unlock()
+		// Another request refreshed the clearance while this one waited.
+		if current != stale {
+			return current, nil
+		}
+
+		endpoint, err := url.JoinPath(os.Getenv("FLARESOLVERR_URL"), "v1")
+		if err != nil {
+			return nil, err
+		}
+
+		var fsRes flaresolverrResponse
+		// maxTimeout only bounds the solve inside FlareSolverr; the HTTP timeout
+		// keeps a hung FlareSolverr from holding solveMu forever.
+		res, err := resty.New().SetTimeout(solveTimeout + 30*time.Second).R().
+			SetBody(map[string]any{
+				"cmd":               "request.get",
+				"url":               target,
+				"maxTimeout":        solveTimeout.Milliseconds(),
+				"returnOnlyCookies": true,
+			}).
+			SetResult(&fsRes).
+			SetError(&fsRes).
+			Post(endpoint)
+		if err != nil {
+			return nil, fmt.Errorf("flaresolverr request failed: %w", err)
+		}
+		if res.StatusCode() != http.StatusOK || fsRes.Status != "ok" {
+			return nil, fmt.Errorf("flaresolverr failed: %d %s", res.StatusCode(), fsRes.Message)
+		}
+
+		solutionURL, err := url.Parse(fsRes.Solution.URL)
+		if err != nil || solutionURL.Hostname() == "" || (solutionURL.Scheme != "http" && solutionURL.Scheme != "https") {
+			return nil, fmt.Errorf("flaresolverr returned an invalid solution URL")
+		}
+
+		cl := &clearance{url: solutionURL, userAgent: fsRes.Solution.UserAgent}
+		for _, c := range fsRes.Solution.Cookies {
+			cl.cookies = append(cl.cookies, &http.Cookie{
+				Name: c.Name, Value: c.Value, Domain: c.Domain, Path: c.Path, Secure: c.Secure,
+			})
+		}
+
+		clearancesMu.Lock()
+		clearances[host] = cl
+		clearancesMu.Unlock()
+
+		return cl, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(*clearance), nil
+}