Bypass CF for FA with hacky cfscrape usage.

Dockerfile

@@ -9,6 +9,7 @@ ENV DB_DIR="/app/db"
 WORKDIR /app
 RUN mkdir $DB_DIR
 COPY --from=builder /app/main .
+COPY ./utils /utils
 VOLUME /app/db
-RUN apk add ca-certificates
+RUN apk add ca-certificates python3 python3-dev nodejs && pip3 install --upgrade pip setuptools cfscrape
 CMD ["./main"]

go.sum

@@ -147,6 +147,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM=
 github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog=
@@ -212,6 +213,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190428183149-804c0c7841b5 h1:m0i9YywO9THhxmJvLEwKJDD/pD8ljCB+EaT/wYS41Is=
 golang.org/x/sys v0.0.0-20190428183149-804c0c7841b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

sites/fa.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
+	"os/exec"
 	"strconv"
 	"strings"
 
@@ -50,6 +51,11 @@ func (fa) IsSupportedURL(url string) bool {
 	return strings.Contains(url, "furaffinity.net/view/") || strings.Contains(url, "furaffinity.net/full/") || strings.Contains(url, "facdn.net/art/")
 }
 
+type cfscrape struct {
+	Cookies   map[string]string `json:"cookies"`
+	UserAgent string            `json:"user_agent"`
+}
+
 func (f fa) handleDirectLink(postURL string) ([]PostInfo, error) {
 	logger.Log.Debug("Attempting to load FA direct link")
 
@@ -110,6 +116,49 @@ func (f fa) GetImageURLs(postURL string, _ tgbotapi.User) ([]PostInfo, error) {
 		return nil, err
 	}
 
+	logger.Log.Debugf("Got status code %d\n", resp.StatusCode)
+
+	if resp.StatusCode == 503 || resp.StatusCode == 429 {
+		out, err := exec.Command("python3", "/utils/cfscrape_cli.py", "https://www.furaffinity.net").Output()
+		if err != nil {
+			logger.Log.Warnf("Unable to run cfscrape: %s\n", err.Error())
+			return nil, err
+		}
+
+		var cf cfscrape
+		err = json.Unmarshal(out, &cf)
+		if err != nil {
+			return nil, err
+		}
+
+		logger.Log.Printf("Got output: %v\n", cf)
+
+		cookies := f.cookies
+
+		for name, value := range cf.Cookies {
+			cookies = append(cookies, &http.Cookie{
+				Name:  name,
+				Value: value,
+			})
+		}
+
+		jar.SetCookies(u, cookies)
+
+		client := &http.Client{Jar: jar}
+
+		req, _ := http.NewRequest("GET", u.String(), nil)
+		req.Header.Set("User-Agent", cf.UserAgent)
+
+		r, err := client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+
+		resp = r
+
+		logger.Log.Debugf("Got new status code %d\n", resp.StatusCode)
+	}
+
 	doc, err := goquery.NewDocumentFromResponse(resp)
 	if err != nil {
 		return nil, err

utils/cfscrape_cli.py

@@ -0,0 +1,22 @@
+import sys
+import json
+
+import cfscrape
+
+def main():
+    url = sys.argv[1]
+    cookies, user_agent = cfscrape.get_tokens(url)
+
+    print(json.dumps({
+        'cookies': cookies,
+        'user_agent': user_agent,
+    }))
+
+def help_message():
+    print('Provide a URL to load, get a JSON dict of data.')
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        help_message()
+    else:
+        main()