fix(lib): make tests pass

Signed-off-by: Xe Iaso <me@xeiaso.net>

lib/anubis.go

@@ -369,9 +369,11 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 	err = encoder.Encode(struct {
 		Rules     *config.ChallengeRules `json:"rules"`
 		Challenge string                 `json:"challenge"`
+		ID        string                 `json:"id"`
 	}{
-		Challenge: chall.RandomData,
 		Rules:     rule.Challenge,
+		Challenge: chall.RandomData,
+		ID:        chall.ID,
 	})
 	if err != nil {
 		lg.Error("failed to encode challenge", "err", err)

lib/anubis_test.go

@@ -35,6 +35,8 @@ func loadPolicies(t *testing.T, fname string, difficulty int) *policy.ParsedConf
 		fname = "./testdata/test_config.yaml"
 	}
 
+	t.Logf("loading policy file: %s", fname)
+
 	anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty)
 	if err != nil {
 		t.Fatal(err)
@@ -59,6 +61,7 @@ func spawnAnubis(t *testing.T, opts Options) *Server {
 }
 
 type challengeResp struct {
+	ID        string `json:"id"`
 	Challenge string `json:"challenge"`
 }
 
@@ -91,6 +94,8 @@ func makeChallenge(t *testing.T, ts *httptest.Server, cli *http.Client) challeng
 func handleChallengeZeroDifficulty(t *testing.T, ts *httptest.Server, cli *http.Client, chall challengeResp) *http.Response {
 	t.Helper()
 
+	t.Logf("%#v", chall)
+
 	nonce := 0
 	elapsedTime := 420
 	redir := "/"
@@ -108,8 +113,11 @@ func handleChallengeZeroDifficulty(t *testing.T, ts *httptest.Server, cli *http.
 	q.Set("nonce", fmt.Sprint(nonce))
 	q.Set("redir", redir)
 	q.Set("elapsedTime", fmt.Sprint(elapsedTime))
+	q.Set("id", chall.ID)
 	req.URL.RawQuery = q.Encode()
 
+	t.Log(q.Encode())
+
 	resp, err := cli.Do(req)
 	if err != nil {
 		t.Fatalf("can't do request: %v", err)
@@ -155,6 +163,17 @@ func (lcj *loggingCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
 	lcj.cookies[u.Host] = append(lcj.cookies[u.Host], cookies...)
 }
 
+type userAgentRoundTripper struct {
+	rt http.RoundTripper
+}
+
+func (u *userAgentRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Only set if not already present
+	req = req.Clone(req.Context()) // avoid mutating original request
+	req.Header.Set("User-Agent", "Mozilla/5.0")
+	return u.rt.RoundTrip(req)
+}
+
 func httpClient(t *testing.T) *http.Client {
 	t.Helper()
 
@@ -163,6 +182,9 @@ func httpClient(t *testing.T) *http.Client {
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
+		Transport: &userAgentRoundTripper{
+			rt: http.DefaultTransport,
+		},
 	}
 
 	return cli
@@ -325,7 +347,7 @@ func TestCheckDefaultDifficultyMatchesPolicy(t *testing.T) {
 
 	for i := 1; i < 10; i++ {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
-			anubisPolicy := loadPolicies(t, "", i)
+			anubisPolicy := loadPolicies(t, "testdata/test_config_no_thresholds.yaml", i)
 
 			s, err := New(Options{
 				Next:           h,
@@ -476,8 +498,11 @@ func TestBasePrefix(t *testing.T) {
 			q.Set("nonce", fmt.Sprint(nonce))
 			q.Set("redir", redir)
 			q.Set("elapsedTime", fmt.Sprint(elapsedTime))
+			q.Set("id", chall.ID)
 			req.URL.RawQuery = q.Encode()
 
+			t.Log(req.URL.String())
+
 			resp, err = cli.Do(req)
 			if err != nil {
 				t.Fatalf("can't do challenge passing: %v", err)

lib/policy/config/config.go

@@ -194,7 +194,7 @@ type ChallengeRules struct {
 }
 
 var (
-	ErrChallengeDifficultyTooLow  = errors.New("config.ChallengeRules: difficulty is too low (must be >= 1)")
+	ErrChallengeDifficultyTooLow  = errors.New("config.ChallengeRules: difficulty is too low (must be >= 0)")
 	ErrChallengeDifficultyTooHigh = errors.New("config.ChallengeRules: difficulty is too high (must be <= 64)")
 	ErrChallengeMustHaveAlgorithm = errors.New("config.ChallengeRules: must have algorithm name set")
 )
@@ -206,7 +206,7 @@ func (cr ChallengeRules) Valid() error {
 		errs = append(errs, ErrChallengeMustHaveAlgorithm)
 	}
 
-	if cr.Difficulty < 1 {
+	if cr.Difficulty < 0 {
 		errs = append(errs, fmt.Errorf("%w, got: %d", ErrChallengeDifficultyTooLow, cr.Difficulty))
 	}
 

lib/testdata/test_config.yaml

@@ -35,4 +35,11 @@ status_codes:
   CHALLENGE: 200
   DENY: 200
 
-thresholds: []
+thresholds:
+  - name: minimal-suspicion
+    expression: "true"
+    action: CHALLENGE
+    challenge:
+      algorithm: fast
+      difficulty: 1
+      report_as: 1

lib/testdata/test_config_no_thresholds.yaml

@@ -0,0 +1,38 @@
+bots:
+  - import: (data)/bots/_deny-pathological.yaml
+  - import: (data)/bots/aggressive-brazilian-scrapers.yaml
+  - import: (data)/meta/ai-block-aggressive.yaml
+  - import: (data)/crawlers/_allow-good.yaml
+  - import: (data)/clients/x-firefox-ai.yaml
+  - import: (data)/common/keep-internet-working.yaml
+  - name: countries-with-aggressive-scrapers
+    action: WEIGH
+    geoip:
+      countries:
+        - BR
+        - CN
+    weight:
+      adjust: 10
+  - name: aggressive-asns-without-functional-abuse-contact
+    action: WEIGH
+    asns:
+      match:
+        - 13335 # Cloudflare
+        - 136907 # Huawei Cloud
+        - 45102 # Alibaba Cloud
+    weight:
+      adjust: 10
+  - name: generic-browser
+    user_agent_regex: >-
+      Mozilla|Opera
+    action: WEIGH
+    weight:
+      adjust: 10
+
+dnsbl: false
+
+status_codes:
+  CHALLENGE: 200
+  DENY: 200
+
+thresholds: []