From 5a85cd41a2ccbb3616328deeae82e34435cd2749 Mon Sep 17 00:00:00 2001
From: Jason Cameron <git@jasoncameron.dev>
Date: Fri, 13 Jun 2025 13:25:30 -0400
Subject: [PATCH] feat: implement crawl delay handling as weight adjustment in
 Anubis rules

---
 cmd/robots2policy/main.go | 43 ++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/cmd/robots2policy/main.go b/cmd/robots2policy/main.go
index 88e9e0cf..2c08d587 100644
--- a/cmd/robots2policy/main.go
+++ b/cmd/robots2policy/main.go
@@ -243,6 +243,28 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 	for _, robotsRule := range robotsRules {
 		userAgent := robotsRule.UserAgent
 
+		// Handle crawl delay as weight adjustment (do this first before any continues)
+		if robotsRule.CrawlDelay > 0 && *crawlDelay > 0 {
+			ruleCounter++
+			rule := AnubisRule{
+				Name:   fmt.Sprintf("%s-crawl-delay-%d", *policyName, ruleCounter),
+				Action: "WEIGH",
+				Weight: &Weight{Adjust: *crawlDelay},
+			}
+
+			if userAgent == "*" {
+				rule.Expression = map[string]interface{}{
+					"single": "true", // Always applies
+				}
+			} else {
+				rule.Expression = map[string]interface{}{
+					"single": fmt.Sprintf("userAgent.contains(%q)", userAgent),
+				}
+			}
+
+			anubisRules = append(anubisRules, rule)
+		}
+
 		// Handle blacklisted user agents (complete deny/challenge)
 		if robotsRule.IsBlacklist {
 			ruleCounter++
@@ -305,27 +327,6 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 			anubisRules = append(anubisRules, rule)
 		}
 
-		// Handle crawl delay as weight adjustment
-		if robotsRule.CrawlDelay > 0 && *crawlDelay > 0 {
-			ruleCounter++
-			rule := AnubisRule{
-				Name:   fmt.Sprintf("%s-crawl-delay-%d", *policyName, ruleCounter),
-				Action: "WEIGH",
-				Weight: &Weight{Adjust: *crawlDelay},
-			}
-
-			if userAgent == "*" {
-				rule.Expression = map[string]interface{}{
-					"single": "true", // Always applies
-				}
-			} else {
-				rule.Expression = map[string]interface{}{
-					"single": fmt.Sprintf("userAgent.contains(%q)", userAgent),
-				}
-			}
-
-			anubisRules = append(anubisRules, rule)
-		}
 	}
 
 	return anubisRules