fix: collate

cmd/robots2policy/testdata/blacklist.yaml

@@ -3,28 +3,28 @@
   name: robots-txt-policy-crawl-delay-1
   weight:
     adjust: 3
-- action: WEIGH
-  expression: userAgent.contains("Googlebot")
-  name: robots-txt-policy-crawl-delay-2
-  weight:
-    adjust: 3
+- action: CHALLENGE
+  expression: path.startsWith("/admin")
+  name: robots-txt-policy-disallow-2
+- action: DENY
+  expression: userAgent.contains("BadBot")
+  name: robots-txt-policy-blacklist-3
 - action: WEIGH
   expression: userAgent.contains("SpamBot")
-  name: robots-txt-policy-crawl-delay-3
+  name: robots-txt-policy-crawl-delay-4
   weight:
     adjust: 3
 - action: DENY
-  expression:
-    any:
-    - userAgent.contains("BadBot")
-    - userAgent.contains("SpamBot")
-  name: robots-txt-policy-blacklist-4
-- action: CHALLENGE
-  expression: path.startsWith("/admin")
-  name: robots-txt-policy-disallow-5
+  expression: userAgent.contains("SpamBot")
+  name: robots-txt-policy-blacklist-5
+- action: WEIGH
+  expression: userAgent.contains("Googlebot")
+  name: robots-txt-policy-crawl-delay-6
+  weight:
+    adjust: 3
 - action: CHALLENGE
   expression:
     all:
     - userAgent.contains("Googlebot")
     - path.startsWith("/search")
-  name: robots-txt-policy-disallow-6
+  name: robots-txt-policy-disallow-7

cmd/robots2policy/testdata/complex.yaml

@@ -1,68 +1,71 @@
 - action: WEIGH
-  expression: userAgent.contains("Bingbot")
+  expression: "true"
   name: robots-txt-policy-crawl-delay-1
   weight:
     adjust: 5
-- action: WEIGH
-  expression: userAgent.contains("Googlebot")
-  name: robots-txt-policy-crawl-delay-2
-  weight:
-    adjust: 5
-- action: WEIGH
-  expression: userAgent.contains("SeoBot")
-  name: robots-txt-policy-crawl-delay-3
-  weight:
-    adjust: 5
-- action: WEIGH
-  expression: "true"
-  name: robots-txt-policy-crawl-delay-4
-  weight:
-    adjust: 5
-- action: DENY
-  expression:
-    any:
-    - userAgent.contains("BadBot")
-    - userAgent.contains("SeoBot")
-  name: robots-txt-policy-blacklist-5
 - action: CHALLENGE
-  expression:
-    all:
-    - userAgent.contains("TestBot")
-    - path.matches("^/.*/admin")
-  name: robots-txt-policy-disallow-6
-- action: CHALLENGE
-  expression:
-    all:
-    - userAgent.contains("Bingbot")
-    - path.startsWith("/admin/")
-  name: robots-txt-policy-disallow-8
-- action: CHALLENGE
-  expression: path.startsWith("/api/internal/")
-  name: robots-txt-policy-disallow-9
-- action: CHALLENGE
-  expression:
-    all:
-    - userAgent.contains("TestBot")
-    - path.matches("^/file.\\.log")
-  name: robots-txt-policy-disallow-10
+  expression: path.startsWith("/admin/")
+  name: robots-txt-policy-disallow-2
 - action: CHALLENGE
   expression: path.startsWith("/private/")
-  name: robots-txt-policy-disallow-11
+  name: robots-txt-policy-disallow-3
+- action: CHALLENGE
+  expression: path.startsWith("/api/internal/")
+  name: robots-txt-policy-disallow-4
+- action: WEIGH
+  expression: userAgent.contains("Googlebot")
+  name: robots-txt-policy-crawl-delay-5
+  weight:
+    adjust: 5
 - action: CHALLENGE
   expression:
     all:
     - userAgent.contains("Googlebot")
     - path.startsWith("/search/")
-  name: robots-txt-policy-disallow-13
+  name: robots-txt-policy-disallow-6
+- action: WEIGH
+  expression: userAgent.contains("Bingbot")
+  name: robots-txt-policy-crawl-delay-7
+  weight:
+    adjust: 5
 - action: CHALLENGE
   expression:
     all:
     - userAgent.contains("Bingbot")
     - path.startsWith("/search/")
-  name: robots-txt-policy-disallow-14
+  name: robots-txt-policy-disallow-8
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("Bingbot")
+    - path.startsWith("/admin/")
+  name: robots-txt-policy-disallow-9
+- action: DENY
+  expression: userAgent.contains("BadBot")
+  name: robots-txt-policy-blacklist-10
+- action: WEIGH
+  expression: userAgent.contains("SeoBot")
+  name: robots-txt-policy-crawl-delay-11
+  weight:
+    adjust: 5
+- action: DENY
+  expression: userAgent.contains("SeoBot")
+  name: robots-txt-policy-blacklist-12
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("TestBot")
+    - path.matches("^/.*/admin")
+  name: robots-txt-policy-disallow-13
 - action: CHALLENGE
   expression:
     all:
     - userAgent.contains("TestBot")
     - path.matches("^/temp.*\\.html")
+  name: robots-txt-policy-disallow-14
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("TestBot")
+    - path.matches("^/file.\\.log")
   name: robots-txt-policy-disallow-15

cmd/robots2policy/testdata/consecutive.robots.txt

@@ -0,0 +1,25 @@
+# Test consecutive user agents that should be grouped into any: blocks
+User-agent: *
+Disallow: /admin
+Crawl-delay: 10
+
+# Multiple consecutive user agents - should be grouped
+User-agent: BadBot
+User-agent: SpamBot
+User-agent: EvilBot
+Disallow: /
+
+# Single user agent - should be separate
+User-agent: GoodBot
+Disallow: /private
+
+# Multiple consecutive user agents with crawl delay
+User-agent: SlowBot1
+User-agent: SlowBot2
+Crawl-delay: 5
+
+# Multiple consecutive user agents with specific path
+User-agent: SearchBot1
+User-agent: SearchBot2
+User-agent: SearchBot3
+Disallow: /search 

cmd/robots2policy/testdata/consecutive.yaml

@@ -0,0 +1,47 @@
+- action: WEIGH
+  expression: "true"
+  name: robots-txt-policy-crawl-delay-1
+  weight:
+    adjust: 3
+- action: CHALLENGE
+  expression: path.startsWith("/admin")
+  name: robots-txt-policy-disallow-2
+- action: DENY
+  expression:
+    any:
+    - userAgent.contains("BadBot")
+    - userAgent.contains("SpamBot")
+    - userAgent.contains("EvilBot")
+  name: robots-txt-policy-blacklist-3
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("GoodBot")
+    - path.startsWith("/private")
+  name: robots-txt-policy-disallow-4
+- action: WEIGH
+  expression:
+    any:
+    - userAgent.contains("SlowBot1")
+    - userAgent.contains("SlowBot2")
+  name: robots-txt-policy-crawl-delay-5
+  weight:
+    adjust: 3
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("SearchBot1")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-7
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("SearchBot2")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-8
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("SearchBot3")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-9

cmd/robots2policy/testdata/wildcards.yaml

@@ -1,12 +1,12 @@
 - action: CHALLENGE
-  expression: path.matches("^/.*/private")
+  expression: path.matches("^/search.*")
   name: robots-txt-policy-disallow-1
 - action: CHALLENGE
-  expression: path.matches("^/admin/.*.action=delete")
+  expression: path.matches("^/.*/private")
   name: robots-txt-policy-disallow-2
 - action: CHALLENGE
   expression: path.matches("^/file.\\.txt")
   name: robots-txt-policy-disallow-3
 - action: CHALLENGE
-  expression: path.matches("^/search.*")
+  expression: path.matches("^/admin/.*.action=delete")
   name: robots-txt-policy-disallow-4