fix(robots2policy): handle multiple user agents under one block (#925)

2026-04-27 18:42:41 +00:00 · 2025-09-06 22:35:19 -04:00
parent 87c2f1e0e6
commit 82099d9e05
8 changed files with 208 additions and 58 deletions
@@ -78,6 +78,12 @@ func TestDataFileConversion(t *testing.T) {
 			expectedFile: "complex.yaml",
 			options:      TestOptions{format: "yaml", crawlDelayWeight: 5},
 		},
+		{
+			name:         "consecutive_user_agents",
+			robotsFile:   "consecutive.robots.txt",
+			expectedFile: "consecutive.yaml",
+			options:      TestOptions{format: "yaml", crawlDelayWeight: 3},
+		},
 	}

 	for _, tc := range testCases {