fix(js/algorithms/fast): handle old browsers

Closes #1082

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -214,6 +214,7 @@ nicksnyder
 nobots
 NONINFRINGEMENT
 nosleep
+nullglob
 OCOB
 ogtag
 oklch
@@ -278,6 +279,7 @@ Seo
 setsebool
 shellcheck
 shirou
+shopt
 Sidetrade
 simprint
 sitemap

.github/workflows/docker-pr.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-tags: true
           fetch-depth: 0
@@ -25,7 +25,7 @@ jobs:
         uses: Homebrew/actions/setup-homebrew@main
 
       - name: Setup Homebrew cellar cache
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: |
             /home/linuxbrew/.linuxbrew/Cellar
@@ -47,7 +47,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
         with:
           images: ghcr.io/${{ github.repository }}
 

.github/workflows/docker.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-tags: true
           fetch-depth: 0
@@ -35,7 +35,7 @@ jobs:
         uses: Homebrew/actions/setup-homebrew@main
 
       - name: Setup Homebrew cellar cache
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: |
             /home/linuxbrew/.linuxbrew/Cellar
@@ -56,7 +56,7 @@ jobs:
           brew bundle
 
       - name: Log into registry
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -64,7 +64,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
         with:
           images: ${{ env.IMAGE }}
 
@@ -78,7 +78,7 @@ jobs:
           SLOG_LEVEL: debug
 
       - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2.4.0
+        uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
         with:
           subject-name: ${{ env.IMAGE }}
           subject-digest: ${{ steps.build.outputs.digest }}

.github/workflows/docs-deploy.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 
@@ -25,7 +25,7 @@ jobs:
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
       - name: Log into registry
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: techarohq
@@ -33,7 +33,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
         with:
           images: ghcr.io/techarohq/anubis/docs
           tags: |
@@ -53,14 +53,14 @@ jobs:
           push: true
 
       - name: Apply k8s manifests to limsa lominsa
-        uses: actions-hub/kubectl@b5b19eeb6a0ffde16637e398f8b96ef01eb8fdb7 # v1.33.3
+        uses: actions-hub/kubectl@af345ed727f0268738e65be48422e463cc67c220 # v1.34.0
         env:
           KUBE_CONFIG: ${{ secrets.LIMSA_LOMINSA_KUBECONFIG }}
         with:
           args: apply -k docs/manifest
 
       - name: Apply k8s manifests to limsa lominsa
-        uses: actions-hub/kubectl@b5b19eeb6a0ffde16637e398f8b96ef01eb8fdb7 # v1.33.3
+        uses: actions-hub/kubectl@af345ed727f0268738e65be48422e463cc67c220 # v1.34.0
         env:
           KUBE_CONFIG: ${{ secrets.LIMSA_LOMINSA_KUBECONFIG }}
         with:

.github/workflows/docs-test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 
@@ -22,7 +22,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
         with:
           images: ghcr.io/techarohq/anubis/docs
           tags: |

.github/workflows/go.yml

@@ -15,7 +15,7 @@ jobs:
     #runs-on: alrest-techarohq
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       with:
         persist-credentials: false
 
@@ -28,7 +28,7 @@ jobs:
       uses: Homebrew/actions/setup-homebrew@main
 
     - name: Setup Homebrew cellar cache
-      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
       with:
         path: |
           /home/linuxbrew/.linuxbrew/Cellar
@@ -49,7 +49,7 @@ jobs:
         brew bundle
 
     - name: Setup Golang caches
-      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
       with:
         path: |
           ~/.cache/go-build
@@ -59,7 +59,7 @@ jobs:
           ${{ runner.os }}-golang-
 
     - name: Cache playwright binaries
-      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
       id: playwright-cache
       with:
         path: |

.github/workflows/package-builds-stable.yml

@@ -14,7 +14,7 @@ jobs:
     #runs-on: alrest-techarohq
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
           fetch-tags: true
@@ -29,7 +29,7 @@ jobs:
         uses: Homebrew/actions/setup-homebrew@main
 
       - name: Setup Homebrew cellar cache
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: |
             /home/linuxbrew/.linuxbrew/Cellar
@@ -50,7 +50,7 @@ jobs:
           brew bundle
 
       - name: Setup Golang caches
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: |
             ~/.cache/go-build

.github/workflows/package-builds-unstable.yml

@@ -15,7 +15,7 @@ jobs:
     #runs-on: alrest-techarohq
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       with:
         persist-credentials: false
         fetch-tags: true
@@ -30,7 +30,7 @@ jobs:
       uses: Homebrew/actions/setup-homebrew@main
 
     - name: Setup Homebrew cellar cache
-      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
       with:
         path: |
           /home/linuxbrew/.linuxbrew/Cellar
@@ -51,7 +51,7 @@ jobs:
         brew bundle
 
     - name: Setup Golang caches
-      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
       with:
         path: |
           ~/.cache/go-build

.github/workflows/smoke-tests.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 

.github/workflows/ssh-ci-runner-cron.yml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-tags: true
           fetch-depth: 0
           persist-credentials: false
       - name: Log into registry
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/ssh-ci.yml

@@ -20,7 +20,7 @@ jobs:
           - ci@ppc64le.techaro.lol
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-tags: true
           fetch-depth: 0

.github/workflows/zizmor.yml

@@ -16,12 +16,12 @@ jobs:
       security-events: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v6.6.0
 
       - name: Run zizmor 🌈
         run: uvx zizmor --format sarif . > results.sarif 
@@ -29,7 +29,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
 
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@4e828ff8d448a8a6e532957b1811f387a63867e8 # v3.29.4
+        uses: github/codeql-action/upload-sarif@3c3833e0f8c1c83d449a7478aa59c036a9165498 # v3.29.11
         with:
           sarif_file: results.sarif
           category: zizmor

cmd/robots2policy/main.go

@@ -29,7 +29,7 @@ var (
 )
 
 type RobotsRule struct {
-	UserAgent   string
+	UserAgents  []string
 	Disallows   []string
 	Allows      []string
 	CrawlDelay  int
@@ -130,10 +130,26 @@ func main() {
 	}
 }
 
+func createRuleFromAccumulated(userAgents, disallows, allows []string, crawlDelay int) RobotsRule {
+	rule := RobotsRule{
+		UserAgents: make([]string, len(userAgents)),
+		Disallows:  make([]string, len(disallows)),
+		Allows:     make([]string, len(allows)),
+		CrawlDelay: crawlDelay,
+	}
+	copy(rule.UserAgents, userAgents)
+	copy(rule.Disallows, disallows)
+	copy(rule.Allows, allows)
+	return rule
+}
+
 func parseRobotsTxt(input io.Reader) ([]RobotsRule, error) {
 	scanner := bufio.NewScanner(input)
 	var rules []RobotsRule
-	var currentRule *RobotsRule
+	var currentUserAgents []string
+	var currentDisallows []string
+	var currentAllows []string
+	var currentCrawlDelay int
 
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
@@ -154,38 +170,42 @@ func parseRobotsTxt(input io.Reader) ([]RobotsRule, error) {
 
 		switch directive {
 		case "user-agent":
-			// Start a new rule section
-			if currentRule != nil {
-				rules = append(rules, *currentRule)
-			}
-			currentRule = &RobotsRule{
-				UserAgent: value,
-				Disallows: make([]string, 0),
-				Allows:    make([]string, 0),
+			// If we have accumulated rules with directives and encounter a new user-agent,
+			// flush the current rules
+			if len(currentUserAgents) > 0 && (len(currentDisallows) > 0 || len(currentAllows) > 0 || currentCrawlDelay > 0) {
+				rule := createRuleFromAccumulated(currentUserAgents, currentDisallows, currentAllows, currentCrawlDelay)
+				rules = append(rules, rule)
+				// Reset for next group
+				currentUserAgents = nil
+				currentDisallows = nil
+				currentAllows = nil
+				currentCrawlDelay = 0
 			}
+			currentUserAgents = append(currentUserAgents, value)
 
 		case "disallow":
-			if currentRule != nil && value != "" {
-				currentRule.Disallows = append(currentRule.Disallows, value)
+			if len(currentUserAgents) > 0 && value != "" {
+				currentDisallows = append(currentDisallows, value)
 			}
 
 		case "allow":
-			if currentRule != nil && value != "" {
-				currentRule.Allows = append(currentRule.Allows, value)
+			if len(currentUserAgents) > 0 && value != "" {
+				currentAllows = append(currentAllows, value)
 			}
 
 		case "crawl-delay":
-			if currentRule != nil {
+			if len(currentUserAgents) > 0 {
 				if delay, err := parseIntSafe(value); err == nil {
-					currentRule.CrawlDelay = delay
+					currentCrawlDelay = delay
 				}
 			}
 		}
 	}
 
-	// Don't forget the last rule
-	if currentRule != nil {
-		rules = append(rules, *currentRule)
+	// Don't forget the last group of rules
+	if len(currentUserAgents) > 0 {
+		rule := createRuleFromAccumulated(currentUserAgents, currentDisallows, currentAllows, currentCrawlDelay)
+		rules = append(rules, rule)
 	}
 
 	// Mark blacklisted user agents (those with "Disallow: /")
@@ -211,10 +231,11 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 	var anubisRules []AnubisRule
 	ruleCounter := 0
 
+	// Process each robots rule individually
 	for _, robotsRule := range robotsRules {
-		userAgent := robotsRule.UserAgent
+		userAgents := robotsRule.UserAgents
 
-		// Handle crawl delay as weight adjustment (do this first before any continues)
+		// Handle crawl delay
 		if robotsRule.CrawlDelay > 0 && *crawlDelay > 0 {
 			ruleCounter++
 			rule := AnubisRule{
@@ -223,20 +244,32 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 				Weight: &config.Weight{Adjust: *crawlDelay},
 			}
 
-			if userAgent == "*" {
+			if len(userAgents) == 1 && userAgents[0] == "*" {
 				rule.Expression = &config.ExpressionOrList{
 					All: []string{"true"}, // Always applies
 				}
-			} else {
+			} else if len(userAgents) == 1 {
 				rule.Expression = &config.ExpressionOrList{
-					All: []string{fmt.Sprintf("userAgent.contains(%q)", userAgent)},
+					All: []string{fmt.Sprintf("userAgent.contains(%q)", userAgents[0])},
+				}
+			} else {
+				// Multiple user agents - use any block
+				var expressions []string
+				for _, ua := range userAgents {
+					if ua == "*" {
+						expressions = append(expressions, "true")
+					} else {
+						expressions = append(expressions, fmt.Sprintf("userAgent.contains(%q)", ua))
+					}
+				}
+				rule.Expression = &config.ExpressionOrList{
+					Any: expressions,
 				}
 			}
-
 			anubisRules = append(anubisRules, rule)
 		}
 
-		// Handle blacklisted user agents (complete deny/challenge)
+		// Handle blacklisted user agents
 		if robotsRule.IsBlacklist {
 			ruleCounter++
 			rule := AnubisRule{
@@ -244,21 +277,36 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 				Action: *userAgentDeny,
 			}
 
-			if userAgent == "*" {
-				// This would block everything - convert to a weight adjustment instead
-				rule.Name = fmt.Sprintf("%s-global-restriction-%d", *policyName, ruleCounter)
-				rule.Action = "WEIGH"
-				rule.Weight = &config.Weight{Adjust: 20} // Increase difficulty significantly
-				rule.Expression = &config.ExpressionOrList{
-					All: []string{"true"}, // Always applies
+			if len(userAgents) == 1 {
+				userAgent := userAgents[0]
+				if userAgent == "*" {
+					// This would block everything - convert to a weight adjustment instead
+					rule.Name = fmt.Sprintf("%s-global-restriction-%d", *policyName, ruleCounter)
+					rule.Action = "WEIGH"
+					rule.Weight = &config.Weight{Adjust: 20} // Increase difficulty significantly
+					rule.Expression = &config.ExpressionOrList{
+						All: []string{"true"}, // Always applies
+					}
+				} else {
+					rule.Expression = &config.ExpressionOrList{
+						All: []string{fmt.Sprintf("userAgent.contains(%q)", userAgent)},
+					}
 				}
 			} else {
+				// Multiple user agents - use any block
+				var expressions []string
+				for _, ua := range userAgents {
+					if ua == "*" {
+						expressions = append(expressions, "true")
+					} else {
+						expressions = append(expressions, fmt.Sprintf("userAgent.contains(%q)", ua))
+					}
+				}
 				rule.Expression = &config.ExpressionOrList{
-					All: []string{fmt.Sprintf("userAgent.contains(%q)", userAgent)},
+					Any: expressions,
 				}
 			}
 			anubisRules = append(anubisRules, rule)
-			continue
 		}
 
 		// Handle specific disallow rules
@@ -276,9 +324,33 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 			// Build CEL expression
 			var conditions []string
 
-			// Add user agent condition if not wildcard
-			if userAgent != "*" {
-				conditions = append(conditions, fmt.Sprintf("userAgent.contains(%q)", userAgent))
+			// Add user agent conditions
+			if len(userAgents) == 1 && userAgents[0] == "*" {
+				// Wildcard user agent - no user agent condition needed
+			} else if len(userAgents) == 1 {
+				conditions = append(conditions, fmt.Sprintf("userAgent.contains(%q)", userAgents[0]))
+			} else {
+				// For multiple user agents, we need to use a more complex expression
+				// This is a limitation - we can't easily combine any for user agents with all for path
+				// So we'll create separate rules for each user agent
+				for _, ua := range userAgents {
+					if ua == "*" {
+						continue // Skip wildcard as it's handled separately
+					}
+					ruleCounter++
+					subRule := AnubisRule{
+						Name:   fmt.Sprintf("%s-disallow-%d", *policyName, ruleCounter),
+						Action: *baseAction,
+						Expression: &config.ExpressionOrList{
+							All: []string{
+								fmt.Sprintf("userAgent.contains(%q)", ua),
+								buildPathCondition(disallow),
+							},
+						},
+					}
+					anubisRules = append(anubisRules, subRule)
+				}
+				continue
 			}
 
 			// Add path condition
@@ -291,7 +363,6 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
 
 			anubisRules = append(anubisRules, rule)
 		}
-
 	}
 
 	return anubisRules

cmd/robots2policy/robots2policy_test.go

@@ -78,6 +78,12 @@ func TestDataFileConversion(t *testing.T) {
 			expectedFile: "complex.yaml",
 			options:      TestOptions{format: "yaml", crawlDelayWeight: 5},
 		},
+		{
+			name:         "consecutive_user_agents",
+			robotsFile:   "consecutive.robots.txt",
+			expectedFile: "consecutive.yaml",
+			options:      TestOptions{format: "yaml", crawlDelayWeight: 3},
+		},
 	}
 
 	for _, tc := range testCases {

cmd/robots2policy/testdata/blacklist.yaml

@@ -25,6 +25,6 @@
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("Googlebot")
-        - path.startsWith("/search")
-  name: robots-txt-policy-disallow-7
+    - userAgent.contains("Googlebot")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-7

cmd/robots2policy/testdata/complex.yaml

@@ -20,8 +20,8 @@
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("Googlebot")
-        - path.startsWith("/search/")
+    - userAgent.contains("Googlebot")
+    - path.startsWith("/search/")
   name: robots-txt-policy-disallow-6
 - action: WEIGH
   expression: userAgent.contains("Bingbot")
@@ -31,14 +31,14 @@
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("Bingbot")
-        - path.startsWith("/search/")
+    - userAgent.contains("Bingbot")
+    - path.startsWith("/search/")
   name: robots-txt-policy-disallow-8
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("Bingbot")
-        - path.startsWith("/admin/")
+    - userAgent.contains("Bingbot")
+    - path.startsWith("/admin/")
   name: robots-txt-policy-disallow-9
 - action: DENY
   expression: userAgent.contains("BadBot")
@@ -54,18 +54,18 @@
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("TestBot")
-        - path.matches("^/.*/admin")
+    - userAgent.contains("TestBot")
+    - path.matches("^/.*/admin")
   name: robots-txt-policy-disallow-13
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("TestBot")
-        - path.matches("^/temp.*\\.html")
+    - userAgent.contains("TestBot")
+    - path.matches("^/temp.*\\.html")
   name: robots-txt-policy-disallow-14
 - action: CHALLENGE
   expression:
     all:
-        - userAgent.contains("TestBot")
-        - path.matches("^/file.\\.log")
+    - userAgent.contains("TestBot")
+    - path.matches("^/file.\\.log")
   name: robots-txt-policy-disallow-15

cmd/robots2policy/testdata/consecutive.robots.txt

@@ -0,0 +1,25 @@
+# Test consecutive user agents that should be grouped into any: blocks
+User-agent: *
+Disallow: /admin
+Crawl-delay: 10
+
+# Multiple consecutive user agents - should be grouped
+User-agent: BadBot
+User-agent: SpamBot
+User-agent: EvilBot
+Disallow: /
+
+# Single user agent - should be separate
+User-agent: GoodBot
+Disallow: /private
+
+# Multiple consecutive user agents with crawl delay
+User-agent: SlowBot1
+User-agent: SlowBot2
+Crawl-delay: 5
+
+# Multiple consecutive user agents with specific path
+User-agent: SearchBot1
+User-agent: SearchBot2
+User-agent: SearchBot3
+Disallow: /search 

cmd/robots2policy/testdata/consecutive.yaml

@@ -0,0 +1,47 @@
+- action: WEIGH
+  expression: "true"
+  name: robots-txt-policy-crawl-delay-1
+  weight:
+    adjust: 3
+- action: CHALLENGE
+  expression: path.startsWith("/admin")
+  name: robots-txt-policy-disallow-2
+- action: DENY
+  expression:
+    any:
+    - userAgent.contains("BadBot")
+    - userAgent.contains("SpamBot")
+    - userAgent.contains("EvilBot")
+  name: robots-txt-policy-blacklist-3
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("GoodBot")
+    - path.startsWith("/private")
+  name: robots-txt-policy-disallow-4
+- action: WEIGH
+  expression:
+    any:
+    - userAgent.contains("SlowBot1")
+    - userAgent.contains("SlowBot2")
+  name: robots-txt-policy-crawl-delay-5
+  weight:
+    adjust: 3
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("SearchBot1")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-7
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("SearchBot2")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-8
+- action: CHALLENGE
+  expression:
+    all:
+    - userAgent.contains("SearchBot3")
+    - path.startsWith("/search")
+  name: robots-txt-policy-disallow-9

cmd/robots2policy/testdata/simple.json

@@ -1,12 +1,12 @@
 [
   {
-    "action": "CHALLENGE",
     "expression": "path.startsWith(\"/admin/\")",
-    "name": "robots-txt-policy-disallow-1"
+    "name": "robots-txt-policy-disallow-1",
+    "action": "CHALLENGE"
   },
   {
-    "action": "CHALLENGE",
     "expression": "path.startsWith(\"/private\")",
-    "name": "robots-txt-policy-disallow-2"
+    "name": "robots-txt-policy-disallow-2",
+    "action": "CHALLENGE"
   }
 ]

docs/docs/CHANGELOG.md

@@ -13,8 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- This changes the project to: -->
 
+- Document missing environment variables in installation guide: `SLOG_LEVEL`, `COOKIE_PREFIX`, `FORCED_LANGUAGE`, and `TARGET_DISABLE_KEEPALIVE` ([#1086](https://github.com/TecharoHQ/anubis/pull/1086))
+- Fixed `robots2policy` to properly group consecutive user agents into `any:` instead of only processing the last one ([#925](https://github.com/TecharoHQ/anubis/pull/925))
 - Add the [`s3api` storage backend](./admin/policies.mdx#s3api) to allow Anubis to use S3 API compatible object storage as its storage backend.
 
+### Bug Fixes
+
+Sometimes the enhanced temporal assurance in [#1038](https://github.com/TecharoHQ/anubis/pull/1038) and [#1068](https://github.com/TecharoHQ/anubis/pull/1068) could backfire because Chromium and its ilk randomize the amount of time they wait in order to avoid a timing side channel attack. This has been fixed by both increasing the amount of time a client has to wait for the metarefresh and preact challenges as well as making the server side logic more permissive.
+
 ## v1.22.0: Yda Hext
 
 > Someone has to make an effort at reconciliation if these conflicts are ever going to end.

docs/docs/admin/installation.mdx

@@ -59,7 +59,7 @@ Currently the following settings are configurable via the policy file:
 Anubis uses these environment variables for configuration:
 
 | Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| :----------------------------- | :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+|:-------------------------------|:------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints (everything starting with `/.within.website/x/anubis/`). For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                                                                                                                                             |
 | `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                                                                                                                                                                                                                 |
 | `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                                                                                                                                                                                                              |
@@ -67,6 +67,7 @@ Anubis uses these environment variables for configuration:
 | `COOKIE_DYNAMIC_DOMAIN`        | false                   | If set to true, automatically set cookie domain fields based on the hostname of the request. EG: if you are making a request to `anubis.techaro.lol`, the Anubis cookie will be valid for any subdomain of `techaro.lol`.                                                                                                                                                                                                                                                                               |
 | `COOKIE_EXPIRATION_TIME`       | `168h`                  | The amount of time the authorization cookie is valid for.                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                                                                                                                                                                                                          |
+| `COOKIE_PREFIX`                | `anubis-cookie`         | The prefix used for browser cookies created by Anubis. Useful for customization or avoiding conflicts with other applications.                                                                                                                                                                                                                                                                                                                                                                          |
 | `COOKIE_SECURE`                | `true`                  | If set to `true`, enables the [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies), meaning that the cookies will only be transmitted over HTTPS. If Anubis is used in an unsecure context (plain HTTP), this will be need to be set to false                                                                                                                                                                                                   |
 | `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                                                                                                                                                                                                                  |
 | `ED25519_PRIVATE_KEY_HEX`      | unset                   | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. When running multiple instances on the same base domain, the key must be the same across all instances. See below for details.                                                                                                                                                                                                      |
@@ -81,6 +82,7 @@ Anubis uses these environment variables for configuration:
 | `PUBLIC_URL`                   | unset                   | The externally accessible URL for this Anubis instance, used for constructing redirect URLs (e.g., for Traefik forwardAuth).                                                                                                                                                                                                                                                                                                                                                                            |
 | `REDIRECT_DOMAINS`             | unset                   | If set, restrict the domains that Anubis can redirect to when passing a challenge.<br/><br/>If this is unset, Anubis may redirect to any domain which could cause security issues in the unlikely case that an attacker passes a challenge for your browser and then tricks you into clicking a link to your domain.<br/><br/>Note that if you are hosting Anubis on a non-standard port (`https://example:com:8443`, `http://www.example.net:8080`, etc.), you must also include the port number here. |
 | `SERVE_ROBOTS_TXT`             | `false`                 | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file.                                                                                                                                                                                                                |
+| `SLOG_LEVEL`                   | `INFO`                  | The log level for structured logging. Valid values are `DEBUG`, `INFO`, `WARN`, and `ERROR`. Set to `DEBUG` to see all requests, evaluations, and detailed diagnostic information.                                                                                                                                                                                                                                                                                                                      |
 | `SOCKET_MODE`                  | `0770`                  | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets.                                                                                                                                                                                                                                                                                                                                                               |
 | `STRIP_BASE_PREFIX`            | `false`                 | If set to `true`, strips the base prefix from request paths when forwarding to the target server. This is useful when your target service expects to receive requests without the base prefix. For example, with `BASE_PREFIX=/foo` and `STRIP_BASE_PREFIX=true`, a request to `/foo/bar` would be forwarded to the target as `/bar`.                                                                                                                                                                   |
 | `TARGET`                       | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`.                                                                                                                                                                                                                                                                                                                                            |
@@ -100,10 +102,12 @@ If you don't know or understand what these settings mean, ignore them. These are
 
 | Environment Variable          | Default value | Explanation                                                                                                                                                             |
 | :---------------------------- | :------------ | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `TARGET_SNI`                  | unset         | If set, overrides the TLS handshake hostname in requests forwarded to `TARGET`.                                                                                         |
+| `FORCED_LANGUAGE`             | unset         | If set, forces Anubis to display challenge pages in the specified language instead of using the browser's Accept-Language header. Use ISO 639-1 language codes (e.g., `de` for German, `fr` for French). |
+| `HS512_SECRET`                | unset         | Secret string for JWT HS512 algorithm. If this is not set, Anubis will use ED25519 as defined via the variables above. The longer the better; 128 chars should suffice. |
+| `TARGET_DISABLE_KEEPALIVE`    | `false`       | If `true`, disables HTTP keep-alive for connections to the target backend. Useful for backends that don't handle keep-alive properly.                                   |
 | `TARGET_HOST`                 | unset         | If set, overrides the Host header in requests forwarded to `TARGET`.                                                                                                    |
 | `TARGET_INSECURE_SKIP_VERIFY` | `false`       | If `true`, skip TLS certificate validation for targets that listen over `https`. If your backend does not listen over `https`, ignore this setting.                     |
-| `HS512_SECRET`                | unset         | Secret string for JWT HS512 algorithm. If this is not set, Anubis will use ED25519 as defined via the variables above. The longer the better; 128 chars should suffice. |
+| `TARGET_SNI`                  | unset         | If set, overrides the TLS handshake hostname in requests forwarded to `TARGET`.                                                                                         |
 
 </details>
 

lib/challenge/metarefresh/metarefresh.go

@@ -43,7 +43,7 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *challenge.IssueInput)
 }
 
 func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *challenge.ValidateInput) error {
-	wantTime := in.Challenge.IssuedAt.Add(time.Duration(in.Rule.Challenge.Difficulty) * 950 * time.Millisecond)
+	wantTime := in.Challenge.IssuedAt.Add(time.Duration(in.Rule.Challenge.Difficulty) * 800 * time.Millisecond)
 
 	if time.Now().Before(wantTime) {
 		return challenge.NewError("validate", "insufficent time", fmt.Errorf("%w: wanted user to wait until at least %s", challenge.ErrFailed, wantTime.Format(time.RFC3339)))

lib/challenge/metarefresh/metarefresh.templ

@@ -13,6 +13,6 @@ templ page(redir string, difficulty int, loc *localization.SimpleLocalizer) {
 		<img style="display:none;" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version }/>
 		<p id="status">{ loc.T("loading") }</p>
 		<p>{ loc.T("connection_security") }</p>
-		<meta http-equiv="refresh" content={ fmt.Sprintf("%d; url=%s", difficulty, redir) }/>
+		<meta http-equiv="refresh" content={ fmt.Sprintf("%d; url=%s", difficulty+1, redir) }/>
 	</div>
 }

lib/challenge/metarefresh/metarefresh_templ.go

@@ -93,9 +93,9 @@ func page(redir string, difficulty int, loc *localization.SimpleLocalizer) templ
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var6 string
-		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d; url=%s", difficulty, redir))
+		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d; url=%s", difficulty+1, redir))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 16, Col: 83}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 16, Col: 85}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {

lib/challenge/preact/build.sh

@@ -40,9 +40,9 @@ for the JavaScript code in this page.
 
 mkdir -p static/js
 
-for file in js/*.jsx; do
+for file in js/*.tsx; do
   filename="${file##*/}"       # Extracts "app.jsx" from "./js/app.jsx"
-  output="${filename%.jsx}.js"  # Changes "app.jsx" to "app.js"
+  output="${filename%.tsx}.js"  # Changes "app.jsx" to "app.js"
   echo $output
 
   esbuild "${file}" --minify --bundle --outfile=static/"${output}" --banner:js="${LICENSE}"

lib/challenge/preact/js/app.jsx

@@ -1,62 +0,0 @@
-import { render, h, Fragment } from 'preact';
-import { useState, useEffect } from 'preact/hooks';
-import { g, j, u, x } from "./xeact.js";
-import { Sha256 } from '@aws-crypto/sha256-js';
-
-/** @jsx h */
-/** @jsxFrag Fragment */
-
-function toHexString(arr) {
-  return Array.from(arr)
-    .map((c) => c.toString(16).padStart(2, "0"))
-    .join("");
-}
-
-const App = () => {
-  const [state, setState] = useState(null);
-  const [imageURL, setImageURL] = useState(null);
-  const [passed, setPassed] = useState(false);
-  const [challenge, setChallenge] = useState(null);
-
-  useEffect(() => {
-    setState(j("preact_info"));
-  });
-
-  useEffect(() => {
-    setImageURL(state.pensive_url);
-    const hash = new Sha256('');
-    hash.update(state.challenge);
-    setChallenge(toHexString(hash.digestSync()));
-  }, [state]);
-
-  useEffect(() => {
-    const timer = setTimeout(() => {
-      setPassed(true);
-    }, state.difficulty * 100);
-
-    return () => clearTimeout(timer);
-  }, [challenge]);
-
-  useEffect(() => {
-    window.location.href = u(state.redir, {
-      result: challenge,
-    });
-  }, [passed]);
-
-  return (
-    <>
-      {imageURL !== null && (
-        <img src={imageURL} style="width:100%;max-width:256px;" />
-      )}
-      {state !== null && (
-        <>
-          <p id="status">{state.loading_message}</p>
-          <p>{state.connection_security_message}</p>
-        </>
-      )}
-    </>
-  );
-};
-
-x(g("app"));
-render(<App />, g("app"));

lib/challenge/preact/js/app.tsx

@@ -0,0 +1,87 @@
+import { render, h, Fragment } from "preact";
+import { useState, useEffect } from "preact/hooks";
+import { g, j, r, u, x } from "./xeact.js";
+import { Sha256 } from "@aws-crypto/sha256-js";
+
+/** @jsx h */
+/** @jsxFrag Fragment */
+
+function toHexString(arr: Uint8Array) {
+  return Array.from(arr)
+    .map((c) => c.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+interface PreactInfo {
+  redir: string;
+  challenge: string;
+  difficulty: number;
+  connection_security_message: string;
+  loading_message: string;
+  pensive_url: string;
+}
+
+const App = () => {
+  const [state, setState] = useState<PreactInfo>();
+  const [imageURL, setImageURL] = useState<string | null>(null);
+  const [passed, setPassed] = useState<boolean>(false);
+  const [challenge, setChallenge] = useState<string | null>(null);
+
+  useEffect(() => {
+    setState(j("preact_info"));
+  });
+
+  useEffect(() => {
+    if (state === undefined) {
+      return;
+    }
+
+    setImageURL(state?.pensive_url);
+    const hash = new Sha256("");
+    hash.update(state.challenge);
+    setChallenge(toHexString(hash.digestSync()));
+  }, [state]);
+
+  useEffect(() => {
+    if (state === undefined) {
+      return;
+    }
+
+    const timer = setTimeout(() => {
+      setPassed(true);
+    }, state?.difficulty * 125);
+
+    return () => clearTimeout(timer);
+  }, [challenge]);
+
+  useEffect(() => {
+    if (state === undefined) {
+      return;
+    }
+
+    if (challenge === null) {
+      return;
+    }
+
+    window.location.href = u(state.redir, {
+      result: challenge,
+    });
+  }, [passed]);
+
+  return (
+    <>
+      {imageURL !== null && (
+        <img src={imageURL} style={{ width: "100%", maxWidth: "256px" }} />
+      )}
+      {state !== undefined && (
+        <>
+          <p id="status">{state.loading_message}</p>
+          <p>{state.connection_security_message}</p>
+        </>
+      )}
+    </>
+  );
+};
+
+x(g("app"));
+render(<App />, g("app"));

lib/challenge/preact/preact.go

@@ -57,7 +57,7 @@ func (i *impl) Issue(r *http.Request, lg *slog.Logger, in *challenge.IssueInput)
 }
 
 func (i *impl) Validate(r *http.Request, lg *slog.Logger, in *challenge.ValidateInput) error {
-	wantTime := in.Challenge.IssuedAt.Add(time.Duration(in.Rule.Challenge.Difficulty) * 95 * time.Millisecond)
+	wantTime := in.Challenge.IssuedAt.Add(time.Duration(in.Rule.Challenge.Difficulty) * 80 * time.Millisecond)
 
 	if time.Now().Before(wantTime) {
 		return challenge.NewError("validate", "insufficent time", fmt.Errorf("%w: wanted user to wait until at least %s", challenge.ErrFailed, wantTime.Format(time.RFC3339)))

web/build.sh

@@ -39,9 +39,18 @@ for the JavaScript code in this page.
 mkdir -p static/locales
 cp ../lib/localization/locales/*.json static/locales/
 
-for file in js/*.mjs js/worker/*.mjs; do
-  esbuild "${file}" --sourcemap --bundle --minify --outfile=static/"${file}" --banner:js="${LICENSE}"
-  gzip -f -k -n static/${file}
-  zstd -f -k --ultra -22 static/${file}
-  brotli -fZk static/${file}
+shopt -s nullglob globstar
+
+for file in js/**/*.ts js/**/*.mjs; do
+  out="static/${file}"
+  if [[ "$file" == *.ts ]]; then
+    out="static/${file%.ts}.mjs"
+  fi
+
+  mkdir -p "$(dirname "$out")"
+
+  esbuild "$file" --sourcemap --bundle --minify --outfile="$out" --banner:js="$LICENSE"
+  gzip -f -k -n "$out"
+  zstd -f -k --ultra -22 "$out"
+  brotli -fZk "$out"
 done

web/js/algorithms/fast.ts

@@ -1,11 +1,21 @@
+type ProgressCallback = (nonce: number) => void;
+
+interface ProcessOptions {
+  basePrefix: string;
+  version: string;
+}
+
+const getHardwareConcurrency = () =>
+  navigator.hardwareConcurrency !== undefined ? navigator.hardwareConcurrency : 1;
+
 export default function process(
-  { basePrefix, version },
-  data,
-  difficulty = 5,
-  signal = null,
-  progressCallback = null,
-  threads = Math.trunc(Math.max(navigator.hardwareConcurrency / 2, 1)),
-) {
+  options: ProcessOptions,
+  data: string,
+  difficulty: number = 5,
+  signal: AbortSignal | null = null,
+  progressCallback?: ProgressCallback,
+  threads: number = Math.trunc(Math.max(getHardwareConcurrency() / 2, 1)),
+): Promise<string> {
   console.debug("fast algo");
 
   let workerMethod = window.crypto !== undefined ? "webcrypto" : "purejs";
@@ -16,13 +26,17 @@ export default function process(
   }
 
   return new Promise((resolve, reject) => {
-    let webWorkerURL = `${basePrefix}/.within.website/x/cmd/anubis/static/js/worker/sha256-${workerMethod}.mjs?cacheBuster=${version}`;
+    let webWorkerURL = `${options.basePrefix}/.within.website/x/cmd/anubis/static/js/worker/sha256-${workerMethod}.mjs?cacheBuster=${options.version}`;
 
-    console.log(webWorkerURL);
-
-    const workers = [];
+    const workers: Worker[] = [];
     let settled = false;
 
+    const onAbort = () => {
+      console.log("PoW aborted");
+      cleanup();
+      reject(new DOMException("Aborted", "AbortError"));
+    };
+
     const cleanup = () => {
       if (settled) {
         return;
@@ -34,12 +48,6 @@ export default function process(
       }
     };
 
-    const onAbort = () => {
-      console.log("PoW aborted");
-      cleanup();
-      reject(new DOMException("Aborted", "AbortError"));
-    };
-
     if (signal != null) {
       if (signal.aborted) {
         return onAbort();

web/js/algorithms/index.ts

@@ -1,4 +1,4 @@
-import fast from "./fast.mjs";
+import fast from "./fast";
 
 export default {
   fast: fast,

web/js/bench.ts

@@ -1,20 +1,24 @@
-import algorithms from "./algorithms/index.mjs";
+import algorithms from "./algorithms";
 
 const defaultDifficulty = 4;
 
-const status = document.getElementById("status");
-const difficultyInput = document.getElementById("difficulty-input");
-const algorithmSelect = document.getElementById("algorithm-select");
-const compareSelect = document.getElementById("compare-select");
-const header = document.getElementById("table-header");
-const headerCompare = document.getElementById("table-header-compare");
-const results = document.getElementById("results");
+const status: HTMLParagraphElement = document.getElementById("status") as HTMLParagraphElement;
+const difficultyInput: HTMLInputElement = document.getElementById("difficulty-input") as HTMLInputElement;
+const algorithmSelect: HTMLSelectElement = document.getElementById("algorithm-select") as HTMLSelectElement;
+const compareSelect: HTMLSelectElement = document.getElementById("compare-select") as HTMLSelectElement;
+const header: HTMLTableRowElement = document.getElementById("table-header") as HTMLTableRowElement;
+const headerCompare: HTMLTableSectionElement = document.getElementById("table-header-compare") as HTMLTableSectionElement;
+const results: HTMLTableRowElement = document.getElementById("results") as HTMLTableRowElement;
 
 const setupControls = () => {
-  difficultyInput.value = defaultDifficulty;
+  if (defaultDifficulty == null) {
+    return;
+  }
+
+  difficultyInput.value = defaultDifficulty.toString();
   for (const alg of Object.keys(algorithms)) {
     const option1 = document.createElement("option");
-    algorithmSelect.append(option1);
+    algorithmSelect?.append(option1);
     const option2 = document.createElement("option");
     compareSelect.append(option2);
     option1.value = option1.innerText = option2.value = option2.innerText = alg;
@@ -116,13 +120,13 @@ const benchmarkLoop = async (controller) => {
   await benchmarkLoop(controller);
 };
 
-let controller = null;
+let controller: AbortController | null = null;
 const reset = () => {
   stats.time = stats.iters = 0;
   comparison.time = comparison.iters = 0;
   results.innerHTML = status.innerText = "";
 
-  const table = results.parentElement;
+  const table = results.parentElement as HTMLElement;
   if (compareSelect.value !== "NONE") {
     table.style.gridTemplateColumns = "repeat(4,auto)";
     header.style.display = "none";

web/js/main.ts

@@ -1,12 +1,21 @@
-import algorithms from "./algorithms/index.mjs";
+import algorithms from "./algorithms";
 
 // from Xeact
-const u = (url = "", params = {}) => {
+const u = (url: string = "", params: Record<string, any> = {}) => {
   let result = new URL(url, window.location.href);
   Object.entries(params).forEach(([k, v]) => result.searchParams.set(k, v));
   return result.toString();
 };
 
+const j = (id: string): any | null => {
+  const elem = document.getElementById(id);
+  if (elem === null) {
+    return null;
+  }
+
+  return JSON.parse(elem.textContent);
+};
+
 const imageURL = (mood, cacheBuster, basePrefix) =>
   u(`${basePrefix}/.within.website/x/cmd/anubis/static/img/${mood}.webp`, {
     cacheBuster,
@@ -14,9 +23,10 @@ const imageURL = (mood, cacheBuster, basePrefix) =>
 
 // Detect available languages by loading the manifest
 const getAvailableLanguages = async () => {
-  const basePrefix = JSON.parse(
-    document.getElementById("anubis_base_prefix").textContent,
-  );
+  const basePrefix = j("anubis_base_prefix");
+  if (basePrefix === null) {
+    return;
+  }
 
   try {
     const response = await fetch(`${basePrefix}/.within.website/x/cmd/anubis/static/locales/manifest.json`);
@@ -38,9 +48,11 @@ const getBrowserLanguage = async () =>
 
 // Load translations from JSON files
 const loadTranslations = async (lang) => {
-  const basePrefix = JSON.parse(
-    document.getElementById("anubis_base_prefix").textContent,
-  );
+  const basePrefix = j("anubis_base_prefix");
+  if (basePrefix === null) {
+    return;
+  }
+
   try {
     const response = await fetch(`${basePrefix}/.within.website/x/cmd/anubis/static/locales/${lang}.json`);
     return await response.json();
@@ -54,9 +66,10 @@ const loadTranslations = async (lang) => {
 };
 
 const getRedirectUrl = () => {
-  const publicUrl = JSON.parse(
-    document.getElementById("anubis_public_url").textContent,
-  );
+  const publicUrl = j("anubis_public_url");
+  if (publicUrl === null) {
+    return;
+  }
   if (publicUrl && window.location.href.startsWith(publicUrl)) {
     const urlParams = new URLSearchParams(window.location.search);
     return urlParams.get('redir');
@@ -91,16 +104,14 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
       value: navigator.cookieEnabled,
     },
   ];
-  const status = document.getElementById("status");
-  const image = document.getElementById("image");
-  const title = document.getElementById("title");
-  const progress = document.getElementById("progress");
-  const anubisVersion = JSON.parse(
-    document.getElementById("anubis_version").textContent,
-  );
-  const basePrefix = JSON.parse(
-    document.getElementById("anubis_base_prefix").textContent,
-  );
+
+  const status: HTMLParagraphElement = document.getElementById("status") as HTMLParagraphElement;
+  const image: HTMLImageElement = document.getElementById("image") as HTMLImageElement;
+  const title: HTMLHeadingElement = document.getElementById("title") as HTMLHeadingElement;
+  const progress: HTMLDivElement = document.getElementById("progress") as HTMLDivElement;
+
+  const anubisVersion = j("anubis_version");
+  const basePrefix = j("anubis_base_prefix");
   const details = document.querySelector("details");
   let userReadDetails = false;
 
@@ -132,9 +143,7 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
     }
   }
 
-  const { challenge, rules } = JSON.parse(
-    document.getElementById("anubis_challenge").textContent,
-  );
+  const { challenge, rules } = j("anubis_challenge");
 
   const process = algorithms[rules.algorithm];
   if (!process) {
@@ -182,7 +191,9 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
         const probability = Math.pow(1 - likelihood, iters);
         const distance = (1 - Math.pow(probability, 2)) * 100;
         progress["aria-valuenow"] = distance;
-        progress.firstElementChild.style.width = `${distance}%`;
+        if (progress.firstElementChild !== null) {
+          (progress.firstElementChild as HTMLElement).style.width = `${distance}%`;
+        }
 
         if (probability < 0.1 && !showingApology) {
           status.append(
@@ -197,7 +208,7 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
     console.log({ hash, nonce });
 
     if (userReadDetails) {
-      const container = document.getElementById("progress");
+      const container: HTMLDivElement = document.getElementById("progress") as HTMLDivElement;
 
       // Style progress bar as a continue button
       container.style.display = "flex";

web/js/worker/sha256-purejs.ts

@@ -6,7 +6,7 @@ const calculateSHA256 = (text) => {
   return hash.digest();
 };
 
-function toHexString(arr) {
+function toHexString(arr: Uint8Array): string {
   return Array.from(arr)
     .map((c) => c.toString(16).padStart(2, "0"))
     .join("");

web/js/worker/sha256-webcrypto.ts

@@ -1,10 +1,11 @@
 const encoder = new TextEncoder();
-const calculateSHA256 = async (input) => {
+
+const calculateSHA256 = async (input: string) => {
   const data = encoder.encode(input);
   return await crypto.subtle.digest("SHA-256", data);
 };
 
-const toHexString = (byteArray) => {
+const toHexString = (byteArray: Uint8Array) => {
   return byteArray.reduce((str, byte) => str + byte.toString(16).padStart(2, "0"), "");
 };