Revert "build(deps): bump the gomod group across 1 directory with 3 updates (…"

This reverts commit ebad69a4e1.

.github/actions/spelling/allow.txt

@@ -18,3 +18,8 @@ clampip
 pseudoprofound
 reimagining
 iocaine
+admins
+fout
+iplist
+NArg
+blocklists

.github/actions/spelling/excludes.txt

@@ -87,10 +87,14 @@
 ^docs/docs/user/known-instances.md$
 ^docs/manifest/.*$
 ^docs/static/\.nojekyll$
-^lib/policy/config/testdata/bad/unparseable\.json$
 ^internal/glob/glob_test.go$
+^internal/honeypot/naive/affirmations\.txt$
+^internal/honeypot/naive/spintext\.txt$
+^internal/honeypot/naive/titles\.txt$
+^lib/config/testdata/bad/unparseable\.json$
+^lib/localization/.*_test.go$
+^lib/localization/locales/.*\.json$
+^lib/policy/config/testdata/bad/unparseable\.json$
+^test/.*$
 ignore$
 robots.txt
-^lib/localization/locales/.*\.json$
-^lib/localization/.*_test.go$
-^test/.*$

.github/actions/spelling/expect.txt

@@ -73,6 +73,7 @@ Cromite
 crt
 Cscript
 daemonizing
+databento
 dayjob
 DDOS
 Debian
@@ -132,6 +133,7 @@ GHSA
 Ghz
 gipc
 gitea
+GLM
 godotenv
 goland
 gomod
@@ -232,6 +234,7 @@ nepeat
 netsurf
 nginx
 nicksnyder
+nikandfor
 nobots
 NONINFRINGEMENT
 nosleep
@@ -248,6 +251,7 @@ opengraph
 openrc
 oswald
 pag
+pagegen
 palemoon
 Pangu
 parseable
@@ -284,6 +288,7 @@ redirectscheme
 refactors
 remoteip
 reputational
+Rhul
 risc
 ruleset
 runlevels
@@ -303,6 +308,7 @@ Seo
 setsebool
 shellcheck
 shirou
+shoneypot
 shopt
 Sidetrade
 simprint
@@ -311,6 +317,7 @@ sls
 sni
 snipster
 Spambot
+spammer
 sparkline
 spyderbot
 srv
@@ -354,7 +361,6 @@ valkey
 Varis
 Velen
 vendored
-verify
 vhosts
 vkbot
 VKE
@@ -386,6 +392,7 @@ XNG
 XOB
 XOriginal
 XReal
+Y'shtola
 yae
 YAMLTo
 Yda
@@ -397,13 +404,3 @@ Zenos
 zizmor
 zombocom
 zos
-GLM
-iocaine
-nikandfor
-pagegen
-pseudoprofound
-reimagining
-Rhul
-shoneypot
-spammer
-Y'shtola

README.md

@@ -20,6 +20,9 @@ Anubis is brought to you by sponsors and donors like:
 <a href="https://www.raptorcs.com/content/base/products.html">
   <img src="./docs/static/img/sponsors/raptor-computing-logo.webp" alt="Raptor Computing Systems" height=64 />
 </a>
+<a href="https://databento.com/?utm_source=anubis&utm_medium=sponsor&utm_campaign=anubis">
+  <img src="./docs/static/img/sponsors/databento-logo.webp" alt="Databento" />
+</a>
 
 ### Gold Tier
 

data/apps/qualys-ssl-labs.yml

@@ -3,5 +3,6 @@
 - name: qualys-ssl-labs
   action: ALLOW
   remote_addresses:
-  - 64.41.200.0/24
+  - 69.67.183.0/24
   - 2600:C02:1020:4202::/64
+  - 2602:fdaa:c6:2::/64

docs/docs/CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Add iplist2rule tool that lets admins turn an IP address blocklist into an Anubis ruleset.
 - Add Polish locale ([#1292](https://github.com/TecharoHQ/anubis/pull/1309))
 
 <!-- This changes the project to: -->

docs/docs/admin/iplist2rule.mdx

@@ -0,0 +1,50 @@
+---
+title: iplist2rule CLI tool
+---
+
+The `iplist2rule` tool converts IP blocklists into Anubis challenge policies. It reads common IP block list formats and generates the appropriate Anubis policy file for IP address filtering.
+
+## Installation
+
+Install directly with Go
+
+```bash
+go install github.com/TecharoHQ/anubis/utils/cmd/iplist2rule@latest
+```
+
+## Usage
+
+Basic conversion from URL:
+
+```bash
+iplist2rule https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
+```
+
+Explicitly allow every IP address on a list:
+
+```bash
+iplist2rule --action ALLOW https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
+```
+
+Add weight to requests matching IP addresses on a list:
+
+```bash
+iplist2rule --action WEIGH --weight 20 https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
+```
+
+## Options
+
+| Flag          | Description                                                                                      | Default                           |
+| :------------ | :----------------------------------------------------------------------------------------------- | :-------------------------------- |
+| `--action`    | The Anubis action to take for the IP address in question, must be in ALL CAPS.                   | `DENY` (forbids traffic)          |
+| `--rule-name` | The name for the generated Anubis rule, should be in kebab-case.                                 | (not set, inferred from filename) |
+| `--weight`    | When `--action=WEIGH`, how many weight points should be added or removed from matching requests? | 0 (not set)                       |
+
+## Using the Generated Policy
+
+Save the output and import it in your main policy file:
+
+```yaml
+bots:
+  - import: "./filter-tor.yaml"
+```

docs/docs/admin/robots2policy.mdx

@@ -12,6 +12,7 @@ Install directly with Go:
 ```bash
 go install github.com/TecharoHQ/anubis/cmd/robots2policy@latest
 ```
+
 ## Usage
 
 Basic conversion from URL:
@@ -35,8 +36,8 @@ robots2policy -input robots.txt -action DENY -format json
 ## Options
 
 | Flag                  | Description                                                        | Default             |
-|-----------------------|--------------------------------------------------------------------|---------------------|
+| --------------------- | ------------------------------------------------------------------ | ------------------- |
-| `-input`              | robots.txt file path or URL (use `-` for stdin)                    | *required*          |
+| `-input`              | robots.txt file path or URL (use `-` for stdin)                    | _required_          |
 | `-output`             | Output file (use `-` for stdout)                                   | stdout              |
 | `-format`             | Output format: `yaml` or `json`                                    | `yaml`              |
 | `-action`             | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE`         |
@@ -47,6 +48,7 @@ robots2policy -input robots.txt -action DENY -format json
 ## Example
 
 Input robots.txt:
+
 ```txt
 User-agent: *
 Disallow: /admin/
@@ -57,6 +59,7 @@ Disallow: /
 ```
 
 Generated policy:
+
 ```yaml
 - name: robots-txt-policy-disallow-1
   action: CHALLENGE
@@ -77,8 +80,8 @@ Generated policy:
 Save the output and import it in your main policy file:
 
 ```yaml
-import:
+bots:
-  - path: "./robots-policy.yaml"
+  - import: "./robots-policy.yaml"
 ```
 
 The tool handles wildcard patterns, user-agent specific rules, and blacklisted bots automatically.

docs/docs/index.mdx

@@ -29,6 +29,9 @@ Anubis is brought to you by sponsors and donors like:
     height="64"
   />
 </a>
+<a href="https://databento.com/?utm_source=anubis&utm_medium=sponsor&utm_campaign=anubis">
+  <img src="/img/sponsors/databento-logo.webp" alt="Databento" />
+</a>
 
 ### Gold Tier
 

lib/localization/localization.go

@@ -81,7 +81,28 @@ func (ls *LocalizationService) GetLocalizerFromRequest(r *http.Request) *i18n.Lo
 		return i18n.NewLocalizer(bundle, "en")
 	}
 	acceptLanguage := r.Header.Get("Accept-Language")
-	return i18n.NewLocalizer(ls.bundle, acceptLanguage, "en")
+
+	// Parse Accept-Language header to properly handle quality factors
+	// The language.ParseAcceptLanguage function returns tags sorted by quality
+	tags, _, err := language.ParseAcceptLanguage(acceptLanguage)
+	if err != nil || len(tags) == 0 {
+		return i18n.NewLocalizer(ls.bundle, "en")
+	}
+
+	// Convert parsed tags to strings for the localizer
+	// We include both the full tag and base language to ensure proper matching
+	langs := make([]string, 0, len(tags)*2+1)
+	for _, tag := range tags {
+		langs = append(langs, tag.String())
+		// Also add base language (e.g., "en" for "en-GB") to help matching
+		base, _ := tag.Base()
+		if base.String() != tag.String() {
+			langs = append(langs, base.String())
+		}
+	}
+	langs = append(langs, "en") // Always include English as fallback
+
+	return i18n.NewLocalizer(ls.bundle, langs...)
 }
 
 // SimpleLocalizer wraps i18n.Localizer with a more convenient API

lib/localization/localization_test.go

@@ -3,6 +3,7 @@ package localization
 import (
 	"encoding/json"
 	"fmt"
+	"net/http/httptest"
 	"sort"
 	"testing"
 
@@ -138,3 +139,40 @@ func TestComprehensiveTranslations(t *testing.T) {
 		})
 	}
 }
+
+func TestAcceptLanguageQualityFactors(t *testing.T) {
+	service := NewLocalizationService()
+
+	testCases := []struct {
+		name           string
+		acceptLanguage string
+		expectedLang   string
+	}{
+		{"simple_en", "en", "en"},
+		{"simple_de", "de", "de"},
+		{"en_GB_with_lower_priority_de", "en-GB,de-DE;q=0.5", "en"},
+		{"en_GB_only", "en-GB", "en"},
+		{"de_with_lower_priority_en", "de,en;q=0.5", "de"},
+		{"de_DE_with_lower_priority_en", "de-DE,en;q=0.5", "de"},
+		{"fr_with_lower_priority_de", "fr,de;q=0.5", "fr"},
+		{"zh_CN_regional", "zh-CN", "zh-CN"},
+		{"zh_TW_regional", "zh-TW", "zh-TW"},
+		{"pt_BR_regional", "pt-BR", "pt-BR"},
+		{"complex_header", "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7,de;q=0.5", "fr"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", "/", nil)
+			req.Header.Set("Accept-Language", tc.acceptLanguage)
+
+			localizer := service.GetLocalizerFromRequest(req)
+			sl := &SimpleLocalizer{Localizer: localizer}
+
+			gotLang := sl.GetLang()
+			if gotLang != tc.expectedLang {
+				t.Errorf("Accept-Language %q: expected %s, got %s", tc.acceptLanguage, tc.expectedLang, gotLang)
+			}
+		})
+	}
+}

test/docker-registry/docker-compose.yaml

@@ -14,7 +14,7 @@ services:
     ports:
       - 3004:3004
     volumes:
-      - ../pki/registry.local.cetacean.club:/etc/techaro/pki/registry.local.cetacean.club
+      - ./pki/registry.local.cetacean.club:/etc/techaro/pki/registry.local.cetacean.club
 
   anubis:
     image: ko.local/anubis

test/lib/lib.sh

@@ -1,6 +1,9 @@
 REPO_ROOT=$(git rev-parse --show-toplevel)
 (cd $REPO_ROOT && go install ./utils/cmd/...)
 
+mkdir -p pki
+echo '*' >>./pki/.gitignore
+
 function cleanup() {
 	set +e
 
@@ -40,10 +43,10 @@ function mint_cert() {
 	domainName="$1"
 
 	# If the transient local TLS certificate doesn't exist, mint a new one
-  if [ ! -f "${REPO_ROOT}/test/pki/${domainName}/cert.pem" ]; then
+	if [ ! -f "./pki/${domainName}/cert.pem" ]; then
 		# Subshell to contain the directory change
 		(
-      cd ${REPO_ROOT}/test/pki &&
+			cd ./pki &&
 				mkdir -p "${domainName}" &&
 				go tool minica -domains "${domainName}" &&
 				cd "${domainName}" &&

test/nginx/test.sh

@@ -1,24 +1,17 @@
 #!/usr/bin/env bash
 
-export VERSION=$GITHUB_COMMIT-test
-export KO_DOCKER_REPO=ko.local
-
 source ../lib/lib.sh
 
+export KO_DOCKER_REPO=ko.local
+
 set -euo pipefail
 
-build_anubis_ko
 mint_cert mimi.techaro.lol
 
 docker run --rm \
-	-v ./conf/nginx:/etc/nginx:ro \
+	-v $PWD/conf/nginx:/etc/nginx:ro \
-	-v ../pki:/techaro/pki:ro \
+	-v $PWD/pki:/techaro/pki:ro \
 	nginx \
 	nginx -t
 
-docker compose up -d
-
-docker compose down -t 1 || :
-docker compose rm -f || :
-
 exit 0

test/palemoon/amd64/docker-compose.yml

@@ -26,7 +26,7 @@ services:
       KEY_FNAME: key.pem
       PROXY_TO: http://anubis:3000
     volumes:
-      - ../../pki/relayd:/techaro/pki:ro
+      - ./pki/relayd:/techaro/pki:ro
 
   # novnc:
   #   image: geek1011/easy-novnc
@@ -42,7 +42,7 @@ services:
     environment:
       DISPLAY: display:0
     volumes:
-      - ../../pki:/usr/local/share/ca-certificates/minica:ro
+      - ./pki:/usr/local/share/ca-certificates/minica:ro
       - ../scripts:/hack/scripts:ro
     depends_on:
       - anubis

test/palemoon/i386/docker-compose.yml

@@ -24,7 +24,7 @@ services:
       KEY_FNAME: key.pem
       PROXY_TO: http://anubis:3000
     volumes:
-      - ../../pki/relayd:/techaro/pki:ro
+      - ./pki/relayd:/techaro/pki:ro
 
   # novnc:
   #   image: geek1011/easy-novnc
@@ -40,5 +40,5 @@ services:
     environment:
       DISPLAY: display:0
     volumes:
-      - ../../pki:/usr/local/share/ca-certificates/minica:ro
+      - ./pki:/usr/local/share/ca-certificates/minica:ro
       - ../scripts:/hack/scripts:ro

utils/cmd/iplist2rule/blocklist.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/http"
+	"net/netip"
+	"strings"
+)
+
+// FetchBlocklist reads the blocklist over HTTP and returns every non-commented
+// line parsed as an IP address in CIDR notation. IPv4 addresses are returned as
+// /32, IPv6 addresses as /128.
+//
+// This function was generated with GLM 4.7.
+func FetchBlocklist(url string) ([]string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status)
+	}
+
+	var lines []string
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// Skip empty lines and comments (lines starting with #)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		addr, err := netip.ParseAddr(line)
+		if err != nil {
+			// Skip lines that aren't valid IP addresses
+			continue
+		}
+
+		var cidr string
+		if addr.Is4() {
+			cidr = fmt.Sprintf("%s/32", addr.String())
+		} else {
+			cidr = fmt.Sprintf("%s/128", addr.String())
+		}
+		lines = append(lines, cidr)
+	}
+
+	if err := scanner.Err(); err != nil && err != io.EOF {
+		return nil, err
+	}
+
+	return lines, nil
+}

utils/cmd/iplist2rule/main.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/TecharoHQ/anubis/lib/config"
+	"github.com/facebookgo/flagenv"
+	"sigs.k8s.io/yaml"
+)
+
+type Rule struct {
+	Name       string         `yaml:"name" json:"name"`
+	Action     config.Rule    `yaml:"action" json:"action"`
+	RemoteAddr []string       `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
+	Weight     *config.Weight `json:"weight,omitempty" yaml:"weight,omitempty"`
+}
+
+func init() {
+	flag.Usage = func() {
+		fmt.Printf(`Usage of %[1]s:
+
+	%[1]s [flags] <blocklist-url> <filename>
+
+Grabs the contents of the blocklist, converts it to an Anubis ruleset, and writes it to filename.
+
+Flags:
+`, filepath.Base(os.Args[0]))
+
+		flag.PrintDefaults()
+	}
+}
+
+var (
+	action         = flag.String("action", "DENY", "Anubis action to take (ALLOW / DENY / WEIGH)")
+	manualRuleName = flag.String("rule-name", "", "If set, prefer this name over inferring from filename")
+	weight         = flag.Int("weight", 0, "If set to any number, add/subtract this many weight points when --action=WEIGH")
+)
+
+func main() {
+	flagenv.Parse()
+	flag.Parse()
+
+	if flag.NArg() != 2 {
+		flag.Usage()
+		os.Exit(2)
+	}
+
+	blocklistURL := flag.Arg(0)
+	foutName := flag.Arg(1)
+	ruleName := strings.TrimSuffix(foutName, filepath.Ext(foutName))
+
+	if *manualRuleName != "" {
+		ruleName = *manualRuleName
+	}
+
+	ruleAction := config.Rule(*action)
+	if err := ruleAction.Valid(); err != nil {
+		log.Fatalf("--action=%q is invalid: %v", *action, err)
+	}
+
+	result := &Rule{
+		Name:   ruleName,
+		Action: ruleAction,
+	}
+
+	if *weight != 0 {
+		if ruleAction != config.RuleWeigh {
+			log.Fatalf("used --weight=%d but --action=%s", *weight, *action)
+		}
+
+		result.Weight = &config.Weight{
+			Adjust: *weight,
+		}
+	}
+
+	ips, err := FetchBlocklist(blocklistURL)
+	if err != nil {
+		log.Fatalf("can't fetch blocklist %s: %v", blocklistURL, err)
+	}
+
+	result.RemoteAddr = ips
+
+	fout, err := os.Create(foutName)
+	if err != nil {
+		log.Fatalf("can't create output file %q: %v", foutName, err)
+	}
+	defer fout.Close()
+
+	fmt.Fprintf(fout, "# Generated by %s on %s from %s\n\n", filepath.Base(os.Args[0]), time.Now().Format(time.RFC3339), blocklistURL)
+
+	data, err := yaml.Marshal([]*Rule{result})
+	if err != nil {
+		log.Fatalf("can't marshal yaml")
+	}
+
+	fout.Write(data)
+}