arian/anubis-mirror
1
0
Fork 0
mirror of https://github.com/TecharoHQ/anubis.git synced 2026-04-27 10:32:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: handle multiple user agents

Browse Source
This commit is contained in:
Jason Cameron
2025-07-26 20:27:00 -04:00
parent a735770c93
commit 291ed2a084
5 changed files with 269 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/robots2policy/testdata/blacklist.yaml
Vendored
+18 -18
View File
@@ -3,28 +3,28 @@
name: robots-txt-policy-crawl-delay-1
weight:
adjust: 3
- action: CHALLENGE
expression: path.startsWith("/admin")
name: robots-txt-policy-disallow-2
- action: DENY
expression: userAgent.contains("BadBot")
name: robots-txt-policy-blacklist-3
- action: WEIGH
expression: userAgent.contains("SpamBot")
name: robots-txt-policy-crawl-delay-4
weight:
adjust: 3
- action: DENY
expression: userAgent.contains("SpamBot")
name: robots-txt-policy-blacklist-5
- action: WEIGH
expression: userAgent.contains("Googlebot")
name: robots-txt-policy-crawl-delay-6
name: robots-txt-policy-crawl-delay-2
weight:
adjust: 3
- action: WEIGH
expression: userAgent.contains("SpamBot")
name: robots-txt-policy-crawl-delay-3
weight:
adjust: 3
- action: DENY
expression:
any:
- userAgent.contains("BadBot")
- userAgent.contains("SpamBot")
name: robots-txt-policy-blacklist-4
- action: CHALLENGE
expression: path.startsWith("/admin")
name: robots-txt-policy-disallow-5
- action: CHALLENGE
expression:
all:
- userAgent.contains("Googlebot")
- path.startsWith("/search")
name: robots-txt-policy-disallow-7
- userAgent.contains("Googlebot")
- path.startsWith("/search")
name: robots-txt-policy-disallow-6