arian/anubis-mirror
1
0
Fork 0
mirror of https://github.com/TecharoHQ/anubis.git synced 2026-04-11 11:08:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: collate

Browse Source
This commit is contained in:
Jason Cameron
2025-07-26 20:34:04 -04:00
parent 291ed2a084
commit e51b4bd965
7 changed files with 280 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
cmd/robots2policy/testdata/consecutive.yaml vendored Normal file
View File

@@ -0,0 +1,47 @@
- action: WEIGH
expression: "true"
name: robots-txt-policy-crawl-delay-1
weight:
adjust: 3
- action: CHALLENGE
expression: path.startsWith("/admin")
name: robots-txt-policy-disallow-2
- action: DENY
expression:
any:
- userAgent.contains("BadBot")
- userAgent.contains("SpamBot")
- userAgent.contains("EvilBot")
name: robots-txt-policy-blacklist-3
- action: CHALLENGE
expression:
all:
- userAgent.contains("GoodBot")
- path.startsWith("/private")
name: robots-txt-policy-disallow-4
- action: WEIGH
expression:
any:
- userAgent.contains("SlowBot1")
- userAgent.contains("SlowBot2")
name: robots-txt-policy-crawl-delay-5
weight:
adjust: 3
- action: CHALLENGE
expression:
all:
- userAgent.contains("SearchBot1")
- path.startsWith("/search")
name: robots-txt-policy-disallow-7
- action: CHALLENGE
expression:
all:
- userAgent.contains("SearchBot2")
- path.startsWith("/search")
name: robots-txt-policy-disallow-8
- action: CHALLENGE
expression:
all:
- userAgent.contains("SearchBot3")
- path.startsWith("/search")
name: robots-txt-policy-disallow-9