mirror of
https://github.com/TecharoHQ/anubis.git
synced 2026-05-03 13:52:42 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 80a902229c | |||
| 158698827d | |||
| 8e80c9c833 | |||
| b8d64093d3 | |||
| 3990e73134 |
@@ -18,8 +18,3 @@ clampip
|
|||||||
pseudoprofound
|
pseudoprofound
|
||||||
reimagining
|
reimagining
|
||||||
iocaine
|
iocaine
|
||||||
admins
|
|
||||||
fout
|
|
||||||
iplist
|
|
||||||
NArg
|
|
||||||
blocklists
|
|
||||||
|
|||||||
@@ -87,14 +87,10 @@
|
|||||||
^docs/docs/user/known-instances.md$
|
^docs/docs/user/known-instances.md$
|
||||||
^docs/manifest/.*$
|
^docs/manifest/.*$
|
||||||
^docs/static/\.nojekyll$
|
^docs/static/\.nojekyll$
|
||||||
^internal/glob/glob_test.go$
|
|
||||||
^internal/honeypot/naive/affirmations\.txt$
|
|
||||||
^internal/honeypot/naive/spintext\.txt$
|
|
||||||
^internal/honeypot/naive/titles\.txt$
|
|
||||||
^lib/config/testdata/bad/unparseable\.json$
|
|
||||||
^lib/localization/.*_test.go$
|
|
||||||
^lib/localization/locales/.*\.json$
|
|
||||||
^lib/policy/config/testdata/bad/unparseable\.json$
|
^lib/policy/config/testdata/bad/unparseable\.json$
|
||||||
^test/.*$
|
^internal/glob/glob_test.go$
|
||||||
ignore$
|
ignore$
|
||||||
robots.txt
|
robots.txt
|
||||||
|
^lib/localization/locales/.*\.json$
|
||||||
|
^lib/localization/.*_test.go$
|
||||||
|
^test/.*$
|
||||||
|
|||||||
+409
-406
@@ -1,406 +1,409 @@
|
|||||||
acs
|
acs
|
||||||
Actorified
|
Actorified
|
||||||
actorifiedstore
|
actorifiedstore
|
||||||
actorify
|
actorify
|
||||||
Aibrew
|
Aibrew
|
||||||
alibaba
|
alibaba
|
||||||
alrest
|
alrest
|
||||||
amazonbot
|
amazonbot
|
||||||
anthro
|
anthro
|
||||||
anubis
|
anubis
|
||||||
anubistest
|
anubistest
|
||||||
apnic
|
apnic
|
||||||
APNICRANDNETAU
|
APNICRANDNETAU
|
||||||
Applebot
|
Applebot
|
||||||
archlinux
|
archlinux
|
||||||
arpa
|
arpa
|
||||||
asnc
|
asnc
|
||||||
asnchecker
|
asnchecker
|
||||||
asns
|
asns
|
||||||
aspirational
|
aspirational
|
||||||
atuin
|
atuin
|
||||||
azuretools
|
azuretools
|
||||||
badregexes
|
badregexes
|
||||||
bbolt
|
bbolt
|
||||||
bdba
|
bdba
|
||||||
berr
|
berr
|
||||||
bezier
|
bezier
|
||||||
bingbot
|
bingbot
|
||||||
Bitcoin
|
Bitcoin
|
||||||
bitrate
|
bitrate
|
||||||
Bluesky
|
Bluesky
|
||||||
blueskybot
|
blueskybot
|
||||||
boi
|
boi
|
||||||
Bokm
|
Bokm
|
||||||
botnet
|
botnet
|
||||||
botstopper
|
botstopper
|
||||||
BPort
|
BPort
|
||||||
Brightbot
|
Brightbot
|
||||||
broked
|
broked
|
||||||
buildah
|
buildah
|
||||||
byteslice
|
byteslice
|
||||||
Bytespider
|
Bytespider
|
||||||
cachebuster
|
cachebuster
|
||||||
cachediptoasn
|
cachediptoasn
|
||||||
Caddyfile
|
Caddyfile
|
||||||
caninetools
|
caninetools
|
||||||
Cardyb
|
Cardyb
|
||||||
celchecker
|
celchecker
|
||||||
celphase
|
celphase
|
||||||
cerr
|
cerr
|
||||||
certresolver
|
certresolver
|
||||||
cespare
|
cespare
|
||||||
CGNAT
|
CGNAT
|
||||||
cgr
|
cgr
|
||||||
chainguard
|
chainguard
|
||||||
chall
|
chall
|
||||||
challengemozilla
|
challengemozilla
|
||||||
challengetest
|
challengetest
|
||||||
checkpath
|
checkpath
|
||||||
checkresult
|
checkresult
|
||||||
chibi
|
chibi
|
||||||
cidranger
|
cidranger
|
||||||
ckie
|
ckie
|
||||||
cloudflare
|
cloudflare
|
||||||
Codespaces
|
Codespaces
|
||||||
confd
|
confd
|
||||||
connnection
|
connnection
|
||||||
containerbuild
|
containerbuild
|
||||||
containerregistry
|
containerregistry
|
||||||
coreutils
|
coreutils
|
||||||
Cotoyogi
|
Cotoyogi
|
||||||
Cromite
|
Cromite
|
||||||
crt
|
crt
|
||||||
Cscript
|
Cscript
|
||||||
daemonizing
|
daemonizing
|
||||||
databento
|
dayjob
|
||||||
dayjob
|
DDOS
|
||||||
DDOS
|
Debian
|
||||||
Debian
|
debrpm
|
||||||
debrpm
|
decaymap
|
||||||
decaymap
|
devcontainers
|
||||||
devcontainers
|
Diffbot
|
||||||
Diffbot
|
discordapp
|
||||||
discordapp
|
discordbot
|
||||||
discordbot
|
distros
|
||||||
distros
|
dnf
|
||||||
dnf
|
dnsbl
|
||||||
dnsbl
|
dnserr
|
||||||
dnserr
|
DNSTTL
|
||||||
DNSTTL
|
domainhere
|
||||||
domainhere
|
dracula
|
||||||
dracula
|
dronebl
|
||||||
dronebl
|
droneblresponse
|
||||||
droneblresponse
|
dropin
|
||||||
dropin
|
dsilence
|
||||||
dsilence
|
duckduckbot
|
||||||
duckduckbot
|
eerror
|
||||||
eerror
|
ellenjoe
|
||||||
ellenjoe
|
emacs
|
||||||
emacs
|
enbyware
|
||||||
enbyware
|
etld
|
||||||
etld
|
everyones
|
||||||
everyones
|
evilbot
|
||||||
evilbot
|
evilsite
|
||||||
evilsite
|
expressionorlist
|
||||||
expressionorlist
|
externalagent
|
||||||
externalagent
|
externalfetcher
|
||||||
externalfetcher
|
extldflags
|
||||||
extldflags
|
facebookgo
|
||||||
facebookgo
|
Factset
|
||||||
Factset
|
fahedouch
|
||||||
fahedouch
|
fastcgi
|
||||||
fastcgi
|
FCr
|
||||||
FCr
|
fcrdns
|
||||||
fcrdns
|
fediverse
|
||||||
fediverse
|
ffprobe
|
||||||
ffprobe
|
financials
|
||||||
financials
|
finfos
|
||||||
finfos
|
Firecrawl
|
||||||
Firecrawl
|
flagenv
|
||||||
flagenv
|
Fordola
|
||||||
Fordola
|
forgejo
|
||||||
forgejo
|
forwardauth
|
||||||
forwardauth
|
fsys
|
||||||
fsys
|
fullchain
|
||||||
fullchain
|
gaissmai
|
||||||
gaissmai
|
Galvus
|
||||||
Galvus
|
geoip
|
||||||
geoip
|
geoipchecker
|
||||||
geoipchecker
|
gha
|
||||||
gha
|
GHSA
|
||||||
GHSA
|
Ghz
|
||||||
Ghz
|
gipc
|
||||||
gipc
|
gitea
|
||||||
gitea
|
godotenv
|
||||||
GLM
|
goland
|
||||||
godotenv
|
gomod
|
||||||
goland
|
goodbot
|
||||||
gomod
|
googlebot
|
||||||
goodbot
|
gopsutil
|
||||||
googlebot
|
govulncheck
|
||||||
gopsutil
|
goyaml
|
||||||
govulncheck
|
GPG
|
||||||
goyaml
|
GPT
|
||||||
GPG
|
gptbot
|
||||||
GPT
|
Graphene
|
||||||
gptbot
|
grpcprom
|
||||||
Graphene
|
grw
|
||||||
grpcprom
|
gzw
|
||||||
grw
|
Hashcash
|
||||||
gzw
|
hashrate
|
||||||
Hashcash
|
headermap
|
||||||
hashrate
|
healthcheck
|
||||||
headermap
|
healthz
|
||||||
healthcheck
|
hec
|
||||||
healthz
|
helpdesk
|
||||||
hec
|
Hetzner
|
||||||
helpdesk
|
hmc
|
||||||
Hetzner
|
homelab
|
||||||
hmc
|
hostable
|
||||||
homelab
|
htmlc
|
||||||
hostable
|
htmx
|
||||||
htmlc
|
httpdebug
|
||||||
htmx
|
huawei
|
||||||
httpdebug
|
hypertext
|
||||||
huawei
|
iaskspider
|
||||||
hypertext
|
iaso
|
||||||
iaskspider
|
iat
|
||||||
iaso
|
ifm
|
||||||
iat
|
Imagesift
|
||||||
ifm
|
imgproxy
|
||||||
Imagesift
|
impressum
|
||||||
imgproxy
|
inbox
|
||||||
impressum
|
ingressed
|
||||||
inbox
|
inp
|
||||||
ingressed
|
internets
|
||||||
inp
|
IPTo
|
||||||
internets
|
iptoasn
|
||||||
IPTo
|
isp
|
||||||
iptoasn
|
iss
|
||||||
isp
|
isset
|
||||||
iss
|
ivh
|
||||||
isset
|
Jenomis
|
||||||
ivh
|
JGit
|
||||||
Jenomis
|
jhjj
|
||||||
JGit
|
joho
|
||||||
jhjj
|
journalctl
|
||||||
joho
|
jshelter
|
||||||
journalctl
|
JWTs
|
||||||
jshelter
|
kagi
|
||||||
JWTs
|
kagibot
|
||||||
kagi
|
Keyfunc
|
||||||
kagibot
|
keypair
|
||||||
Keyfunc
|
KHTML
|
||||||
keypair
|
kinda
|
||||||
KHTML
|
KUBECONFIG
|
||||||
kinda
|
lcj
|
||||||
KUBECONFIG
|
ldflags
|
||||||
lcj
|
letsencrypt
|
||||||
ldflags
|
Lexentale
|
||||||
letsencrypt
|
lfc
|
||||||
Lexentale
|
lgbt
|
||||||
lfc
|
licend
|
||||||
lgbt
|
licstart
|
||||||
licend
|
lightpanda
|
||||||
licstart
|
limsa
|
||||||
lightpanda
|
Linting
|
||||||
limsa
|
listor
|
||||||
Linting
|
LLU
|
||||||
listor
|
loadbalancer
|
||||||
LLU
|
lol
|
||||||
loadbalancer
|
lominsa
|
||||||
lol
|
maintainership
|
||||||
lominsa
|
malware
|
||||||
maintainership
|
mcr
|
||||||
malware
|
memes
|
||||||
mcr
|
metarefresh
|
||||||
memes
|
metrix
|
||||||
metarefresh
|
mimi
|
||||||
metrix
|
Minfilia
|
||||||
mimi
|
mistralai
|
||||||
Minfilia
|
mnt
|
||||||
mistralai
|
Mojeek
|
||||||
mnt
|
mojeekbot
|
||||||
Mojeek
|
mozilla
|
||||||
mojeekbot
|
myclient
|
||||||
mozilla
|
mymaster
|
||||||
myclient
|
mypass
|
||||||
mymaster
|
myuser
|
||||||
mypass
|
nbf
|
||||||
myuser
|
nepeat
|
||||||
nbf
|
netsurf
|
||||||
nepeat
|
nginx
|
||||||
netsurf
|
nicksnyder
|
||||||
nginx
|
nobots
|
||||||
nicksnyder
|
NONINFRINGEMENT
|
||||||
nikandfor
|
nosleep
|
||||||
nobots
|
nullglob
|
||||||
NONINFRINGEMENT
|
oci
|
||||||
nosleep
|
OCOB
|
||||||
nullglob
|
ogtag
|
||||||
oci
|
oklch
|
||||||
OCOB
|
omgili
|
||||||
ogtag
|
omgilibot
|
||||||
oklch
|
openai
|
||||||
omgili
|
opendns
|
||||||
omgilibot
|
opengraph
|
||||||
openai
|
openrc
|
||||||
opendns
|
oswald
|
||||||
opengraph
|
pag
|
||||||
openrc
|
palemoon
|
||||||
oswald
|
Pangu
|
||||||
pag
|
parseable
|
||||||
pagegen
|
passthrough
|
||||||
palemoon
|
Patreon
|
||||||
Pangu
|
pgrep
|
||||||
parseable
|
phrik
|
||||||
passthrough
|
pidfile
|
||||||
Patreon
|
pids
|
||||||
pgrep
|
pipefail
|
||||||
phrik
|
pki
|
||||||
pidfile
|
podkova
|
||||||
pids
|
podman
|
||||||
pipefail
|
Postgre
|
||||||
pki
|
poststart
|
||||||
podkova
|
prebaked
|
||||||
podman
|
privkey
|
||||||
Postgre
|
promauto
|
||||||
poststart
|
promhttp
|
||||||
prebaked
|
proofofwork
|
||||||
privkey
|
publicsuffix
|
||||||
promauto
|
purejs
|
||||||
promhttp
|
pwcmd
|
||||||
proofofwork
|
pwuser
|
||||||
publicsuffix
|
qualys
|
||||||
purejs
|
qwant
|
||||||
pwcmd
|
qwantbot
|
||||||
pwuser
|
rac
|
||||||
qualys
|
rawler
|
||||||
qwant
|
rcvar
|
||||||
qwantbot
|
redhat
|
||||||
rac
|
redir
|
||||||
rawler
|
redirectscheme
|
||||||
rcvar
|
refactors
|
||||||
redhat
|
remoteip
|
||||||
redir
|
reputational
|
||||||
redirectscheme
|
risc
|
||||||
refactors
|
ruleset
|
||||||
remoteip
|
runlevels
|
||||||
reputational
|
RUnlock
|
||||||
Rhul
|
runtimedir
|
||||||
risc
|
runtimedirectory
|
||||||
ruleset
|
Ryzen
|
||||||
runlevels
|
sas
|
||||||
RUnlock
|
sasl
|
||||||
runtimedir
|
screenshots
|
||||||
runtimedirectory
|
searchbot
|
||||||
Ryzen
|
searx
|
||||||
sas
|
sebest
|
||||||
sasl
|
secretplans
|
||||||
screenshots
|
Semrush
|
||||||
searchbot
|
Seo
|
||||||
searx
|
setsebool
|
||||||
sebest
|
shellcheck
|
||||||
secretplans
|
shirou
|
||||||
Semrush
|
shopt
|
||||||
Seo
|
Sidetrade
|
||||||
setsebool
|
simprint
|
||||||
shellcheck
|
sitemap
|
||||||
shirou
|
sls
|
||||||
shoneypot
|
sni
|
||||||
shopt
|
snipster
|
||||||
Sidetrade
|
Spambot
|
||||||
simprint
|
sparkline
|
||||||
sitemap
|
spyderbot
|
||||||
sls
|
srv
|
||||||
sni
|
stackoverflow
|
||||||
snipster
|
startprecmd
|
||||||
Spambot
|
stoppostcmd
|
||||||
spammer
|
storetest
|
||||||
sparkline
|
subgrid
|
||||||
spyderbot
|
subr
|
||||||
srv
|
subrequest
|
||||||
stackoverflow
|
SVCNAME
|
||||||
startprecmd
|
tagline
|
||||||
stoppostcmd
|
tarballs
|
||||||
storetest
|
tarrif
|
||||||
subgrid
|
taviso
|
||||||
subr
|
tbn
|
||||||
subrequest
|
tbr
|
||||||
SVCNAME
|
techaro
|
||||||
tagline
|
techarohq
|
||||||
tarballs
|
telegrambot
|
||||||
tarrif
|
templ
|
||||||
taviso
|
templruntime
|
||||||
tbn
|
testarea
|
||||||
tbr
|
Thancred
|
||||||
techaro
|
thoth
|
||||||
techarohq
|
thothmock
|
||||||
telegrambot
|
Tik
|
||||||
templ
|
Timpibot
|
||||||
templruntime
|
TLog
|
||||||
testarea
|
traefik
|
||||||
Thancred
|
trunc
|
||||||
thoth
|
uberspace
|
||||||
thothmock
|
Unbreak
|
||||||
Tik
|
unbreakdocker
|
||||||
Timpibot
|
unifiedjs
|
||||||
TLog
|
unmarshal
|
||||||
traefik
|
unparseable
|
||||||
trunc
|
uvx
|
||||||
uberspace
|
UXP
|
||||||
Unbreak
|
valkey
|
||||||
unbreakdocker
|
Varis
|
||||||
unifiedjs
|
Velen
|
||||||
unmarshal
|
vendored
|
||||||
unparseable
|
verify
|
||||||
uvx
|
vhosts
|
||||||
UXP
|
vkbot
|
||||||
valkey
|
VKE
|
||||||
Varis
|
vnd
|
||||||
Velen
|
VPS
|
||||||
vendored
|
Vultr
|
||||||
vhosts
|
weblate
|
||||||
vkbot
|
webmaster
|
||||||
VKE
|
webpage
|
||||||
vnd
|
websecure
|
||||||
VPS
|
websites
|
||||||
Vultr
|
Webzio
|
||||||
weblate
|
whois
|
||||||
webmaster
|
wildbase
|
||||||
webpage
|
withthothmock
|
||||||
websecure
|
wolfbeast
|
||||||
websites
|
wordpress
|
||||||
Webzio
|
workaround
|
||||||
whois
|
workdir
|
||||||
wildbase
|
wpbot
|
||||||
withthothmock
|
XCircle
|
||||||
wolfbeast
|
xeiaso
|
||||||
wordpress
|
xeserv
|
||||||
workaround
|
xesite
|
||||||
workdir
|
xess
|
||||||
wpbot
|
xff
|
||||||
XCircle
|
XForwarded
|
||||||
xeiaso
|
XNG
|
||||||
xeserv
|
XOB
|
||||||
xesite
|
XOriginal
|
||||||
xess
|
XReal
|
||||||
xff
|
yae
|
||||||
XForwarded
|
YAMLTo
|
||||||
XNG
|
Yda
|
||||||
XOB
|
yeet
|
||||||
XOriginal
|
yeetfile
|
||||||
XReal
|
yourdomain
|
||||||
Y'shtola
|
yyz
|
||||||
yae
|
Zenos
|
||||||
YAMLTo
|
zizmor
|
||||||
Yda
|
zombocom
|
||||||
yeet
|
zos
|
||||||
yeetfile
|
GLM
|
||||||
yourdomain
|
iocaine
|
||||||
yyz
|
nikandfor
|
||||||
Zenos
|
pagegen
|
||||||
zizmor
|
pseudoprofound
|
||||||
zombocom
|
reimagining
|
||||||
zos
|
Rhul
|
||||||
|
shoneypot
|
||||||
|
spammer
|
||||||
|
Y'shtola
|
||||||
|
|||||||
@@ -20,9 +20,6 @@ Anubis is brought to you by sponsors and donors like:
|
|||||||
<a href="https://www.raptorcs.com/content/base/products.html">
|
<a href="https://www.raptorcs.com/content/base/products.html">
|
||||||
<img src="./docs/static/img/sponsors/raptor-computing-logo.webp" alt="Raptor Computing Systems" height=64 />
|
<img src="./docs/static/img/sponsors/raptor-computing-logo.webp" alt="Raptor Computing Systems" height=64 />
|
||||||
</a>
|
</a>
|
||||||
<a href="https://databento.com/?utm_source=anubis&utm_medium=sponsor&utm_campaign=anubis">
|
|
||||||
<img src="./docs/static/img/sponsors/databento-logo.webp" alt="Databento" />
|
|
||||||
</a>
|
|
||||||
|
|
||||||
### Gold Tier
|
### Gold Tier
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,5 @@
|
|||||||
- name: qualys-ssl-labs
|
- name: qualys-ssl-labs
|
||||||
action: ALLOW
|
action: ALLOW
|
||||||
remote_addresses:
|
remote_addresses:
|
||||||
- 69.67.183.0/24
|
- 64.41.200.0/24
|
||||||
- 2600:C02:1020:4202::/64
|
- 2600:C02:1020:4202::/64
|
||||||
- 2602:fdaa:c6:2::/64
|
|
||||||
@@ -11,7 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
- Add iplist2rule tool that lets admins turn an IP address blocklist into an Anubis ruleset.
|
|
||||||
- Add Polish locale ([#1292](https://github.com/TecharoHQ/anubis/pull/1309))
|
- Add Polish locale ([#1292](https://github.com/TecharoHQ/anubis/pull/1309))
|
||||||
|
|
||||||
<!-- This changes the project to: -->
|
<!-- This changes the project to: -->
|
||||||
|
|||||||
@@ -1,50 +0,0 @@
|
|||||||
---
|
|
||||||
title: iplist2rule CLI tool
|
|
||||||
---
|
|
||||||
|
|
||||||
The `iplist2rule` tool converts IP blocklists into Anubis challenge policies. It reads common IP block list formats and generates the appropriate Anubis policy file for IP address filtering.
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
Install directly with Go
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go install github.com/TecharoHQ/anubis/utils/cmd/iplist2rule@latest
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
Basic conversion from URL:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
iplist2rule https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
Explicitly allow every IP address on a list:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
iplist2rule --action ALLOW https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
Add weight to requests matching IP addresses on a list:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
iplist2rule --action WEIGH --weight 20 https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
## Options
|
|
||||||
|
|
||||||
| Flag | Description | Default |
|
|
||||||
| :------------ | :----------------------------------------------------------------------------------------------- | :-------------------------------- |
|
|
||||||
| `--action` | The Anubis action to take for the IP address in question, must be in ALL CAPS. | `DENY` (forbids traffic) |
|
|
||||||
| `--rule-name` | The name for the generated Anubis rule, should be in kebab-case. | (not set, inferred from filename) |
|
|
||||||
| `--weight` | When `--action=WEIGH`, how many weight points should be added or removed from matching requests? | 0 (not set) |
|
|
||||||
|
|
||||||
## Using the Generated Policy
|
|
||||||
|
|
||||||
Save the output and import it in your main policy file:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
bots:
|
|
||||||
- import: "./filter-tor.yaml"
|
|
||||||
```
|
|
||||||
@@ -12,7 +12,6 @@ Install directly with Go:
|
|||||||
```bash
|
```bash
|
||||||
go install github.com/TecharoHQ/anubis/cmd/robots2policy@latest
|
go install github.com/TecharoHQ/anubis/cmd/robots2policy@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
Basic conversion from URL:
|
Basic conversion from URL:
|
||||||
@@ -36,8 +35,8 @@ robots2policy -input robots.txt -action DENY -format json
|
|||||||
## Options
|
## Options
|
||||||
|
|
||||||
| Flag | Description | Default |
|
| Flag | Description | Default |
|
||||||
| --------------------- | ------------------------------------------------------------------ | ------------------- |
|
|-----------------------|--------------------------------------------------------------------|---------------------|
|
||||||
| `-input` | robots.txt file path or URL (use `-` for stdin) | _required_ |
|
| `-input` | robots.txt file path or URL (use `-` for stdin) | *required* |
|
||||||
| `-output` | Output file (use `-` for stdout) | stdout |
|
| `-output` | Output file (use `-` for stdout) | stdout |
|
||||||
| `-format` | Output format: `yaml` or `json` | `yaml` |
|
| `-format` | Output format: `yaml` or `json` | `yaml` |
|
||||||
| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` |
|
| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` |
|
||||||
@@ -48,7 +47,6 @@ robots2policy -input robots.txt -action DENY -format json
|
|||||||
## Example
|
## Example
|
||||||
|
|
||||||
Input robots.txt:
|
Input robots.txt:
|
||||||
|
|
||||||
```txt
|
```txt
|
||||||
User-agent: *
|
User-agent: *
|
||||||
Disallow: /admin/
|
Disallow: /admin/
|
||||||
@@ -59,7 +57,6 @@ Disallow: /
|
|||||||
```
|
```
|
||||||
|
|
||||||
Generated policy:
|
Generated policy:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
- name: robots-txt-policy-disallow-1
|
- name: robots-txt-policy-disallow-1
|
||||||
action: CHALLENGE
|
action: CHALLENGE
|
||||||
@@ -80,8 +77,8 @@ Generated policy:
|
|||||||
Save the output and import it in your main policy file:
|
Save the output and import it in your main policy file:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
bots:
|
import:
|
||||||
- import: "./robots-policy.yaml"
|
- path: "./robots-policy.yaml"
|
||||||
```
|
```
|
||||||
|
|
||||||
The tool handles wildcard patterns, user-agent specific rules, and blacklisted bots automatically.
|
The tool handles wildcard patterns, user-agent specific rules, and blacklisted bots automatically.
|
||||||
|
|||||||
@@ -29,9 +29,6 @@ Anubis is brought to you by sponsors and donors like:
|
|||||||
height="64"
|
height="64"
|
||||||
/>
|
/>
|
||||||
</a>
|
</a>
|
||||||
<a href="https://databento.com/?utm_source=anubis&utm_medium=sponsor&utm_campaign=anubis">
|
|
||||||
<img src="/img/sponsors/databento-logo.webp" alt="Databento" />
|
|
||||||
</a>
|
|
||||||
|
|
||||||
### Gold Tier
|
### Gold Tier
|
||||||
|
|
||||||
|
|||||||
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 4.3 KiB |
@@ -81,28 +81,7 @@ func (ls *LocalizationService) GetLocalizerFromRequest(r *http.Request) *i18n.Lo
|
|||||||
return i18n.NewLocalizer(bundle, "en")
|
return i18n.NewLocalizer(bundle, "en")
|
||||||
}
|
}
|
||||||
acceptLanguage := r.Header.Get("Accept-Language")
|
acceptLanguage := r.Header.Get("Accept-Language")
|
||||||
|
return i18n.NewLocalizer(ls.bundle, acceptLanguage, "en")
|
||||||
// Parse Accept-Language header to properly handle quality factors
|
|
||||||
// The language.ParseAcceptLanguage function returns tags sorted by quality
|
|
||||||
tags, _, err := language.ParseAcceptLanguage(acceptLanguage)
|
|
||||||
if err != nil || len(tags) == 0 {
|
|
||||||
return i18n.NewLocalizer(ls.bundle, "en")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert parsed tags to strings for the localizer
|
|
||||||
// We include both the full tag and base language to ensure proper matching
|
|
||||||
langs := make([]string, 0, len(tags)*2+1)
|
|
||||||
for _, tag := range tags {
|
|
||||||
langs = append(langs, tag.String())
|
|
||||||
// Also add base language (e.g., "en" for "en-GB") to help matching
|
|
||||||
base, _ := tag.Base()
|
|
||||||
if base.String() != tag.String() {
|
|
||||||
langs = append(langs, base.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
langs = append(langs, "en") // Always include English as fallback
|
|
||||||
|
|
||||||
return i18n.NewLocalizer(ls.bundle, langs...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SimpleLocalizer wraps i18n.Localizer with a more convenient API
|
// SimpleLocalizer wraps i18n.Localizer with a more convenient API
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package localization
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http/httptest"
|
|
||||||
"sort"
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -139,40 +138,3 @@ func TestComprehensiveTranslations(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAcceptLanguageQualityFactors(t *testing.T) {
|
|
||||||
service := NewLocalizationService()
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
name string
|
|
||||||
acceptLanguage string
|
|
||||||
expectedLang string
|
|
||||||
}{
|
|
||||||
{"simple_en", "en", "en"},
|
|
||||||
{"simple_de", "de", "de"},
|
|
||||||
{"en_GB_with_lower_priority_de", "en-GB,de-DE;q=0.5", "en"},
|
|
||||||
{"en_GB_only", "en-GB", "en"},
|
|
||||||
{"de_with_lower_priority_en", "de,en;q=0.5", "de"},
|
|
||||||
{"de_DE_with_lower_priority_en", "de-DE,en;q=0.5", "de"},
|
|
||||||
{"fr_with_lower_priority_de", "fr,de;q=0.5", "fr"},
|
|
||||||
{"zh_CN_regional", "zh-CN", "zh-CN"},
|
|
||||||
{"zh_TW_regional", "zh-TW", "zh-TW"},
|
|
||||||
{"pt_BR_regional", "pt-BR", "pt-BR"},
|
|
||||||
{"complex_header", "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7,de;q=0.5", "fr"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
req := httptest.NewRequest("GET", "/", nil)
|
|
||||||
req.Header.Set("Accept-Language", tc.acceptLanguage)
|
|
||||||
|
|
||||||
localizer := service.GetLocalizerFromRequest(req)
|
|
||||||
sl := &SimpleLocalizer{Localizer: localizer}
|
|
||||||
|
|
||||||
gotLang := sl.GetLang()
|
|
||||||
if gotLang != tc.expectedLang {
|
|
||||||
t.Errorf("Accept-Language %q: expected %s, got %s", tc.acceptLanguage, tc.expectedLang, gotLang)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FetchBlocklist reads the blocklist over HTTP and returns every non-commented
|
|
||||||
// line parsed as an IP address in CIDR notation. IPv4 addresses are returned as
|
|
||||||
// /32, IPv6 addresses as /128.
|
|
||||||
//
|
|
||||||
// This function was generated with GLM 4.7.
|
|
||||||
func FetchBlocklist(url string) ([]string, error) {
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
var lines []string
|
|
||||||
scanner := bufio.NewScanner(resp.Body)
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := scanner.Text()
|
|
||||||
// Skip empty lines and comments (lines starting with #)
|
|
||||||
if line == "" || strings.HasPrefix(line, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(line)
|
|
||||||
if err != nil {
|
|
||||||
// Skip lines that aren't valid IP addresses
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var cidr string
|
|
||||||
if addr.Is4() {
|
|
||||||
cidr = fmt.Sprintf("%s/32", addr.String())
|
|
||||||
} else {
|
|
||||||
cidr = fmt.Sprintf("%s/128", addr.String())
|
|
||||||
}
|
|
||||||
lines = append(lines, cidr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := scanner.Err(); err != nil && err != io.EOF {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines, nil
|
|
||||||
}
|
|
||||||
@@ -1,103 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/TecharoHQ/anubis/lib/config"
|
|
||||||
"github.com/facebookgo/flagenv"
|
|
||||||
"sigs.k8s.io/yaml"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Rule struct {
|
|
||||||
Name string `yaml:"name" json:"name"`
|
|
||||||
Action config.Rule `yaml:"action" json:"action"`
|
|
||||||
RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
|
|
||||||
Weight *config.Weight `json:"weight,omitempty" yaml:"weight,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
flag.Usage = func() {
|
|
||||||
fmt.Printf(`Usage of %[1]s:
|
|
||||||
|
|
||||||
%[1]s [flags] <blocklist-url> <filename>
|
|
||||||
|
|
||||||
Grabs the contents of the blocklist, converts it to an Anubis ruleset, and writes it to filename.
|
|
||||||
|
|
||||||
Flags:
|
|
||||||
`, filepath.Base(os.Args[0]))
|
|
||||||
|
|
||||||
flag.PrintDefaults()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
action = flag.String("action", "DENY", "Anubis action to take (ALLOW / DENY / WEIGH)")
|
|
||||||
manualRuleName = flag.String("rule-name", "", "If set, prefer this name over inferring from filename")
|
|
||||||
weight = flag.Int("weight", 0, "If set to any number, add/subtract this many weight points when --action=WEIGH")
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
flagenv.Parse()
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if flag.NArg() != 2 {
|
|
||||||
flag.Usage()
|
|
||||||
os.Exit(2)
|
|
||||||
}
|
|
||||||
|
|
||||||
blocklistURL := flag.Arg(0)
|
|
||||||
foutName := flag.Arg(1)
|
|
||||||
ruleName := strings.TrimSuffix(foutName, filepath.Ext(foutName))
|
|
||||||
|
|
||||||
if *manualRuleName != "" {
|
|
||||||
ruleName = *manualRuleName
|
|
||||||
}
|
|
||||||
|
|
||||||
ruleAction := config.Rule(*action)
|
|
||||||
if err := ruleAction.Valid(); err != nil {
|
|
||||||
log.Fatalf("--action=%q is invalid: %v", *action, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result := &Rule{
|
|
||||||
Name: ruleName,
|
|
||||||
Action: ruleAction,
|
|
||||||
}
|
|
||||||
|
|
||||||
if *weight != 0 {
|
|
||||||
if ruleAction != config.RuleWeigh {
|
|
||||||
log.Fatalf("used --weight=%d but --action=%s", *weight, *action)
|
|
||||||
}
|
|
||||||
|
|
||||||
result.Weight = &config.Weight{
|
|
||||||
Adjust: *weight,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ips, err := FetchBlocklist(blocklistURL)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("can't fetch blocklist %s: %v", blocklistURL, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result.RemoteAddr = ips
|
|
||||||
|
|
||||||
fout, err := os.Create(foutName)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("can't create output file %q: %v", foutName, err)
|
|
||||||
}
|
|
||||||
defer fout.Close()
|
|
||||||
|
|
||||||
fmt.Fprintf(fout, "# Generated by %s on %s from %s\n\n", filepath.Base(os.Args[0]), time.Now().Format(time.RFC3339), blocklistURL)
|
|
||||||
|
|
||||||
data, err := yaml.Marshal([]*Rule{result})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("can't marshal yaml")
|
|
||||||
}
|
|
||||||
|
|
||||||
fout.Write(data)
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user