mirror of
https://github.com/TecharoHQ/anubis.git
synced 2026-05-01 20:41:56 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2b44b29013 | |||
| ebad69a4e1 | |||
| 71147b4857 | |||
| cee7871ef8 | |||
| 26d258fb94 | |||
| 80a8e0a8ae | |||
| 359613f35a | |||
| 1d8e98c5ec |
@@ -18,3 +18,8 @@ clampip
|
|||||||
pseudoprofound
|
pseudoprofound
|
||||||
reimagining
|
reimagining
|
||||||
iocaine
|
iocaine
|
||||||
|
admins
|
||||||
|
fout
|
||||||
|
iplist
|
||||||
|
NArg
|
||||||
|
blocklists
|
||||||
|
|||||||
@@ -87,10 +87,14 @@
|
|||||||
^docs/docs/user/known-instances.md$
|
^docs/docs/user/known-instances.md$
|
||||||
^docs/manifest/.*$
|
^docs/manifest/.*$
|
||||||
^docs/static/\.nojekyll$
|
^docs/static/\.nojekyll$
|
||||||
^lib/policy/config/testdata/bad/unparseable\.json$
|
|
||||||
^internal/glob/glob_test.go$
|
^internal/glob/glob_test.go$
|
||||||
|
^internal/honeypot/naive/affirmations\.txt$
|
||||||
|
^internal/honeypot/naive/spintext\.txt$
|
||||||
|
^internal/honeypot/naive/titles\.txt$
|
||||||
|
^lib/config/testdata/bad/unparseable\.json$
|
||||||
|
^lib/localization/.*_test.go$
|
||||||
|
^lib/localization/locales/.*\.json$
|
||||||
|
^lib/policy/config/testdata/bad/unparseable\.json$
|
||||||
|
^test/.*$
|
||||||
ignore$
|
ignore$
|
||||||
robots.txt
|
robots.txt
|
||||||
^lib/localization/locales/.*\.json$
|
|
||||||
^lib/localization/.*_test.go$
|
|
||||||
^test/.*$
|
|
||||||
|
|||||||
+406
-409
@@ -1,409 +1,406 @@
|
|||||||
acs
|
acs
|
||||||
Actorified
|
Actorified
|
||||||
actorifiedstore
|
actorifiedstore
|
||||||
actorify
|
actorify
|
||||||
Aibrew
|
Aibrew
|
||||||
alibaba
|
alibaba
|
||||||
alrest
|
alrest
|
||||||
amazonbot
|
amazonbot
|
||||||
anthro
|
anthro
|
||||||
anubis
|
anubis
|
||||||
anubistest
|
anubistest
|
||||||
apnic
|
apnic
|
||||||
APNICRANDNETAU
|
APNICRANDNETAU
|
||||||
Applebot
|
Applebot
|
||||||
archlinux
|
archlinux
|
||||||
arpa
|
arpa
|
||||||
asnc
|
asnc
|
||||||
asnchecker
|
asnchecker
|
||||||
asns
|
asns
|
||||||
aspirational
|
aspirational
|
||||||
atuin
|
atuin
|
||||||
azuretools
|
azuretools
|
||||||
badregexes
|
badregexes
|
||||||
bbolt
|
bbolt
|
||||||
bdba
|
bdba
|
||||||
berr
|
berr
|
||||||
bezier
|
bezier
|
||||||
bingbot
|
bingbot
|
||||||
Bitcoin
|
Bitcoin
|
||||||
bitrate
|
bitrate
|
||||||
Bluesky
|
Bluesky
|
||||||
blueskybot
|
blueskybot
|
||||||
boi
|
boi
|
||||||
Bokm
|
Bokm
|
||||||
botnet
|
botnet
|
||||||
botstopper
|
botstopper
|
||||||
BPort
|
BPort
|
||||||
Brightbot
|
Brightbot
|
||||||
broked
|
broked
|
||||||
buildah
|
buildah
|
||||||
byteslice
|
byteslice
|
||||||
Bytespider
|
Bytespider
|
||||||
cachebuster
|
cachebuster
|
||||||
cachediptoasn
|
cachediptoasn
|
||||||
Caddyfile
|
Caddyfile
|
||||||
caninetools
|
caninetools
|
||||||
Cardyb
|
Cardyb
|
||||||
celchecker
|
celchecker
|
||||||
celphase
|
celphase
|
||||||
cerr
|
cerr
|
||||||
certresolver
|
certresolver
|
||||||
cespare
|
cespare
|
||||||
CGNAT
|
CGNAT
|
||||||
cgr
|
cgr
|
||||||
chainguard
|
chainguard
|
||||||
chall
|
chall
|
||||||
challengemozilla
|
challengemozilla
|
||||||
challengetest
|
challengetest
|
||||||
checkpath
|
checkpath
|
||||||
checkresult
|
checkresult
|
||||||
chibi
|
chibi
|
||||||
cidranger
|
cidranger
|
||||||
ckie
|
ckie
|
||||||
cloudflare
|
cloudflare
|
||||||
Codespaces
|
Codespaces
|
||||||
confd
|
confd
|
||||||
connnection
|
connnection
|
||||||
containerbuild
|
containerbuild
|
||||||
containerregistry
|
containerregistry
|
||||||
coreutils
|
coreutils
|
||||||
Cotoyogi
|
Cotoyogi
|
||||||
Cromite
|
Cromite
|
||||||
crt
|
crt
|
||||||
Cscript
|
Cscript
|
||||||
daemonizing
|
daemonizing
|
||||||
dayjob
|
databento
|
||||||
DDOS
|
dayjob
|
||||||
Debian
|
DDOS
|
||||||
debrpm
|
Debian
|
||||||
decaymap
|
debrpm
|
||||||
devcontainers
|
decaymap
|
||||||
Diffbot
|
devcontainers
|
||||||
discordapp
|
Diffbot
|
||||||
discordbot
|
discordapp
|
||||||
distros
|
discordbot
|
||||||
dnf
|
distros
|
||||||
dnsbl
|
dnf
|
||||||
dnserr
|
dnsbl
|
||||||
DNSTTL
|
dnserr
|
||||||
domainhere
|
DNSTTL
|
||||||
dracula
|
domainhere
|
||||||
dronebl
|
dracula
|
||||||
droneblresponse
|
dronebl
|
||||||
dropin
|
droneblresponse
|
||||||
dsilence
|
dropin
|
||||||
duckduckbot
|
dsilence
|
||||||
eerror
|
duckduckbot
|
||||||
ellenjoe
|
eerror
|
||||||
emacs
|
ellenjoe
|
||||||
enbyware
|
emacs
|
||||||
etld
|
enbyware
|
||||||
everyones
|
etld
|
||||||
evilbot
|
everyones
|
||||||
evilsite
|
evilbot
|
||||||
expressionorlist
|
evilsite
|
||||||
externalagent
|
expressionorlist
|
||||||
externalfetcher
|
externalagent
|
||||||
extldflags
|
externalfetcher
|
||||||
facebookgo
|
extldflags
|
||||||
Factset
|
facebookgo
|
||||||
fahedouch
|
Factset
|
||||||
fastcgi
|
fahedouch
|
||||||
FCr
|
fastcgi
|
||||||
fcrdns
|
FCr
|
||||||
fediverse
|
fcrdns
|
||||||
ffprobe
|
fediverse
|
||||||
financials
|
ffprobe
|
||||||
finfos
|
financials
|
||||||
Firecrawl
|
finfos
|
||||||
flagenv
|
Firecrawl
|
||||||
Fordola
|
flagenv
|
||||||
forgejo
|
Fordola
|
||||||
forwardauth
|
forgejo
|
||||||
fsys
|
forwardauth
|
||||||
fullchain
|
fsys
|
||||||
gaissmai
|
fullchain
|
||||||
Galvus
|
gaissmai
|
||||||
geoip
|
Galvus
|
||||||
geoipchecker
|
geoip
|
||||||
gha
|
geoipchecker
|
||||||
GHSA
|
gha
|
||||||
Ghz
|
GHSA
|
||||||
gipc
|
Ghz
|
||||||
gitea
|
gipc
|
||||||
godotenv
|
gitea
|
||||||
goland
|
GLM
|
||||||
gomod
|
godotenv
|
||||||
goodbot
|
goland
|
||||||
googlebot
|
gomod
|
||||||
gopsutil
|
goodbot
|
||||||
govulncheck
|
googlebot
|
||||||
goyaml
|
gopsutil
|
||||||
GPG
|
govulncheck
|
||||||
GPT
|
goyaml
|
||||||
gptbot
|
GPG
|
||||||
Graphene
|
GPT
|
||||||
grpcprom
|
gptbot
|
||||||
grw
|
Graphene
|
||||||
gzw
|
grpcprom
|
||||||
Hashcash
|
grw
|
||||||
hashrate
|
gzw
|
||||||
headermap
|
Hashcash
|
||||||
healthcheck
|
hashrate
|
||||||
healthz
|
headermap
|
||||||
hec
|
healthcheck
|
||||||
helpdesk
|
healthz
|
||||||
Hetzner
|
hec
|
||||||
hmc
|
helpdesk
|
||||||
homelab
|
Hetzner
|
||||||
hostable
|
hmc
|
||||||
htmlc
|
homelab
|
||||||
htmx
|
hostable
|
||||||
httpdebug
|
htmlc
|
||||||
huawei
|
htmx
|
||||||
hypertext
|
httpdebug
|
||||||
iaskspider
|
huawei
|
||||||
iaso
|
hypertext
|
||||||
iat
|
iaskspider
|
||||||
ifm
|
iaso
|
||||||
Imagesift
|
iat
|
||||||
imgproxy
|
ifm
|
||||||
impressum
|
Imagesift
|
||||||
inbox
|
imgproxy
|
||||||
ingressed
|
impressum
|
||||||
inp
|
inbox
|
||||||
internets
|
ingressed
|
||||||
IPTo
|
inp
|
||||||
iptoasn
|
internets
|
||||||
isp
|
IPTo
|
||||||
iss
|
iptoasn
|
||||||
isset
|
isp
|
||||||
ivh
|
iss
|
||||||
Jenomis
|
isset
|
||||||
JGit
|
ivh
|
||||||
jhjj
|
Jenomis
|
||||||
joho
|
JGit
|
||||||
journalctl
|
jhjj
|
||||||
jshelter
|
joho
|
||||||
JWTs
|
journalctl
|
||||||
kagi
|
jshelter
|
||||||
kagibot
|
JWTs
|
||||||
Keyfunc
|
kagi
|
||||||
keypair
|
kagibot
|
||||||
KHTML
|
Keyfunc
|
||||||
kinda
|
keypair
|
||||||
KUBECONFIG
|
KHTML
|
||||||
lcj
|
kinda
|
||||||
ldflags
|
KUBECONFIG
|
||||||
letsencrypt
|
lcj
|
||||||
Lexentale
|
ldflags
|
||||||
lfc
|
letsencrypt
|
||||||
lgbt
|
Lexentale
|
||||||
licend
|
lfc
|
||||||
licstart
|
lgbt
|
||||||
lightpanda
|
licend
|
||||||
limsa
|
licstart
|
||||||
Linting
|
lightpanda
|
||||||
listor
|
limsa
|
||||||
LLU
|
Linting
|
||||||
loadbalancer
|
listor
|
||||||
lol
|
LLU
|
||||||
lominsa
|
loadbalancer
|
||||||
maintainership
|
lol
|
||||||
malware
|
lominsa
|
||||||
mcr
|
maintainership
|
||||||
memes
|
malware
|
||||||
metarefresh
|
mcr
|
||||||
metrix
|
memes
|
||||||
mimi
|
metarefresh
|
||||||
Minfilia
|
metrix
|
||||||
mistralai
|
mimi
|
||||||
mnt
|
Minfilia
|
||||||
Mojeek
|
mistralai
|
||||||
mojeekbot
|
mnt
|
||||||
mozilla
|
Mojeek
|
||||||
myclient
|
mojeekbot
|
||||||
mymaster
|
mozilla
|
||||||
mypass
|
myclient
|
||||||
myuser
|
mymaster
|
||||||
nbf
|
mypass
|
||||||
nepeat
|
myuser
|
||||||
netsurf
|
nbf
|
||||||
nginx
|
nepeat
|
||||||
nicksnyder
|
netsurf
|
||||||
nobots
|
nginx
|
||||||
NONINFRINGEMENT
|
nicksnyder
|
||||||
nosleep
|
nikandfor
|
||||||
nullglob
|
nobots
|
||||||
oci
|
NONINFRINGEMENT
|
||||||
OCOB
|
nosleep
|
||||||
ogtag
|
nullglob
|
||||||
oklch
|
oci
|
||||||
omgili
|
OCOB
|
||||||
omgilibot
|
ogtag
|
||||||
openai
|
oklch
|
||||||
opendns
|
omgili
|
||||||
opengraph
|
omgilibot
|
||||||
openrc
|
openai
|
||||||
oswald
|
opendns
|
||||||
pag
|
opengraph
|
||||||
palemoon
|
openrc
|
||||||
Pangu
|
oswald
|
||||||
parseable
|
pag
|
||||||
passthrough
|
pagegen
|
||||||
Patreon
|
palemoon
|
||||||
pgrep
|
Pangu
|
||||||
phrik
|
parseable
|
||||||
pidfile
|
passthrough
|
||||||
pids
|
Patreon
|
||||||
pipefail
|
pgrep
|
||||||
pki
|
phrik
|
||||||
podkova
|
pidfile
|
||||||
podman
|
pids
|
||||||
Postgre
|
pipefail
|
||||||
poststart
|
pki
|
||||||
prebaked
|
podkova
|
||||||
privkey
|
podman
|
||||||
promauto
|
Postgre
|
||||||
promhttp
|
poststart
|
||||||
proofofwork
|
prebaked
|
||||||
publicsuffix
|
privkey
|
||||||
purejs
|
promauto
|
||||||
pwcmd
|
promhttp
|
||||||
pwuser
|
proofofwork
|
||||||
qualys
|
publicsuffix
|
||||||
qwant
|
purejs
|
||||||
qwantbot
|
pwcmd
|
||||||
rac
|
pwuser
|
||||||
rawler
|
qualys
|
||||||
rcvar
|
qwant
|
||||||
redhat
|
qwantbot
|
||||||
redir
|
rac
|
||||||
redirectscheme
|
rawler
|
||||||
refactors
|
rcvar
|
||||||
remoteip
|
redhat
|
||||||
reputational
|
redir
|
||||||
risc
|
redirectscheme
|
||||||
ruleset
|
refactors
|
||||||
runlevels
|
remoteip
|
||||||
RUnlock
|
reputational
|
||||||
runtimedir
|
Rhul
|
||||||
runtimedirectory
|
risc
|
||||||
Ryzen
|
ruleset
|
||||||
sas
|
runlevels
|
||||||
sasl
|
RUnlock
|
||||||
screenshots
|
runtimedir
|
||||||
searchbot
|
runtimedirectory
|
||||||
searx
|
Ryzen
|
||||||
sebest
|
sas
|
||||||
secretplans
|
sasl
|
||||||
Semrush
|
screenshots
|
||||||
Seo
|
searchbot
|
||||||
setsebool
|
searx
|
||||||
shellcheck
|
sebest
|
||||||
shirou
|
secretplans
|
||||||
shopt
|
Semrush
|
||||||
Sidetrade
|
Seo
|
||||||
simprint
|
setsebool
|
||||||
sitemap
|
shellcheck
|
||||||
sls
|
shirou
|
||||||
sni
|
shoneypot
|
||||||
snipster
|
shopt
|
||||||
Spambot
|
Sidetrade
|
||||||
sparkline
|
simprint
|
||||||
spyderbot
|
sitemap
|
||||||
srv
|
sls
|
||||||
stackoverflow
|
sni
|
||||||
startprecmd
|
snipster
|
||||||
stoppostcmd
|
Spambot
|
||||||
storetest
|
spammer
|
||||||
subgrid
|
sparkline
|
||||||
subr
|
spyderbot
|
||||||
subrequest
|
srv
|
||||||
SVCNAME
|
stackoverflow
|
||||||
tagline
|
startprecmd
|
||||||
tarballs
|
stoppostcmd
|
||||||
tarrif
|
storetest
|
||||||
taviso
|
subgrid
|
||||||
tbn
|
subr
|
||||||
tbr
|
subrequest
|
||||||
techaro
|
SVCNAME
|
||||||
techarohq
|
tagline
|
||||||
telegrambot
|
tarballs
|
||||||
templ
|
tarrif
|
||||||
templruntime
|
taviso
|
||||||
testarea
|
tbn
|
||||||
Thancred
|
tbr
|
||||||
thoth
|
techaro
|
||||||
thothmock
|
techarohq
|
||||||
Tik
|
telegrambot
|
||||||
Timpibot
|
templ
|
||||||
TLog
|
templruntime
|
||||||
traefik
|
testarea
|
||||||
trunc
|
Thancred
|
||||||
uberspace
|
thoth
|
||||||
Unbreak
|
thothmock
|
||||||
unbreakdocker
|
Tik
|
||||||
unifiedjs
|
Timpibot
|
||||||
unmarshal
|
TLog
|
||||||
unparseable
|
traefik
|
||||||
uvx
|
trunc
|
||||||
UXP
|
uberspace
|
||||||
valkey
|
Unbreak
|
||||||
Varis
|
unbreakdocker
|
||||||
Velen
|
unifiedjs
|
||||||
vendored
|
unmarshal
|
||||||
verify
|
unparseable
|
||||||
vhosts
|
uvx
|
||||||
vkbot
|
UXP
|
||||||
VKE
|
valkey
|
||||||
vnd
|
Varis
|
||||||
VPS
|
Velen
|
||||||
Vultr
|
vendored
|
||||||
weblate
|
vhosts
|
||||||
webmaster
|
vkbot
|
||||||
webpage
|
VKE
|
||||||
websecure
|
vnd
|
||||||
websites
|
VPS
|
||||||
Webzio
|
Vultr
|
||||||
whois
|
weblate
|
||||||
wildbase
|
webmaster
|
||||||
withthothmock
|
webpage
|
||||||
wolfbeast
|
websecure
|
||||||
wordpress
|
websites
|
||||||
workaround
|
Webzio
|
||||||
workdir
|
whois
|
||||||
wpbot
|
wildbase
|
||||||
XCircle
|
withthothmock
|
||||||
xeiaso
|
wolfbeast
|
||||||
xeserv
|
wordpress
|
||||||
xesite
|
workaround
|
||||||
xess
|
workdir
|
||||||
xff
|
wpbot
|
||||||
XForwarded
|
XCircle
|
||||||
XNG
|
xeiaso
|
||||||
XOB
|
xeserv
|
||||||
XOriginal
|
xesite
|
||||||
XReal
|
xess
|
||||||
yae
|
xff
|
||||||
YAMLTo
|
XForwarded
|
||||||
Yda
|
XNG
|
||||||
yeet
|
XOB
|
||||||
yeetfile
|
XOriginal
|
||||||
yourdomain
|
XReal
|
||||||
yyz
|
Y'shtola
|
||||||
Zenos
|
yae
|
||||||
zizmor
|
YAMLTo
|
||||||
zombocom
|
Yda
|
||||||
zos
|
yeet
|
||||||
GLM
|
yeetfile
|
||||||
iocaine
|
yourdomain
|
||||||
nikandfor
|
yyz
|
||||||
pagegen
|
Zenos
|
||||||
pseudoprofound
|
zizmor
|
||||||
reimagining
|
zombocom
|
||||||
Rhul
|
zos
|
||||||
shoneypot
|
|
||||||
spammer
|
|
||||||
Y'shtola
|
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ Anubis is brought to you by sponsors and donors like:
|
|||||||
<a href="https://www.raptorcs.com/content/base/products.html">
|
<a href="https://www.raptorcs.com/content/base/products.html">
|
||||||
<img src="./docs/static/img/sponsors/raptor-computing-logo.webp" alt="Raptor Computing Systems" height=64 />
|
<img src="./docs/static/img/sponsors/raptor-computing-logo.webp" alt="Raptor Computing Systems" height=64 />
|
||||||
</a>
|
</a>
|
||||||
|
<a href="https://databento.com/?utm_source=anubis&utm_medium=sponsor&utm_campaign=anubis">
|
||||||
|
<img src="./docs/static/img/sponsors/databento-logo.webp" alt="Databento" />
|
||||||
|
</a>
|
||||||
|
|
||||||
### Gold Tier
|
### Gold Tier
|
||||||
|
|
||||||
|
|||||||
@@ -3,5 +3,6 @@
|
|||||||
- name: qualys-ssl-labs
|
- name: qualys-ssl-labs
|
||||||
action: ALLOW
|
action: ALLOW
|
||||||
remote_addresses:
|
remote_addresses:
|
||||||
- 64.41.200.0/24
|
- 69.67.183.0/24
|
||||||
- 2600:C02:1020:4202::/64
|
- 2600:C02:1020:4202::/64
|
||||||
|
- 2602:fdaa:c6:2::/64
|
||||||
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
- Add iplist2rule tool that lets admins turn an IP address blocklist into an Anubis ruleset.
|
||||||
- Add Polish locale ([#1292](https://github.com/TecharoHQ/anubis/pull/1309))
|
- Add Polish locale ([#1292](https://github.com/TecharoHQ/anubis/pull/1309))
|
||||||
|
|
||||||
<!-- This changes the project to: -->
|
<!-- This changes the project to: -->
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
---
|
||||||
|
title: iplist2rule CLI tool
|
||||||
|
---
|
||||||
|
|
||||||
|
The `iplist2rule` tool converts IP blocklists into Anubis challenge policies. It reads common IP block list formats and generates the appropriate Anubis policy file for IP address filtering.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
Install directly with Go
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go install github.com/TecharoHQ/anubis/utils/cmd/iplist2rule@latest
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
Basic conversion from URL:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
iplist2rule https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Explicitly allow every IP address on a list:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
iplist2rule --action ALLOW https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Add weight to requests matching IP addresses on a list:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
iplist2rule --action WEIGH --weight 20 https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Options
|
||||||
|
|
||||||
|
| Flag | Description | Default |
|
||||||
|
| :------------ | :----------------------------------------------------------------------------------------------- | :-------------------------------- |
|
||||||
|
| `--action` | The Anubis action to take for the IP address in question, must be in ALL CAPS. | `DENY` (forbids traffic) |
|
||||||
|
| `--rule-name` | The name for the generated Anubis rule, should be in kebab-case. | (not set, inferred from filename) |
|
||||||
|
| `--weight` | When `--action=WEIGH`, how many weight points should be added or removed from matching requests? | 0 (not set) |
|
||||||
|
|
||||||
|
## Using the Generated Policy
|
||||||
|
|
||||||
|
Save the output and import it in your main policy file:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
bots:
|
||||||
|
- import: "./filter-tor.yaml"
|
||||||
|
```
|
||||||
@@ -12,6 +12,7 @@ Install directly with Go:
|
|||||||
```bash
|
```bash
|
||||||
go install github.com/TecharoHQ/anubis/cmd/robots2policy@latest
|
go install github.com/TecharoHQ/anubis/cmd/robots2policy@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
Basic conversion from URL:
|
Basic conversion from URL:
|
||||||
@@ -35,8 +36,8 @@ robots2policy -input robots.txt -action DENY -format json
|
|||||||
## Options
|
## Options
|
||||||
|
|
||||||
| Flag | Description | Default |
|
| Flag | Description | Default |
|
||||||
|-----------------------|--------------------------------------------------------------------|---------------------|
|
| --------------------- | ------------------------------------------------------------------ | ------------------- |
|
||||||
| `-input` | robots.txt file path or URL (use `-` for stdin) | *required* |
|
| `-input` | robots.txt file path or URL (use `-` for stdin) | _required_ |
|
||||||
| `-output` | Output file (use `-` for stdout) | stdout |
|
| `-output` | Output file (use `-` for stdout) | stdout |
|
||||||
| `-format` | Output format: `yaml` or `json` | `yaml` |
|
| `-format` | Output format: `yaml` or `json` | `yaml` |
|
||||||
| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` |
|
| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` |
|
||||||
@@ -47,6 +48,7 @@ robots2policy -input robots.txt -action DENY -format json
|
|||||||
## Example
|
## Example
|
||||||
|
|
||||||
Input robots.txt:
|
Input robots.txt:
|
||||||
|
|
||||||
```txt
|
```txt
|
||||||
User-agent: *
|
User-agent: *
|
||||||
Disallow: /admin/
|
Disallow: /admin/
|
||||||
@@ -57,6 +59,7 @@ Disallow: /
|
|||||||
```
|
```
|
||||||
|
|
||||||
Generated policy:
|
Generated policy:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
- name: robots-txt-policy-disallow-1
|
- name: robots-txt-policy-disallow-1
|
||||||
action: CHALLENGE
|
action: CHALLENGE
|
||||||
@@ -77,8 +80,8 @@ Generated policy:
|
|||||||
Save the output and import it in your main policy file:
|
Save the output and import it in your main policy file:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
import:
|
bots:
|
||||||
- path: "./robots-policy.yaml"
|
- import: "./robots-policy.yaml"
|
||||||
```
|
```
|
||||||
|
|
||||||
The tool handles wildcard patterns, user-agent specific rules, and blacklisted bots automatically.
|
The tool handles wildcard patterns, user-agent specific rules, and blacklisted bots automatically.
|
||||||
|
|||||||
@@ -29,6 +29,9 @@ Anubis is brought to you by sponsors and donors like:
|
|||||||
height="64"
|
height="64"
|
||||||
/>
|
/>
|
||||||
</a>
|
</a>
|
||||||
|
<a href="https://databento.com/?utm_source=anubis&utm_medium=sponsor&utm_campaign=anubis">
|
||||||
|
<img src="/img/sponsors/databento-logo.webp" alt="Databento" />
|
||||||
|
</a>
|
||||||
|
|
||||||
### Gold Tier
|
### Gold Tier
|
||||||
|
|
||||||
|
|||||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
@@ -81,7 +81,28 @@ func (ls *LocalizationService) GetLocalizerFromRequest(r *http.Request) *i18n.Lo
|
|||||||
return i18n.NewLocalizer(bundle, "en")
|
return i18n.NewLocalizer(bundle, "en")
|
||||||
}
|
}
|
||||||
acceptLanguage := r.Header.Get("Accept-Language")
|
acceptLanguage := r.Header.Get("Accept-Language")
|
||||||
return i18n.NewLocalizer(ls.bundle, acceptLanguage, "en")
|
|
||||||
|
// Parse Accept-Language header to properly handle quality factors
|
||||||
|
// The language.ParseAcceptLanguage function returns tags sorted by quality
|
||||||
|
tags, _, err := language.ParseAcceptLanguage(acceptLanguage)
|
||||||
|
if err != nil || len(tags) == 0 {
|
||||||
|
return i18n.NewLocalizer(ls.bundle, "en")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert parsed tags to strings for the localizer
|
||||||
|
// We include both the full tag and base language to ensure proper matching
|
||||||
|
langs := make([]string, 0, len(tags)*2+1)
|
||||||
|
for _, tag := range tags {
|
||||||
|
langs = append(langs, tag.String())
|
||||||
|
// Also add base language (e.g., "en" for "en-GB") to help matching
|
||||||
|
base, _ := tag.Base()
|
||||||
|
if base.String() != tag.String() {
|
||||||
|
langs = append(langs, base.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
langs = append(langs, "en") // Always include English as fallback
|
||||||
|
|
||||||
|
return i18n.NewLocalizer(ls.bundle, langs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SimpleLocalizer wraps i18n.Localizer with a more convenient API
|
// SimpleLocalizer wraps i18n.Localizer with a more convenient API
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package localization
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http/httptest"
|
||||||
"sort"
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -138,3 +139,40 @@ func TestComprehensiveTranslations(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAcceptLanguageQualityFactors(t *testing.T) {
|
||||||
|
service := NewLocalizationService()
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
acceptLanguage string
|
||||||
|
expectedLang string
|
||||||
|
}{
|
||||||
|
{"simple_en", "en", "en"},
|
||||||
|
{"simple_de", "de", "de"},
|
||||||
|
{"en_GB_with_lower_priority_de", "en-GB,de-DE;q=0.5", "en"},
|
||||||
|
{"en_GB_only", "en-GB", "en"},
|
||||||
|
{"de_with_lower_priority_en", "de,en;q=0.5", "de"},
|
||||||
|
{"de_DE_with_lower_priority_en", "de-DE,en;q=0.5", "de"},
|
||||||
|
{"fr_with_lower_priority_de", "fr,de;q=0.5", "fr"},
|
||||||
|
{"zh_CN_regional", "zh-CN", "zh-CN"},
|
||||||
|
{"zh_TW_regional", "zh-TW", "zh-TW"},
|
||||||
|
{"pt_BR_regional", "pt-BR", "pt-BR"},
|
||||||
|
{"complex_header", "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7,de;q=0.5", "fr"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
req := httptest.NewRequest("GET", "/", nil)
|
||||||
|
req.Header.Set("Accept-Language", tc.acceptLanguage)
|
||||||
|
|
||||||
|
localizer := service.GetLocalizerFromRequest(req)
|
||||||
|
sl := &SimpleLocalizer{Localizer: localizer}
|
||||||
|
|
||||||
|
gotLang := sl.GetLang()
|
||||||
|
if gotLang != tc.expectedLang {
|
||||||
|
t.Errorf("Accept-Language %q: expected %s, got %s", tc.acceptLanguage, tc.expectedLang, gotLang)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- 3004:3004
|
- 3004:3004
|
||||||
volumes:
|
volumes:
|
||||||
- ../pki/registry.local.cetacean.club:/etc/techaro/pki/registry.local.cetacean.club
|
- ./pki/registry.local.cetacean.club:/etc/techaro/pki/registry.local.cetacean.club
|
||||||
|
|
||||||
anubis:
|
anubis:
|
||||||
image: ko.local/anubis
|
image: ko.local/anubis
|
||||||
|
|||||||
+39
-36
@@ -1,53 +1,56 @@
|
|||||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||||
(cd $REPO_ROOT && go install ./utils/cmd/...)
|
(cd $REPO_ROOT && go install ./utils/cmd/...)
|
||||||
|
|
||||||
|
mkdir -p pki
|
||||||
|
echo '*' >>./pki/.gitignore
|
||||||
|
|
||||||
function cleanup() {
|
function cleanup() {
|
||||||
set +e
|
set +e
|
||||||
|
|
||||||
pkill -P $$
|
pkill -P $$
|
||||||
|
|
||||||
if [ -f "docker-compose.yaml" ]; then
|
if [ -f "docker-compose.yaml" ]; then
|
||||||
docker compose down -t 1 || :
|
docker compose down -t 1 || :
|
||||||
docker compose rm -f || :
|
docker compose rm -f || :
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
trap cleanup EXIT SIGINT
|
trap cleanup EXIT SIGINT
|
||||||
|
|
||||||
function build_anubis_ko() {
|
function build_anubis_ko() {
|
||||||
(
|
(
|
||||||
cd $REPO_ROOT && npm ci && npm run assets
|
cd $REPO_ROOT && npm ci && npm run assets
|
||||||
)
|
)
|
||||||
(
|
(
|
||||||
cd $REPO_ROOT &&
|
cd $REPO_ROOT &&
|
||||||
VERSION=devel ko build \
|
VERSION=devel ko build \
|
||||||
--platform=all \
|
--platform=all \
|
||||||
--base-import-paths \
|
--base-import-paths \
|
||||||
--tags="latest" \
|
--tags="latest" \
|
||||||
--image-user=1000 \
|
--image-user=1000 \
|
||||||
--image-annotation="" \
|
--image-annotation="" \
|
||||||
--image-label="" \
|
--image-label="" \
|
||||||
./cmd/anubis \
|
./cmd/anubis \
|
||||||
--local
|
--local
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
function mint_cert() {
|
function mint_cert() {
|
||||||
if [ "$#" -ne 1 ]; then
|
if [ "$#" -ne 1 ]; then
|
||||||
echo "Usage: mint_cert <domain.name>"
|
echo "Usage: mint_cert <domain.name>"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
domainName="$1"
|
domainName="$1"
|
||||||
|
|
||||||
# If the transient local TLS certificate doesn't exist, mint a new one
|
# If the transient local TLS certificate doesn't exist, mint a new one
|
||||||
if [ ! -f "${REPO_ROOT}/test/pki/${domainName}/cert.pem" ]; then
|
if [ ! -f "./pki/${domainName}/cert.pem" ]; then
|
||||||
# Subshell to contain the directory change
|
# Subshell to contain the directory change
|
||||||
(
|
(
|
||||||
cd ${REPO_ROOT}/test/pki &&
|
cd ./pki &&
|
||||||
mkdir -p "${domainName}" &&
|
mkdir -p "${domainName}" &&
|
||||||
go tool minica -domains "${domainName}" &&
|
go tool minica -domains "${domainName}" &&
|
||||||
cd "${domainName}" &&
|
cd "${domainName}" &&
|
||||||
chmod 666 *
|
chmod 666 *
|
||||||
)
|
)
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-11
@@ -1,24 +1,17 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
export VERSION=$GITHUB_COMMIT-test
|
|
||||||
export KO_DOCKER_REPO=ko.local
|
|
||||||
|
|
||||||
source ../lib/lib.sh
|
source ../lib/lib.sh
|
||||||
|
|
||||||
|
export KO_DOCKER_REPO=ko.local
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
build_anubis_ko
|
|
||||||
mint_cert mimi.techaro.lol
|
mint_cert mimi.techaro.lol
|
||||||
|
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
-v ./conf/nginx:/etc/nginx:ro \
|
-v $PWD/conf/nginx:/etc/nginx:ro \
|
||||||
-v ../pki:/techaro/pki:ro \
|
-v $PWD/pki:/techaro/pki:ro \
|
||||||
nginx \
|
nginx \
|
||||||
nginx -t
|
nginx -t
|
||||||
|
|
||||||
docker compose up -d
|
|
||||||
|
|
||||||
docker compose down -t 1 || :
|
|
||||||
docker compose rm -f || :
|
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ services:
|
|||||||
KEY_FNAME: key.pem
|
KEY_FNAME: key.pem
|
||||||
PROXY_TO: http://anubis:3000
|
PROXY_TO: http://anubis:3000
|
||||||
volumes:
|
volumes:
|
||||||
- ../../pki/relayd:/techaro/pki:ro
|
- ./pki/relayd:/techaro/pki:ro
|
||||||
|
|
||||||
# novnc:
|
# novnc:
|
||||||
# image: geek1011/easy-novnc
|
# image: geek1011/easy-novnc
|
||||||
@@ -42,7 +42,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
DISPLAY: display:0
|
DISPLAY: display:0
|
||||||
volumes:
|
volumes:
|
||||||
- ../../pki:/usr/local/share/ca-certificates/minica:ro
|
- ./pki:/usr/local/share/ca-certificates/minica:ro
|
||||||
- ../scripts:/hack/scripts:ro
|
- ../scripts:/hack/scripts:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
- anubis
|
- anubis
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ services:
|
|||||||
KEY_FNAME: key.pem
|
KEY_FNAME: key.pem
|
||||||
PROXY_TO: http://anubis:3000
|
PROXY_TO: http://anubis:3000
|
||||||
volumes:
|
volumes:
|
||||||
- ../../pki/relayd:/techaro/pki:ro
|
- ./pki/relayd:/techaro/pki:ro
|
||||||
|
|
||||||
# novnc:
|
# novnc:
|
||||||
# image: geek1011/easy-novnc
|
# image: geek1011/easy-novnc
|
||||||
@@ -40,5 +40,5 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
DISPLAY: display:0
|
DISPLAY: display:0
|
||||||
volumes:
|
volumes:
|
||||||
- ../../pki:/usr/local/share/ca-certificates/minica:ro
|
- ./pki:/usr/local/share/ca-certificates/minica:ro
|
||||||
- ../scripts:/hack/scripts:ro
|
- ../scripts:/hack/scripts:ro
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FetchBlocklist reads the blocklist over HTTP and returns every non-commented
|
||||||
|
// line parsed as an IP address in CIDR notation. IPv4 addresses are returned as
|
||||||
|
// /32, IPv6 addresses as /128.
|
||||||
|
//
|
||||||
|
// This function was generated with GLM 4.7.
|
||||||
|
func FetchBlocklist(url string) ([]string, error) {
|
||||||
|
resp, err := http.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lines []string
|
||||||
|
scanner := bufio.NewScanner(resp.Body)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
// Skip empty lines and comments (lines starting with #)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(line)
|
||||||
|
if err != nil {
|
||||||
|
// Skip lines that aren't valid IP addresses
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var cidr string
|
||||||
|
if addr.Is4() {
|
||||||
|
cidr = fmt.Sprintf("%s/32", addr.String())
|
||||||
|
} else {
|
||||||
|
cidr = fmt.Sprintf("%s/128", addr.String())
|
||||||
|
}
|
||||||
|
lines = append(lines, cidr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil && err != io.EOF {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/TecharoHQ/anubis/lib/config"
|
||||||
|
"github.com/facebookgo/flagenv"
|
||||||
|
"sigs.k8s.io/yaml"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Rule struct {
|
||||||
|
Name string `yaml:"name" json:"name"`
|
||||||
|
Action config.Rule `yaml:"action" json:"action"`
|
||||||
|
RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
|
||||||
|
Weight *config.Weight `json:"weight,omitempty" yaml:"weight,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
flag.Usage = func() {
|
||||||
|
fmt.Printf(`Usage of %[1]s:
|
||||||
|
|
||||||
|
%[1]s [flags] <blocklist-url> <filename>
|
||||||
|
|
||||||
|
Grabs the contents of the blocklist, converts it to an Anubis ruleset, and writes it to filename.
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
`, filepath.Base(os.Args[0]))
|
||||||
|
|
||||||
|
flag.PrintDefaults()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
action = flag.String("action", "DENY", "Anubis action to take (ALLOW / DENY / WEIGH)")
|
||||||
|
manualRuleName = flag.String("rule-name", "", "If set, prefer this name over inferring from filename")
|
||||||
|
weight = flag.Int("weight", 0, "If set to any number, add/subtract this many weight points when --action=WEIGH")
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
flagenv.Parse()
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if flag.NArg() != 2 {
|
||||||
|
flag.Usage()
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
blocklistURL := flag.Arg(0)
|
||||||
|
foutName := flag.Arg(1)
|
||||||
|
ruleName := strings.TrimSuffix(foutName, filepath.Ext(foutName))
|
||||||
|
|
||||||
|
if *manualRuleName != "" {
|
||||||
|
ruleName = *manualRuleName
|
||||||
|
}
|
||||||
|
|
||||||
|
ruleAction := config.Rule(*action)
|
||||||
|
if err := ruleAction.Valid(); err != nil {
|
||||||
|
log.Fatalf("--action=%q is invalid: %v", *action, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := &Rule{
|
||||||
|
Name: ruleName,
|
||||||
|
Action: ruleAction,
|
||||||
|
}
|
||||||
|
|
||||||
|
if *weight != 0 {
|
||||||
|
if ruleAction != config.RuleWeigh {
|
||||||
|
log.Fatalf("used --weight=%d but --action=%s", *weight, *action)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.Weight = &config.Weight{
|
||||||
|
Adjust: *weight,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ips, err := FetchBlocklist(blocklistURL)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("can't fetch blocklist %s: %v", blocklistURL, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.RemoteAddr = ips
|
||||||
|
|
||||||
|
fout, err := os.Create(foutName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("can't create output file %q: %v", foutName, err)
|
||||||
|
}
|
||||||
|
defer fout.Close()
|
||||||
|
|
||||||
|
fmt.Fprintf(fout, "# Generated by %s on %s from %s\n\n", filepath.Base(os.Args[0]), time.Now().Format(time.RFC3339), blocklistURL)
|
||||||
|
|
||||||
|
data, err := yaml.Marshal([]*Rule{result})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("can't marshal yaml")
|
||||||
|
}
|
||||||
|
|
||||||
|
fout.Write(data)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user