Compare commits

..

2 Commits

Author SHA1 Message Date
Xe Iaso
208ceca723 chore: move checker package to top level
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-01 23:58:42 +00:00
Xe Iaso
dc0dde3053 chore: start refactor of checkers into separate packages
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-01 22:27:01 +00:00
34 changed files with 428 additions and 373 deletions

View File

@@ -20,13 +20,11 @@ bdba
berr
bingbot
bitcoin
bitrate
blogging
Bluesky
blueskybot
boi
botnet
botstopper
BPort
Brightbot
broked
@@ -54,7 +52,6 @@ ckie
cloudflare
Codespaces
confd
connnection
containerbuild
coreutils
Cotoyogi
@@ -78,7 +75,6 @@ domainhere
dracula
dronebl
droneblresponse
dropin
duckduckbot
eerror
ellenjoe
@@ -96,7 +92,6 @@ facebookgo
Factset
fastcgi
fediverse
ffprobe
finfos
Firecrawl
flagenv
@@ -203,7 +198,6 @@ omgilibot
openai
opengraph
openrc
oswald
pag
palemoon
Pangu
@@ -243,7 +237,6 @@ risc
ruleset
runlevels
RUnlock
runtimedir
sas
sasl
Scumm
@@ -277,8 +270,6 @@ SVCNAME
tagline
tarballs
tarrif
tbn
tbr
techaro
techarohq
templ
@@ -337,4 +328,5 @@ yoursite
Zenos
zizmor
zombocom
Zonbocom
zos

View File

@@ -32,7 +32,3 @@ const APIPrefix = "/.within.website/x/cmd/anubis/api/"
// DefaultDifficulty is the default "difficulty" (number of leading zeroes)
// that must be met by the client in order to pass the challenge.
const DefaultDifficulty = 4
// ForcedLanguage is the language being used instead of the one of the request's Accept-Language header
// if being set.
var ForcedLanguage = ""

View File

@@ -50,7 +50,6 @@ var (
cookieExpiration = flag.Duration("cookie-expiration-time", anubis.CookieDefaultExpirationTime, "The amount of time the authorization cookie is valid for")
cookiePrefix = flag.String("cookie-prefix", "techaro.lol-anubis", "prefix for browser cookies created by Anubis")
cookiePartitioned = flag.Bool("cookie-partitioned", false, "if true, sets the partitioned flag on Anubis cookies, enabling CHIPS support")
forcedLanguage = flag.String("forced-language", "", "if set, this language is being used instead of the one from the request's Accept-Language header")
hs512Secret = flag.String("hs512-secret", "", "secret used to sign JWTs, uses ed25519 if not set")
cookieSecure = flag.Bool("cookie-secure", true, "if true, sets the secure flag on Anubis cookies")
ed25519PrivateKeyHex = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned")
@@ -379,7 +378,6 @@ func main() {
anubis.CookieName = *cookiePrefix + "-auth"
anubis.TestCookieName = *cookiePrefix + "-cookie-verification"
anubis.ForcedLanguage = *forcedLanguage
// If OpenGraph configuration values are not set in the config file, use the
// values from flags / envvars.

View File

@@ -7,5 +7,5 @@
# Warning: May contain user agents that _must_ be blocked in robots.txt, or the opt-out will have no effect.
- name: "ai-catchall"
user_agent_regex: >-
AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Brightbot 1.0|Bytespider|Claude-Web|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|GoogleOther|GoogleOther-Image|GoogleOther-Video|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|NovaAct|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YouBot
AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|anthropic-ai|Brightbot 1.0|Bytespider|CCBot|Claude-Web|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|GoogleOther|GoogleOther-Image|GoogleOther-Video|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|NovaAct|omgili|omgilibot|Operator|PanguBot|Perplexity-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YouBot
action: DENY

View File

@@ -1,8 +1,6 @@
# Warning: Contains user agents that _must_ be blocked in robots.txt, or the opt-out will have no effect.
# Note: Blocks human-directed/non-training user agents
#
# CCBot is allowed because if Common Crawl is allowed, then scrapers don't need to scrape to get the data.
- name: "ai-robots-txt"
user_agent_regex: >-
AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|Andibot|anthropic-ai|Applebot|Applebot-Extended|bedrockbot|Brightbot 1.0|Bytespider|ChatGPT-User|Claude-SearchBot|Claude-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|MistralAI-User/1.0|MyCentralAIScraperBot|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient.com|Perplexity-User|PerplexityBot|PetalBot|PhindBot|Poseidon Research Crawler|QualifiedBot|QuillBot|quillbot.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot-BA|SemrushBot-CT|SemrushBot-OCOB|SemrushBot-SI|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot
AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|Andibot|anthropic-ai|Applebot|Applebot-Extended|bedrockbot|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-SearchBot|Claude-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|MistralAI-User/1.0|MyCentralAIScraperBot|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient.com|Perplexity-User|PerplexityBot|PetalBot|PhindBot|Poseidon Research Crawler|QualifiedBot|QuillBot|quillbot.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot-BA|SemrushBot-CT|SemrushBot-OCOB|SemrushBot-SI|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot
action: DENY

View File

@@ -6,5 +6,4 @@
- import: (data)/crawlers/internet-archive.yaml
- import: (data)/crawlers/kagibot.yaml
- import: (data)/crawlers/marginalia.yaml
- import: (data)/crawlers/mojeekbot.yaml
- import: (data)/crawlers/commoncrawl.yaml
- import: (data)/crawlers/mojeekbot.yaml

View File

@@ -1,12 +0,0 @@
- name: common-crawl
user_agent_regex: CCBot
action: ALLOW
# https://index.commoncrawl.org/ccbot.json
remote_addresses:
[
"2600:1f28:365:80b0::/60",
"18.97.9.168/29",
"18.97.14.80/29",
"18.97.14.88/30",
"98.85.178.216/32",
]

View File

@@ -10,36 +10,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
<!-- This changes the project to: -->
- Add `COOKIE_SECURE` option to set the cookie [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies)
- Sets cookie defaults to use [SameSite: None](https://web.dev/articles/samesite-cookies-explained)
- Determine the `BIND_NETWORK`/`--bind-network` value from the bind address ([#677](https://github.com/TecharoHQ/anubis/issues/677)).
- Implement localization system. Find locale files in lib/localization/locales/.
- Implement a [development container](https://containers.dev/) manifest to make contributions easier.
- Fix dynamic cookie domains functionality ([#731](https://github.com/TecharoHQ/anubis/pull/731))
- Add option for custom cookie prefix ([#732](https://github.com/TecharoHQ/anubis/pull/732))
- Add translation for German language ([#741](https://github.com/TecharoHQ/anubis/pull/741))
- Remove the "Success" interstitial after a proof of work challenge is concluded.
- Add option for forcing a specific language ([#742](https://github.com/TecharoHQ/anubis/pull/742))
- Allow [Common Crawl](https://commoncrawl.org/) by default so scrapers have less incentive to scrape
### Potentially breaking changes
The following potentially breaking change applies to native installs with systemd only:
Each instance of systemd service template now has a unique `RuntimeDirectory`, as opposed to each instance of the service sharing a `RuntimeDirectory`. This change was made to avoid [the `RuntimeDirectory` getting nuked any time one of the Anubis instances restarts](https://github.com/TecharoHQ/anubis/issues/748).
If you configured Anubis' unix sockets to listen on `/run/anubis/foo.sock` for instance `anubis@foo`, you will need to configure Anubis to listen on `/run/anubis/foo/sock` and additionally configure your HTTP load balancer as appropriate.
If you need the legacy behaviour, install this [systemd unit dropin](https://www.flatcar.org/docs/latest/setup/systemd/drop-in-units/):
```systemd
# /etc/systemd/system/anubis@.service.d/50-runtimedir.conf
[Service]
RuntimeDirectory=anubis
```
## v1.20.0: Thancred Waters

View File

@@ -1,215 +0,0 @@
---
title: "Commercial support and an unbranded version"
---
If you want to use Anubis but organizational policies prevent you from using the branding that the open source project ships, we offer a commercial version of Anubis named BotStopper. BotStopper builds off of the open source core of Anubis and offers organizations more control over the branding, including but not limited to:
- Custom images for different states of the challenge process (in process, success, failure)
- Custom CSS and fonts
- Custom titles for the challenge and error pages
- "Anubis" replaced with "BotStopper" across the UI
- A private bug tracker for issues
In the near future this will expand to:
- A private challenge implementation that does advanced fingerprinting to check if the client is a genuine browser or not
- Advanced fingerprinting via [Thoth-based advanced checks](./thoth.mdx)
In order to sign up for BotStopper, please do one of the following:
- Sign up [on GitHub Sponsors](https://github.com/sponsors/Xe) at the $50 per month tier or higher
- Email [sales@techaro.lol](mailto:sales@techaro.lol) with your requirements for invoicing, please note that custom invoicing will cost more than using GitHub Sponsors for understandable overhead reasons
## Installation
Install BotStopper like you would Anubis, but replace the image reference. EG:
```diff
-ghcr.io/techarohq/anubis:latest
+ghcr.io/techarohq/botstopper/anubis:latest
```
### Binary packages
Binary packages are available [in the GitHub Releases page](https://github.com/TecharoHQ/botstopper/releases), the main difference is that the package name is `techaro-botstopper`, the systemd service is `techaro-botstopper@your-instance.service`, the binary is `/usr/bin/botstopper`, and the configuration is in `/etc/techaro-botstopper`. All other instructions in the [native package install guide](./native-install.mdx) apply.
### Docker / Podman
In order to pull the BotStopper image, you need to [authenticate with GitHub's Container Registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry).
```text
docker login ghcr.io -u your-username --password-stdin
```
Then you can use the image as normal.
### Kubernetes
If you are using Kubernetes, you will need to create an image pull secret:
```text
kubectl create secret docker-registry \
techarohq-botstopper \
--docker-server ghcr.io \
--docker-username your-username \
--docker-password your-access-token \
--docker-email your@email.address
```
Then attach it to your Deployment:
```diff
spec:
securityContext:
fsGroup: 1000
+ imagePullSecrets:
+ - name: techarohq-botstopper
```
## Configuration
### Docker compose
Follow [the upstream Docker compose directions](https://anubis.techaro.lol/docs/admin/environments/docker-compose) with the following additional options:
```diff
anubis:
image: ghcr.io/techarohq/botstopper/anubis:latest
environment:
BIND: ":8080"
DIFFICULTY: "4"
METRICS_BIND: ":9090"
SERVE_ROBOTS_TXT: "true"
TARGET: "http://nginx"
OG_PASSTHROUGH: "true"
OG_EXPIRY_TIME: "24h"
+ # botstopper config here
+ CHALLENGE_TITLE: "Doing math for your connnection!"
+ ERROR_TITLE: "Something went wrong!"
+ OVERLAY_FOLDER: /assets
+ volumes:
+ - "./your_folder:/assets"
```
#### Example
There is an example in [docker-compose.yaml](https://github.com/TecharoHQ/botstopper/blob/main/docker-compose.yaml). Start the example with `docker compose up`:
```text
docker compose up -d
```
And then open [https://botstopper.local.cetacean.club:8443](https://botstopper.local.cetacean.club:8443) in your browser.
> [!NOTE]
> This uses locally signed sacrificial TLS certificates stored in `./demo/pki`. Your browser will rightly reject these. Here is what the example looks like:
>
> ![](/img/botstopper/example-screenshot.webp)
## Custom images and CSS
Anubis uses an internal filesystem that contains CSS, JavaScript, and images. The BotStopper variant of Anubis lets you specify an overlay folder with the environment variable `OVERLAY_FOLDER`. The contents of this folder will be overlaid on top of Anubis' internal filesystem, allowing you to easily customize the images and CSS.
Your directory tree should look like this, assuming your data is in `./your_folder`:
```text
./your_folder
└── static
├── css
│ └── custom.css
└── img
├── happy.webp
├── pensive.webp
└── reject.webp
```
For an example directory tree using some off-the-shelf images the Tango icon set, see the [testdata](https://github.com/TecharoHQ/botstopper/tree/main/testdata/static/img) folder.
### Custom CSS
CSS customization is done mainly with CSS variables. View [the example custom CSS file](https://github.com/TecharoHQ/botstopper/blob/main/testdata/static/css/custom.css) for more information about what can be customized.
### Custom fonts
If you want to add custom fonts, copy the `woff2` files alongside your `custom.css` file and then include them with the [`@font-face` CSS at-rule](https://developer.mozilla.org/en-US/docs/Web/CSS/@font-face):
```css
@font-face {
font-family: "Oswald";
font-style: normal;
font-weight: 200 900;
font-display: swap;
src: url("./fonts/oswald.woff2") format("woff2");
}
```
Then adjust your CSS variables accordingly:
```css
:root {
--body-sans-font: Oswald, sans-serif;
--body-preformatted-font: monospace;
--body-title-font: serif;
}
```
To convert `.ttf` fonts to [Web-optimized woff2 fonts](https://www.w3.org/TR/WOFF2/), use the `woff2_compress` command from the `woff2` or `woff2-tools` package:
```console
$ woff2_compress oswald.ttf
Processing oswald.ttf => oswald.woff2
Compressed 159517 to 70469.
```
Then you can import and use it as normal.
### Customizing images
Anubis uses three images to visually communicate the state of the program. These are:
| Image name | Intended message | Example |
| :------------- | :----------------------------------------------- | :-------------------------------- |
| `happy.webp` | You have passed validation, all is good | ![](/img/botstopper/happy.webp) |
| `pensive.webp` | Checking is running, hold steady until it's done | ![](/img/botstopper/pensive.webp) |
| `reject.webp` | Something went wrong, this is a terminal state | ![](/img/botstopper/reject.webp) |
To make your own images at the optimal quality, use the following ffmpeg command:
```text
ffmpeg -i /path/to/image -vf scale=-1:384 happy.webp
```
`ffprobe` should report something like this on the generated images:
```text
Input #0, webp_pipe, from 'happy.webp':
Duration: N/A, bitrate: N/A
Stream #0:0: Video: webp, none, 25 fps, 25 tbr, 25 tbn
```
In testing 384 by 384 pixels gives the best balance between filesize, quality, and clarity.
```text
$ du -hs *
4.0K happy.webp
12K pensive.webp
8.0K reject.webp
```
## Customizing messages
You can customize messages using the following environment variables:
| Message | Environment variable | Default |
| :------------------- | :------------------- | :----------------------------------------- |
| Challenge page title | `CHALLENGE_TITLE` | `Ensuring the security of your connection` |
| Error page title | `ERROR_TITLE` | `Error` |
For example:
```sh
# /etc/techaro-botstopper/gitea.env
CHALLENGE_TITLE="Wait a moment please!"
ERROR_TITLE="Client error"
```

Binary file not shown.

Before

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.1 KiB

View File

@@ -10,7 +10,7 @@ import (
"time"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
)

View File

@@ -6,7 +6,7 @@ import (
"testing"
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
)

View File

@@ -9,7 +9,7 @@ import (
"strings"
"time"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
)

View File

@@ -6,7 +6,7 @@ import (
"testing"
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
)
var _ checker.Impl = &thoth.GeoIPChecker{}

View File

@@ -26,14 +26,17 @@ import (
"github.com/TecharoHQ/anubis/internal/dnsbl"
"github.com/TecharoHQ/anubis/internal/ogtags"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
// challenge implementations
_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
_ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork"
// checker implementations
_ "github.com/TecharoHQ/anubis/lib/checker/remoteaddress"
)
var (

View File

@@ -1,6 +1,6 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.906
// templ: version: v0.3.898
package metarefresh
//lint:file-ignore SA4006 This context is only used if a nested component is present.

View File

@@ -10,7 +10,7 @@ import (
)
type Impl interface {
Check(*http.Request) (bool, error)
Check(*http.Request) (matches bool, err error)
Hash() string
}

42
lib/checker/registry.go Normal file
View File

@@ -0,0 +1,42 @@
package checker
import (
"encoding/json"
"sort"
"sync"
)
type Factory interface {
ValidateConfig(json.RawMessage) error
Create(json.RawMessage) (Impl, error)
}
var (
registry map[string]Factory = map[string]Factory{}
regLock sync.RWMutex
)
func Register(name string, factory Factory) {
regLock.Lock()
defer regLock.Unlock()
registry[name] = factory
}
func Get(name string) (Factory, bool) {
regLock.RLock()
defer regLock.RUnlock()
result, ok := registry[name]
return result, ok
}
func Methods() []string {
regLock.RLock()
defer regLock.RUnlock()
var result []string
for method := range registry {
result = append(result, method)
}
sort.Strings(result)
return result
}

View File

@@ -0,0 +1,106 @@
package remoteaddress
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/netip"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/gaissmai/bart"
)
var (
ErrNoRemoteAddresses = errors.New("remoteaddress: no remote addresses defined")
)
func init() {
checker.Register("remote_address", Factory{})
}
type Factory struct{}
func (Factory) ValidateConfig(inp json.RawMessage) error {
var fc fileConfig
if err := json.Unmarshal([]byte(inp), &fc); err != nil {
return fmt.Errorf("%w: %w", config.ErrUnparseableConfig, err)
}
if err := fc.Valid(); err != nil {
return err
}
return nil
}
func (Factory) Create(inp json.RawMessage) (checker.Impl, error) {
c := struct {
RemoteAddr []netip.Prefix `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
}{}
if err := json.Unmarshal([]byte(inp), &c); err != nil {
return nil, fmt.Errorf("%w: %w", config.ErrUnparseableConfig, err)
}
table := new(bart.Lite)
for _, cidr := range c.RemoteAddr {
table.Insert(cidr)
}
return &Impl{
prefixTable: table,
hash: internal.FastHash(string(inp)),
}, nil
}
type fileConfig struct {
RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
}
func (fc fileConfig) Valid() error {
var errs []error
if len(fc.RemoteAddr) == 0 {
errs = append(errs, ErrNoRemoteAddresses)
}
for _, cidr := range fc.RemoteAddr {
if _, err := netip.ParsePrefix(cidr); err != nil {
errs = append(errs, fmt.Errorf("%w: cidr %q is invalid: %w", config.ErrInvalidCIDR, cidr, err))
}
}
if len(errs) != 0 {
return fmt.Errorf("%w: %w", policy.ErrMisconfiguration, errors.Join(errs...))
}
return nil
}
type Impl struct {
prefixTable *bart.Lite
hash string
}
func (rac *Impl) Check(r *http.Request) (bool, error) {
host := r.Header.Get("X-Real-Ip")
if host == "" {
return false, fmt.Errorf("%w: header X-Real-Ip is not set", policy.ErrMisconfiguration)
}
addr, err := netip.ParseAddr(host)
if err != nil {
return false, fmt.Errorf("%w: %s is not an IP address: %w", policy.ErrMisconfiguration, host, err)
}
return rac.prefixTable.Contains(addr), nil
}
func (rac *Impl) Hash() string {
return rac.hash
}

View File

@@ -0,0 +1,238 @@
package remoteaddress
import (
_ "embed"
"encoding/json"
"errors"
"net/http"
"net/netip"
"strings"
"testing"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/gaissmai/bart"
)
func TestFactoryIsCheckerFactory(t *testing.T) {
if _, ok := (any(Factory{})).(checker.Factory); !ok {
t.Fatal("Factory is not an instance of checker.Factory")
}
}
func TestFactoryValidateConfig(t *testing.T) {
f := Factory{}
for _, tt := range []struct {
name string
data []byte
err error
}{
{
name: "basic valid",
data: []byte(`{
"remote_addresses": [
"1.1.1.1/32"
]
}`),
},
{
name: "not json",
data: []byte(`]`),
err: config.ErrUnparseableConfig,
},
{
name: "no cidr",
data: []byte(`{
"remote_addresses": []
}`),
err: ErrNoRemoteAddresses,
},
{
name: "bad cidr",
data: []byte(`{
"remote_addresses": [
"according to all laws of aviation"
]
}`),
err: config.ErrInvalidCIDR,
},
} {
t.Run(tt.name, func(t *testing.T) {
data := json.RawMessage(tt.data)
if err := f.ValidateConfig(data); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Fatal("validation didn't do what was expected")
}
})
}
}
func TestFactoryCreate(t *testing.T) {
f := Factory{}
for _, tt := range []struct {
name string
data []byte
err error
ip string
match bool
}{
{
name: "basic valid",
data: []byte(`{
"remote_addresses": [
"1.1.1.1/32"
]
}`),
ip: "1.1.1.1",
match: true,
},
{
name: "bad cidr",
data: []byte(`{
"remote_addresses": [
"according to all laws of aviation"
]
}`),
err: config.ErrUnparseableConfig,
},
} {
t.Run(tt.name, func(t *testing.T) {
data := json.RawMessage(tt.data)
impl, err := f.Create(data)
if !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Fatal("creation didn't do what was expected")
}
if tt.err != nil {
return
}
r, err := http.NewRequest(http.MethodGet, "/", nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
if tt.ip != "" {
r.Header.Add("X-Real-Ip", tt.ip)
}
match, err := impl.Check(r)
if tt.match != match {
t.Errorf("match: %v, wanted: %v", match, tt.match)
}
if err != nil && tt.err != nil && !errors.Is(err, tt.err) {
t.Errorf("err: %v, wanted: %v", err, tt.err)
}
if impl.Hash() == "" {
t.Error("hash method returns empty string")
}
})
}
}
func racFromCidrs(t *testing.T, inp []string) *Impl {
t.Helper()
var result Impl
result.prefixTable = new(bart.Lite)
result.hash = internal.FastHash(strings.Join(inp, ","))
for _, cidr := range inp {
pfx, err := netip.ParsePrefix(cidr)
if err != nil {
t.Errorf("prefix %q is invalid: %v", cidr, err)
continue
}
result.prefixTable.Insert(pfx)
}
return &result
}
func TestRemoteAddrChecker(t *testing.T) {
for _, tt := range []struct {
err error
name string
ip string
cidrs []string
ok bool
}{
{
name: "match_ipv4",
cidrs: []string{"0.0.0.0/0"},
ip: "1.1.1.1",
ok: true,
err: nil,
},
{
name: "match_ipv6",
cidrs: []string{"::/0"},
ip: "cafe:babe::",
ok: true,
err: nil,
},
{
name: "not_match_ipv4",
cidrs: []string{"1.1.1.1/32"},
ip: "1.1.1.2",
ok: false,
err: nil,
},
{
name: "not_match_ipv6",
cidrs: []string{"cafe:babe::/128"},
ip: "cafe:babe:4::/128",
ok: false,
err: nil,
},
{
name: "no_ip_set",
cidrs: []string{"::/0"},
ok: false,
err: policy.ErrMisconfiguration,
},
{
name: "invalid_ip",
cidrs: []string{"::/0"},
ip: "According to all natural laws of aviation",
ok: false,
err: policy.ErrMisconfiguration,
},
} {
t.Run(tt.name, func(t *testing.T) {
rac := racFromCidrs(t, tt.cidrs)
r, err := http.NewRequest(http.MethodGet, "/", nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
if tt.ip != "" {
r.Header.Add("X-Real-Ip", tt.ip)
}
ok, err := rac.Check(r)
if tt.ok != ok {
t.Errorf("ok: %v, wanted: %v", ok, tt.ok)
}
if err != nil && tt.err != nil && !errors.Is(err, tt.err) {
t.Errorf("err: %v, wanted: %v", err, tt.err)
}
})
}
}

View File

@@ -1,63 +0,0 @@
{
"loading": "Ladevorgang...",
"why_am_i_seeing": "Warum sehe ich diese Seite?",
"protected_by": "Geschützt durch",
"made_with": "Mit ❤️ gemacht in 🇨🇦",
"mascot_design": "Maskottchen erstellt von",
"ai_companies_explanation": "Diese Seite wird angezeigt, da der Betreiber der Webseite Anubis eingerichtet hat, um sie vor aggressiven KI-Website-Scrapern zu schützen. Diese können Ausfälle der Webseite verursachen, wodurch die Webseite für jeden nicht erreichbar ist.",
"anubis_compromise": "Anubis ist eine Art Kompromiss. Es verwendet die sogenannte Proof-of-Work Methode nach Hashcash, ein Mechanismus, der ursprünglich zur E-Mail-Spam-Bekämpfung entwickelt wurde. Die Idee dahinter ist, dass ein einziger User nur eine kleine Verzögerung hat, auf die Webseite zu gelangen; bei Scrapern kann das allerdings große Auswirkungen haben.",
"hack_purpose": "Man könnte dies als eine Lösung bezeichnen, die gut genug ist, einem etwas Zeit zu verschaffen für Fingerprinting und dem Identifizieren von Headless Browsern, sodass im besten Fall normale User diese Seite garnicht erst zu sehen bekommen.",
"jshelter_note": "Anubis benötigt moderne JavaScript-Features, welche von Plugins wie zB JShelter deaktiviert werden. Bitte deaktiviere also JShelter oder ähnliche Plugins für diese Domain.",
"version_info": "Diese Webseite läuft mit Anubis version",
"try_again": "Nochmal probieren",
"go_home": "Zur Hauptseite",
"contact_webmaster": "oder wenn es sich hier um einen Fehler handelt, kontaktiere bitte den Administrator der Webseite unter",
"connection_security": "Bitte warte einen Moment während wir sicherstellen, dass eine sichere Verbindung verwendet wird.",
"javascript_required": "Es muss leider JavaScript aktiviert werden, um den Check durchführen zu können. Dies ist leider notwendig weil Firmen im KI-Sektor die sozialen Verhältnisse geändert haben, wie Website-Hosting funktioniert. Eine Lösung ohne JavaScript ist in Entwicklung.",
"benchmark_requires_js": "Das Benchmark-Tool benötigt das Aktivieren von JavaScript.",
"difficulty": "Schwierigkeit:",
"algorithm": "Algorithmus:",
"compare": "Vergleich:",
"time": "Zeit",
"iters": "Iterationen",
"time_a": "Zeit A",
"iters_a": "Iterationen A",
"time_b": "Zeit B",
"iters_b": "Iterationen B",
"static_check_endpoint": "Dies ist nur ein Check-Endpunkt, der von beispielsweise einem Reverse-Proxy geprüft werden kann.",
"authorization_required": "Zugriffserlaubnis benötigt",
"cookies_disabled": "Cookies sind in Ihrem Browser deaktiviert. Anubis benötigt Cookies um sicherzustellen, dass es sich hierbei um einen validen Zugriff handelt. Bitte aktiviere Cookies für diese Domain.",
"access_denied": "Zugriff verweigert: Fehlercode",
"dronebl_entry": "Eintrag in DroneBL",
"see_dronebl_lookup": "anzeigen",
"internal_server_error": "Interner Server Error: Misskonfiguration von Anubis. Bitte kontatkiere den Administrator damit dieser die Logs prüfen kann.",
"invalid_redirect": "Ungültige Weiterleitung",
"redirect_not_parseable": "URL der Weiterleitung kann nicht verarbeitet werden",
"redirect_domain_not_allowed": "Domain der Weiterleitung nicht erlaubt",
"failed_to_sign_jwt": "Signierung des JWT fehlgeschlagen",
"invalid_invocation": "Aufrufen von MakeChallenge ungültig",
"client_error_browser": "Client Error: Bitte stelle sicher, dass der Browser aktuell ist und probiere es später erneut.",
"oh_noes": "Vermaledeit!",
"benchmarking_anubis": "Benchmark wird durchgeführt!",
"you_are_not_a_bot": "Sie sind kein Bot!",
"making_sure_not_bot": "Ihr Browser wird geprüft!",
"celphase": "CELPHASE",
"js_web_crypto_error": "Ihr Browser hat leider kein funktionierendes web.crypto Element. Wird eine sichere Verbindung verwendet?",
"js_web_workers_error": "Ihr Browser unterstützt keine Web-Worker (Anubis verwendet diese, damit der Browser nicht unresponsive wird). Ist eventuell ein Plugin wie zB JShelter installiert?",
"js_cookies_error": "Ihr Browser speichert keine Cookies. Anubis verwendet Cookies um ein gültiges Token zu speichern damit es wissen kann, welche Browser bereits geprüft wurden. Bitte aktiviere Cookies für diese Domain. Die Cookie-Namen von Anubis könnten sich jederzeit ändern. Cookie-Namen sind kein Teil der öffentlichen API.",
"js_context_not_secure": "Diese Verbindung ist nicht sicher!",
"js_context_not_secure_msg": "Bitte probiere, dich via HTTPS zu verbinden und lass den Webseiten-Administrator wissen, sauber HTTPS einzurichten. Mehr Informationen unter: <a href=\"https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts#when_is_a_context_considered_secure\">MDN</a>.",
"js_calculating": "Berechnung wird durchgeführt...",
"js_missing_feature": "Fehlendes Feature",
"js_challenge_error": "Fehler während des Checks!",
"js_challenge_error_msg": "Der Check-Algorithmus konnte nicht geladen werden. Bitte lade diese Seite erneut.",
"js_calculating_difficulty": "Berechnung wird durchgeführt...<br/>Schwierigkeit:",
"js_speed": "Geschwindigkeit:",
"js_verification_longer": "Der Check benötigt länger als erwartet. Bitte bleibe auf der Seite.",
"js_success": "Erfolgreich!",
"js_done_took": "Fertig! Dauer:",
"js_iterations": "Iterationen",
"js_finished_reading": "Fertig gelesen, weiter zur Seite →",
"js_calculation_error": "Fehler bei der Berechnung!",
"js_calculation_error_msg": "Fehler bei der Berechnung des Checks:"
}

View File

@@ -1,3 +1,3 @@
{
"supportedLanguages": ["en", "fr", "es", "pt-BR", "de"]
"supportedLanguages": ["en", "fr", "es", "pt-BR"]
}

View File

@@ -3,7 +3,6 @@ package localization
import (
"embed"
"encoding/json"
"github.com/TecharoHQ/anubis"
"net/http"
"strings"
"sync"
@@ -58,14 +57,14 @@ func NewLocalizationService() *LocalizationService {
globalService = &LocalizationService{bundle: bundle}
})
// Safety check - if globalService is still nil, create a minimal one
if globalService == nil {
bundle := i18n.NewBundle(language.English)
bundle.RegisterUnmarshalFunc("json", json.Unmarshal)
globalService = &LocalizationService{bundle: bundle}
}
return globalService
}
@@ -94,13 +93,8 @@ func (sl *SimpleLocalizer) T(messageID string) string {
return sl.Localizer.MustLocalize(&i18n.LocalizeConfig{MessageID: messageID})
}
// GetLocalizer creates a localizer based on the request's Accept-Language header or forcedLanguage option
// GetLocalizer creates a localizer based on the request's Accept-Language header
func GetLocalizer(r *http.Request) *SimpleLocalizer {
var localizer *i18n.Localizer
if anubis.ForcedLanguage == "" {
localizer = NewLocalizationService().GetLocalizerFromRequest(r)
} else {
localizer = NewLocalizationService().GetLocalizer(anubis.ForcedLanguage)
}
localizer := NewLocalizationService().GetLocalizerFromRequest(r)
return &SimpleLocalizer{Localizer: localizer}
}

View File

@@ -27,14 +27,6 @@ func TestLocalizationService(t *testing.T) {
}
})
t.Run("German localization", func(t *testing.T) {
localizer := service.GetLocalizer("de")
result := localizer.MustLocalize(&i18n.LocalizeConfig{MessageID: "loading"})
if result != "Ladevorgang..." {
t.Errorf("Expected 'Ladevorgang...', got '%s'", result)
}
})
t.Run("All required keys exist in English", func(t *testing.T) {
localizer := service.GetLocalizer("en")
requiredKeys := []string{

View File

@@ -4,7 +4,7 @@ import (
"fmt"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
)

View File

@@ -9,7 +9,7 @@ import (
"strings"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/gaissmai/bart"
)

View File

@@ -31,6 +31,7 @@ var (
ErrCantSetBotAndImportValuesAtOnce = errors.New("config.BotOrImport: can't set bot rules and import values at the same time")
ErrMustSetBotOrImportRules = errors.New("config.BotOrImport: rule definition is invalid, you must set either bot rules or an import statement, not both")
ErrStatusCodeNotValid = errors.New("config.StatusCode: status code not valid, must be between 100 and 599")
ErrUnparseableConfig = errors.New("config: can't parse configuration file")
)
type Rule string

View File

@@ -9,7 +9,7 @@ import (
"sync/atomic"
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"

View File

@@ -12,7 +12,7 @@ CacheDirectory=anubis/%i
CacheDirectoryMode=0755
StateDirectory=anubis/%i
StateDirectoryMode=0755
RuntimeDirectory=anubis/%i
RuntimeDirectory=anubis
RuntimeDirectoryMode=0755
ReadWritePaths=/run

2
web/index_templ.go generated
View File

@@ -1,6 +1,6 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.906
// templ: version: v0.3.898
package web
//lint:file-ignore SA4006 This context is only used if a nested component is present.

View File

@@ -212,6 +212,11 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
const t1 = Date.now();
console.log({ hash, nonce });
title.innerHTML = t('success');
status.innerHTML = `${t('done_took')} ${t1 - t0}ms, ${nonce} ${t('iterations')}`;
image.src = imageURL("happy", anubisVersion, basePrefix);
progress.style.display = "none";
if (userReadDetails) {
const container = document.getElementById("progress");
@@ -246,15 +251,17 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
container.onclick = onDetailsExpand;
setTimeout(onDetailsExpand, 30000);
} else {
const redir = window.location.href;
window.location.replace(
u(`${basePrefix}/.within.website/x/cmd/anubis/api/pass-challenge`, {
response: hash,
nonce,
redir,
elapsedTime: t1 - t0,
}),
);
setTimeout(() => {
const redir = window.location.href;
window.location.replace(
u(`${basePrefix}/.within.website/x/cmd/anubis/api/pass-challenge`, {
response: hash,
nonce,
redir,
elapsedTime: t1 - t0,
}),
);
}, 250);
}
} catch (err) {
ohNoes({