Compare commits

..

10 Commits

Author SHA1 Message Date
Jason Cameron
4092180626 Merge branch 'main' into Xe/osiris
Signed-off-by: Jason Cameron <git@jasoncameron.dev>
2025-07-20 21:00:21 -04:00
Xe Iaso
153da4f5ac ci: refactor dockerfile to build with many speed
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 23:52:58 +00:00
Xe Iaso
89b6af05a3 feat(osiris): reload config upon SIGHUP
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 23:52:06 +00:00
Xe Iaso
9a711f1635 ci: amd64 only right now
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 23:39:09 +00:00
Xe Iaso
dabbe63bb6 ci: fix docker build
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 23:38:06 +00:00
Xe Iaso
0aed7d3688 ci: add Docker manifests for Osiris
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 23:32:14 +00:00
Xe Iaso
2af731033c test(osiris): test osiris features and ensure it works
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 22:58:03 +00:00
Xe Iaso
d9c4e37978 feat(osiris): add TCP and TLS fingerprinting
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 20:38:19 +00:00
Xe Iaso
1eafebedbc feat(osiris): serve metrics and healthz
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 20:13:03 +00:00
Xe Iaso
115ee97d1d feat: add initial implementation of osiris, the TLS terminator for Anubis
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-07-18 19:56:22 +00:00
172 changed files with 3251 additions and 3609 deletions

View File

@@ -22,7 +22,8 @@
"unifiedjs.vscode-mdx",
"a-h.templ",
"redhat.vscode-yaml",
"streetsidesoftware.code-spell-checker"
"hashicorp.hcl",
"fredwangwang.vscode-hcl-format"
]
}
}

25
.dockerignore Normal file
View File

@@ -0,0 +1,25 @@
.env
*.deb
*.rpm
# Additional package locks
pnpm-lock.yaml
yarn.lock
# Go binaries and test artifacts
main
*.test
node_modules
# MacOS
.DS_store
# Intellij
.idea
# how does this get here
doc/VERSION
web/static/js/*
!web/static/js/.gitignore

View File

@@ -3,5 +3,4 @@ https
ssh
ubuntu
workarounds
rjack
msgbox
rjack

View File

@@ -23,7 +23,6 @@ bitrate
Bluesky
blueskybot
boi
Bokm
botnet
botstopper
BPort
@@ -149,7 +148,6 @@ inp
internets
IPTo
iptoasn
isp
iss
isset
ivh
@@ -249,7 +247,6 @@ ruleset
runlevels
RUnlock
runtimedir
runtimedirectory
sas
sasl
searchbot
@@ -319,7 +316,6 @@ websites
Webzio
wildbase
withthothmock
wolfbeast
wordpress
Workaround
workdir

View File

@@ -132,7 +132,3 @@ go install(?:\s+[a-z]+\.[-@\w/.]+)+
# hit-count: 1 file-count: 1
# microsoft
\b(?:https?://|)(?:(?:(?:blogs|download\.visualstudio|docs|msdn2?|research)\.|)microsoft|blogs\.msdn)\.co(?:m|\.\w\w)/[-_a-zA-Z0-9()=./%]*
# hit-count: 1 file-count: 1
# data url
\bdata:[-a-zA-Z=;:/0-9+]*,\S*

View File

@@ -2,7 +2,7 @@ name: Docker image builds (pull requests)
on:
pull_request:
branches: [ "main" ]
branches: ["main"]
env:
DOCKER_METADATA_SET_OUTPUT_ENV: "true"
@@ -11,7 +11,32 @@ permissions:
contents: read
jobs:
build:
buildx-bake:
runs-on: ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-tags: true
fetch-depth: 0
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Build and push
id: build
uses: docker/bake-action@76f9fa3a758507623da19f6092dc4089a7e61592 # v6.6.0
with:
source: .
push: true
sbom: true
cache-from: type=gha
cache-to: type=gha,mode=max
set: |
osiris.tags=ttl.sh/techaro/pr-${{ github.event.number }}/osiris:24h
containerbuild:
runs-on: ubuntu-24.04
steps:
- name: Checkout code

View File

@@ -17,7 +17,38 @@ permissions:
pull-requests: write
jobs:
build:
buildx-bake:
runs-on: ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-tags: true
fetch-depth: 0
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Log into registry
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
id: build
uses: docker/bake-action@76f9fa3a758507623da19f6092dc4089a7e61592 # v6.6.0
with:
source: .
push: true
sbom: true
cache-from: type=gha
cache-to: type=gha,mode=max
set: ""
containerbuild:
runs-on: ubuntu-24.04
steps:
- name: Checkout code

View File

@@ -18,9 +18,7 @@ jobs:
- git-push
- healthcheck
- i18n
- palemoon/amd64
#- palemoon/i386
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -45,14 +43,3 @@ jobs:
run: |
cd test/${{ matrix.test }}
backoff-retry --try-count 10 ./test.sh
- name: Sanitize artifact name
if: always()
run: echo "ARTIFACT_NAME=${{ matrix.test }}" | sed 's|/|-|g' >> $GITHUB_ENV
- name: Upload artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
if: always()
with:
name: ${{ env.ARTIFACT_NAME }}
path: test/${{ matrix.test }}/var

View File

@@ -6,6 +6,7 @@
"unifiedjs.vscode-mdx",
"a-h.templ",
"redhat.vscode-yaml",
"streetsidesoftware.code-spell-checker"
"hashicorp.hcl",
"fredwangwang.vscode-hcl-format"
]
}

View File

@@ -1 +1 @@
1.21.3
1.21.0

View File

@@ -30,11 +30,10 @@ import (
"github.com/TecharoHQ/anubis"
"github.com/TecharoHQ/anubis/data"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/internal/thoth"
libanubis "github.com/TecharoHQ/anubis/lib"
"github.com/TecharoHQ/anubis/lib/checker/headerexists"
botPolicy "github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/TecharoHQ/anubis/web"
"github.com/facebookgo/flagenv"
_ "github.com/joho/godotenv/autoload"
@@ -324,7 +323,7 @@ func main() {
if *debugBenchmarkJS {
policy.Bots = []botPolicy.Bot{{
Name: "",
Rules: headerexists.New("User-Agent"),
Rules: botPolicy.NewHeaderExistsChecker("User-Agent"),
Action: config.RuleBenchmark,
}}
}

View File

@@ -0,0 +1,39 @@
package config
import (
"errors"
"fmt"
"net"
)
var (
ErrInvalidHostpost = errors.New("bind: invalid host:port")
)
type Bind struct {
HTTP string `hcl:"http"`
HTTPS string `hcl:"https"`
Metrics string `hcl:"metrics"`
}
func (b *Bind) Valid() error {
var errs []error
if _, _, err := net.SplitHostPort(b.HTTP); err != nil {
errs = append(errs, fmt.Errorf("%w %q: %w", ErrInvalidHostpost, b.HTTP, err))
}
if _, _, err := net.SplitHostPort(b.HTTPS); err != nil {
errs = append(errs, fmt.Errorf("%w %q: %w", ErrInvalidHostpost, b.HTTPS, err))
}
if _, _, err := net.SplitHostPort(b.Metrics); err != nil {
errs = append(errs, fmt.Errorf("%w %q: %w", ErrInvalidHostpost, b.Metrics, err))
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}

View File

@@ -0,0 +1,55 @@
package config
import (
"errors"
"net"
"testing"
)
func TestBindValid(t *testing.T) {
for _, tt := range []struct {
name string
precondition func(t *testing.T)
bind Bind
err error
}{
{
name: "basic",
precondition: nil,
bind: Bind{
HTTP: ":8081",
HTTPS: ":8082",
Metrics: ":8083",
},
err: nil,
},
{
name: "invalid ports",
precondition: func(t *testing.T) {
ln, err := net.Listen("tcp", ":8081")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
},
bind: Bind{
HTTP: "",
HTTPS: "",
Metrics: "",
},
err: ErrInvalidHostpost,
},
} {
t.Run(tt.name, func(t *testing.T) {
if tt.precondition != nil {
tt.precondition(t)
}
if err := tt.bind.Valid(); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("got wrong error from validation function")
}
})
}
}

View File

@@ -0,0 +1,31 @@
package config
import (
"errors"
"fmt"
)
type Toplevel struct {
Bind Bind `hcl:"bind,block"`
Domains []Domain `hcl:"domain,block"`
}
func (t *Toplevel) Valid() error {
var errs []error
if err := t.Bind.Valid(); err != nil {
errs = append(errs, fmt.Errorf("invalid bind block:\n%w", err))
}
for _, d := range t.Domains {
if err := d.Valid(); err != nil {
errs = append(errs, fmt.Errorf("when parsing domain %s: %w", d.Name, err))
}
}
if len(errs) != 0 {
return fmt.Errorf("invalid configuration file:\n%w", errors.Join(errs...))
}
return nil
}

View File

@@ -0,0 +1,66 @@
package config
import (
"errors"
"fmt"
"net/url"
"golang.org/x/net/idna"
)
var (
ErrInvalidDomainName = errors.New("domain: name is invalid")
ErrInvalidDomainTLSConfig = errors.New("domain: TLS config is invalid")
ErrInvalidURL = errors.New("invalid URL")
ErrInvalidURLScheme = errors.New("URL has invalid scheme")
)
type Domain struct {
Name string `hcl:"name,label"`
TLS TLS `hcl:"tls,block"`
Target string `hcl:"target"`
InsecureSkipVerify bool `hcl:"insecure_skip_verify,optional"`
HealthTarget string `hcl:"health_target"`
}
func (d Domain) Valid() error {
var errs []error
if _, err := idna.Lookup.ToASCII(d.Name); err != nil {
errs = append(errs, fmt.Errorf("%w %q: %w", ErrInvalidDomainName, d.Name, err))
}
if err := d.TLS.Valid(); err != nil {
errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidDomainTLSConfig, err))
}
if err := isURLValid(d.Target); err != nil {
errs = append(errs, fmt.Errorf("target has %w %q: %w", ErrInvalidURL, d.Target, err))
}
if err := isURLValid(d.HealthTarget); err != nil {
errs = append(errs, fmt.Errorf("health_target has %w %q: %w", ErrInvalidURL, d.HealthTarget, err))
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}
func isURLValid(input string) error {
u, err := url.Parse(input)
if err != nil {
return err
}
switch u.Scheme {
case "http", "https", "h2c", "unix":
// do nothing
default:
return fmt.Errorf("%w %s has scheme %s (want http, https, h2c, unix)", ErrInvalidURLScheme, input, u.Scheme)
}
return nil
}

View File

@@ -0,0 +1,89 @@
package config
import (
"errors"
"testing"
)
func TestDomainValid(t *testing.T) {
for _, tt := range []struct {
name string
input Domain
err error
}{
{
name: "simple happy path",
input: Domain{
Name: "anubis.techaro.lol",
TLS: TLS{
Cert: "./testdata/tls/selfsigned.crt",
Key: "./testdata/tls/selfsigned.key",
},
Target: "http://localhost:3000",
HealthTarget: "http://localhost:9091/healthz",
},
},
{
name: "invalid domain name",
input: Domain{
Name: "\uFFFD.techaro.lol",
TLS: TLS{
Cert: "./testdata/tls/selfsigned.crt",
Key: "./testdata/tls/selfsigned.key",
},
Target: "http://localhost:3000",
HealthTarget: "http://localhost:9091/healthz",
},
err: ErrInvalidDomainName,
},
{
name: "invalid tls config",
input: Domain{
Name: "anubis.techaro.lol",
TLS: TLS{
Cert: "./testdata/tls/invalid.crt",
Key: "./testdata/tls/invalid.key",
},
Target: "http://localhost:3000",
HealthTarget: "http://localhost:9091/healthz",
},
err: ErrInvalidDomainTLSConfig,
},
{
name: "invalid URL",
input: Domain{
Name: "anubis.techaro.lol",
TLS: TLS{
Cert: "./testdata/tls/selfsigned.crt",
Key: "./testdata/tls/selfsigned.key",
},
Target: "file://[::1:3000",
HealthTarget: "file://[::1:9091/healthz",
},
err: ErrInvalidURL,
},
{
name: "wrong URL scheme",
input: Domain{
Name: "anubis.techaro.lol",
TLS: TLS{
Cert: "./testdata/tls/selfsigned.crt",
Key: "./testdata/tls/selfsigned.key",
},
Target: "file://localhost:3000",
HealthTarget: "file://localhost:9091/healthz",
},
err: ErrInvalidURLScheme,
},
} {
t.Run(tt.name, func(t *testing.T) {
if err := tt.input.Valid(); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("got wrong error from validation function")
} else {
t.Log(err)
}
})
}
}

View File

@@ -0,0 +1 @@
aorsentaeiorsntoiearnstoieanrsoietnaioresntoeiar

View File

@@ -0,0 +1 @@
aorsentaeiorsntoiearnstoieanrsoietnaioresntoeiar

View File

@@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBnzCCAVGgAwIBAgIUAw8funCpiB3ZAAPoWdSCWnzbsFIwBQYDK2VwMEUxCzAJ
BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l
dCBXaWRnaXRzIFB0eSBMdGQwHhcNMjUwNzE4MTkwMjM1WhcNMjUwODE3MTkwMjM1
WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwY
SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCowBQYDK2VwAyEAcXDHXV3vgpvjtTaz
s0Oj/73rMr06bhyGGhleYS1MNoWjUzBRMB0GA1UdDgQWBBQwmfKPthucFHB6Wfgz
2Nj5nkMQOjAfBgNVHSMEGDAWgBQwmfKPthucFHB6Wfgz2Nj5nkMQOjAPBgNVHRMB
Af8EBTADAQH/MAUGAytlcANBALBYbULlGwB7Ro0UTgUoQDNxEvayn3qzVFHIt7lC
/2/NzNBkk4yPT+a4mbRuydxLkv+JIvmQbarZxpksYnWlCAM=
-----END CERTIFICATE-----

View File

@@ -0,0 +1,3 @@
-----BEGIN PRIVATE KEY-----
MC4CAQAwBQYDK2VwBCIEIOHKoX22Mha6SnnpLm34fSSfTUDbRiDCi6N1nOgTOlds
-----END PRIVATE KEY-----

View File

@@ -0,0 +1,40 @@
package config
import (
"crypto/tls"
"errors"
"fmt"
"os"
)
var (
ErrCantReadTLS = errors.New("tls: can't read TLS")
ErrInvalidTLSKeypair = errors.New("tls: can't parse TLS keypair")
)
type TLS struct {
Cert string `hcl:"cert"`
Key string `hcl:"key"`
}
func (t TLS) Valid() error {
var errs []error
if _, err := os.Stat(t.Cert); err != nil {
errs = append(errs, fmt.Errorf("%w certificate %s: %w", ErrCantReadTLS, t.Cert, err))
}
if _, err := os.Stat(t.Key); err != nil {
errs = append(errs, fmt.Errorf("%w key %s: %w", ErrCantReadTLS, t.Key, err))
}
if _, err := tls.LoadX509KeyPair(t.Cert, t.Key); err != nil {
errs = append(errs, fmt.Errorf("%w (%s, %s): %w", ErrInvalidTLSKeypair, t.Cert, t.Key, err))
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}

View File

@@ -0,0 +1,48 @@
package config
import (
"errors"
"testing"
)
func TestTLSValid(t *testing.T) {
for _, tt := range []struct {
name string
input TLS
err error
}{
{
name: "simple selfsigned",
input: TLS{
Cert: "./testdata/tls/selfsigned.crt",
Key: "./testdata/tls/selfsigned.key",
},
},
{
name: "files don't exist",
input: TLS{
Cert: "./testdata/tls/nonexistent.crt",
Key: "./testdata/tls/nonexistent.key",
},
err: ErrCantReadTLS,
},
{
name: "invalid keypair",
input: TLS{
Cert: "./testdata/tls/invalid.crt",
Key: "./testdata/tls/invalid.key",
},
err: ErrInvalidTLSKeypair,
},
} {
t.Run(tt.name, func(t *testing.T) {
if err := tt.input.Valid(); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("got wrong error from validation function")
} else {
t.Log(err)
}
})
}
}

View File

@@ -0,0 +1,85 @@
package entrypoint
import (
"context"
"fmt"
"log/slog"
"net"
"github.com/TecharoHQ/anubis/cmd/osiris/internal/config"
"github.com/TecharoHQ/anubis/internal"
"github.com/hashicorp/hcl/v2/hclsimple"
"golang.org/x/sync/errgroup"
healthv1 "google.golang.org/grpc/health/grpc_health_v1"
)
type Options struct {
ConfigFname string
}
func Main(ctx context.Context, opts Options) error {
internal.SetHealth("osiris", healthv1.HealthCheckResponse_NOT_SERVING)
var cfg config.Toplevel
if err := hclsimple.DecodeFile(opts.ConfigFname, nil, &cfg); err != nil {
return fmt.Errorf("can't read configuration file %s:\n\n%w", opts.ConfigFname, err)
}
if err := cfg.Valid(); err != nil {
return fmt.Errorf("configuration file %s is invalid:\n\n%w", opts.ConfigFname, err)
}
rtr, err := NewRouter(cfg)
if err != nil {
return err
}
rtr.opts = opts
go rtr.backgroundReloadConfig(ctx)
g, gCtx := errgroup.WithContext(ctx)
// HTTP
g.Go(func() error {
ln, err := net.Listen("tcp", cfg.Bind.HTTP)
if err != nil {
return fmt.Errorf("(HTTP) can't bind to tcp %s: %w", cfg.Bind.HTTP, err)
}
defer ln.Close()
go func(ctx context.Context) {
<-ctx.Done()
ln.Close()
}(ctx)
slog.Info("listening", "for", "http", "bind", cfg.Bind.HTTP)
return rtr.HandleHTTP(gCtx, ln)
})
// HTTPS
g.Go(func() error {
ln, err := net.Listen("tcp", cfg.Bind.HTTPS)
if err != nil {
return fmt.Errorf("(https) can't bind to tcp %s: %w", cfg.Bind.HTTPS, err)
}
defer ln.Close()
go func(ctx context.Context) {
<-ctx.Done()
ln.Close()
}(ctx)
slog.Info("listening", "for", "https", "bind", cfg.Bind.HTTPS)
return rtr.HandleHTTPS(gCtx, ln)
})
// Metrics
g.Go(func() error {
return rtr.ListenAndServeMetrics(gCtx, cfg.Bind.Metrics)
})
internal.SetHealth("osiris", healthv1.HealthCheckResponse_SERVING)
return g.Wait()
}

View File

@@ -0,0 +1,93 @@
package entrypoint
import (
"context"
"errors"
"net"
"net/http"
"os"
"path/filepath"
"testing"
"time"
)
func TestMainGoodConfig(t *testing.T) {
files, err := os.ReadDir("./testdata/good")
if err != nil {
t.Fatal(err)
}
for _, st := range files {
t.Run(st.Name(), func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cfg := loadConfig(t, filepath.Join("testdata", "good", st.Name()))
go func(ctx context.Context) {
if err := Main(ctx, Options{
ConfigFname: filepath.Join("testdata", "good", st.Name()),
}); err != nil {
var netOpErr *net.OpError
switch {
case errors.Is(err, context.Canceled):
// Context was canceled, this is expected
return
case errors.As(err, &netOpErr):
// Network operation error occurred
t.Logf("Network operation error: %v", netOpErr)
return
case errors.Is(err, http.ErrServerClosed):
// Server was closed, this is expected
return
default:
// Other unexpected error
panic(err)
}
}
}(ctx)
wait := 5 * time.Millisecond
for i := range make([]struct{}, 10) {
if i != 0 {
time.Sleep(wait)
wait = wait * 2
}
t.Logf("try %d (wait=%s)", i+1, wait)
resp, err := http.Get("http://localhost" + cfg.Bind.Metrics + "/readyz")
if err != nil {
continue
}
if resp.StatusCode != http.StatusOK {
continue
}
cancel()
return
}
t.Fatal("router initialization did not work")
})
}
}
func TestMainBadConfig(t *testing.T) {
files, err := os.ReadDir("./testdata/bad")
if err != nil {
t.Fatal(err)
}
for _, st := range files {
t.Run(st.Name(), func(t *testing.T) {
if err := Main(t.Context(), Options{
ConfigFname: filepath.Join("testdata", "bad", st.Name()),
}); err == nil {
t.Error("wanted an error but got none")
} else {
t.Log(err)
}
})
}
}

View File

@@ -0,0 +1,35 @@
package entrypoint
import (
"crypto/tls"
"net"
"net/http"
"net/http/httputil"
"net/url"
"golang.org/x/net/http2"
)
func newH2CReverseProxy(target *url.URL) *httputil.ReverseProxy {
target.Scheme = "http"
director := func(req *http.Request) {
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.Host = target.Host
}
// Use h2c transport
transport := &http2.Transport{
AllowHTTP: true,
DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
// Just do plain TCP (h2c)
return net.Dial(network, addr)
},
}
return &httputil.ReverseProxy{
Director: director,
Transport: transport,
}
}

View File

@@ -0,0 +1,51 @@
package entrypoint
import (
"net/http"
"net/http/httptest"
"net/url"
"testing"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
)
func newH2cServer(t *testing.T, h http.Handler) *httptest.Server {
t.Helper()
h2s := &http2.Server{}
srv := httptest.NewServer(h2c.NewHandler(h, h2s))
t.Cleanup(func() {
srv.Close()
})
return srv
}
func TestH2CReverseProxy(t *testing.T) {
h := &ackHandler{}
srv := newH2cServer(t, h)
u, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
rp := httptest.NewServer(newH2CReverseProxy(u))
defer rp.Close()
resp, err := rp.Client().Get(rp.URL)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("wrong status code from reverse proxy: %d", resp.StatusCode)
}
if !h.ack {
t.Error("h2c handler was not executed")
}
}

View File

@@ -0,0 +1,72 @@
package entrypoint
import (
"bytes"
"fmt"
"log/slog"
"net/http"
"sort"
"github.com/TecharoHQ/anubis/internal"
healthv1 "google.golang.org/grpc/health/grpc_health_v1"
)
func healthz(w http.ResponseWriter, r *http.Request) {
services, err := internal.HealthSrv.List(r.Context(), nil)
if err != nil {
slog.Error("can't get list of services", "err", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var keys []string
for k := range services.Statuses {
if k == "" {
continue
}
keys = append(keys, k)
}
sort.Strings(keys)
var msg bytes.Buffer
var healthy bool = true
for _, k := range keys {
st := services.Statuses[k].GetStatus()
fmt.Fprintf(&msg, "%s: %s\n", k, st)
switch st {
case healthv1.HealthCheckResponse_SERVING:
// do nothing
default:
healthy = false
}
}
if !healthy {
w.WriteHeader(http.StatusInternalServerError)
}
w.Write(msg.Bytes())
}
func readyz(w http.ResponseWriter, r *http.Request) {
st, ok := internal.GetHealth("osiris")
if !ok {
slog.Error("health service osiris does not exist, file a bug")
http.Error(w, "health service osiris does not exist", http.StatusExpectationFailed)
}
switch st {
case healthv1.HealthCheckResponse_NOT_SERVING:
http.Error(w, "NOT OK", http.StatusInternalServerError)
return
case healthv1.HealthCheckResponse_SERVING:
fmt.Fprintln(w, "OK")
return
default:
http.Error(w, "UNKNOWN", http.StatusFailedDependency)
return
}
}

View File

@@ -0,0 +1,66 @@
package entrypoint
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/TecharoHQ/anubis/internal"
healthv1 "google.golang.org/grpc/health/grpc_health_v1"
)
func TestHealthz(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(healthz))
internal.SetHealth("osiris", healthv1.HealthCheckResponse_NOT_SERVING)
resp, err := srv.Client().Get(srv.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
t.Errorf("wanted not ready but got %d", resp.StatusCode)
}
internal.SetHealth("osiris", healthv1.HealthCheckResponse_SERVING)
resp, err = srv.Client().Get(srv.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("wanted ready but got %d", resp.StatusCode)
}
}
func TestReadyz(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(readyz))
internal.SetHealth("osiris", healthv1.HealthCheckResponse_NOT_SERVING)
resp, err := srv.Client().Get(srv.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
t.Errorf("wanted not ready but got %d", resp.StatusCode)
}
internal.SetHealth("osiris", healthv1.HealthCheckResponse_SERVING)
resp, err = srv.Client().Get(srv.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("wanted ready but got %d", resp.StatusCode)
}
}

View File

@@ -0,0 +1,320 @@
package entrypoint
import (
"context"
"crypto/tls"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/TecharoHQ/anubis/cmd/osiris/internal/config"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/internal/fingerprint"
"github.com/felixge/httpsnoop"
"github.com/hashicorp/hcl/v2/hclsimple"
"github.com/lum8rjack/go-ja4h"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
ErrTargetInvalid = errors.New("[unexpected] target invalid")
ErrNoHandler = errors.New("[unexpected] no handler for domain")
ErrInvalidTLSKeypair = errors.New("[unexpected] invalid TLS keypair")
ErrNoCert = errors.New("this server does not have a certificate for that domain")
requestsPerDomain = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "techaro",
Subsystem: "osiris",
Name: "request_count",
}, []string{"domain", "method", "response_code"})
responseTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "techaro",
Subsystem: "osiris",
Name: "response_time",
}, []string{"domain"})
unresolvedRequests = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: "techaro",
Subsystem: "osiris",
Name: "unresolved_requests",
})
)
type Router struct {
lock sync.RWMutex
routes map[string]http.Handler
tlsCerts map[string]*tls.Certificate
opts Options
}
func (rtr *Router) setConfig(c config.Toplevel) error {
var errs []error
newMap := map[string]http.Handler{}
newCerts := map[string]*tls.Certificate{}
for _, d := range c.Domains {
var domainErrs []error
u, err := url.Parse(d.Target)
if err != nil {
domainErrs = append(domainErrs, fmt.Errorf("%w %q: %v", ErrTargetInvalid, d.Target, err))
}
var h http.Handler
if u != nil {
switch u.Scheme {
case "http", "https":
rp := httputil.NewSingleHostReverseProxy(u)
if d.InsecureSkipVerify {
rp.Transport = &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
}
h = rp
case "h2c":
h = newH2CReverseProxy(u)
case "unix":
h = &httputil.ReverseProxy{
Director: func(r *http.Request) {
r.URL.Scheme = "http"
r.URL.Host = d.Name
r.Host = d.Name
},
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", strings.TrimPrefix(d.Target, "unix://"))
},
},
}
}
}
if h == nil {
domainErrs = append(domainErrs, ErrNoHandler)
}
newMap[d.Name] = h
cert, err := tls.LoadX509KeyPair(d.TLS.Cert, d.TLS.Key)
if err != nil {
domainErrs = append(domainErrs, fmt.Errorf("%w: %w", ErrInvalidTLSKeypair, err))
}
newCerts[d.Name] = &cert
if len(domainErrs) != 0 {
errs = append(errs, fmt.Errorf("invalid domain %s: %w", d.Name, errors.Join(domainErrs...)))
}
}
if len(errs) != 0 {
return fmt.Errorf("can't compile config to routing map: %w", errors.Join(errs...))
}
rtr.lock.Lock()
rtr.routes = newMap
rtr.tlsCerts = newCerts
rtr.lock.Unlock()
return nil
}
func (rtr *Router) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
rtr.lock.RLock()
cert, ok := rtr.tlsCerts[hello.ServerName]
rtr.lock.RUnlock()
if !ok {
return nil, ErrNoCert
}
return cert, nil
}
func (rtr *Router) loadConfig() error {
slog.Info("reloading config", "fname", rtr.opts.ConfigFname)
var cfg config.Toplevel
if err := hclsimple.DecodeFile(rtr.opts.ConfigFname, nil, &cfg); err != nil {
return err
}
if err := cfg.Valid(); err != nil {
return err
}
if err := rtr.setConfig(cfg); err != nil {
return err
}
slog.Info("done!")
return nil
}
func (rtr *Router) backgroundReloadConfig(ctx context.Context) {
t := time.NewTicker(time.Hour)
defer t.Stop()
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGHUP)
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := rtr.loadConfig(); err != nil {
slog.Error("can't reload config", "fname", rtr.opts.ConfigFname, "err", err)
}
case <-ch:
if err := rtr.loadConfig(); err != nil {
slog.Error("can't reload config", "fname", rtr.opts.ConfigFname, "err", err)
}
}
}
}
func NewRouter(c config.Toplevel) (*Router, error) {
result := &Router{
routes: map[string]http.Handler{},
}
if err := result.setConfig(c); err != nil {
return nil, err
}
return result, nil
}
func (rtr *Router) HandleHTTP(ctx context.Context, ln net.Listener) error {
srv := http.Server{
Handler: rtr,
ErrorLog: internal.GetFilteredHTTPLogger(),
}
go func(ctx context.Context) {
<-ctx.Done()
srv.Close()
}(ctx)
return srv.Serve(ln)
}
func (rtr *Router) HandleHTTPS(ctx context.Context, ln net.Listener) error {
tc := &tls.Config{
GetCertificate: rtr.GetCertificate,
}
srv := &http.Server{
Handler: rtr,
ErrorLog: internal.GetFilteredHTTPLogger(),
TLSConfig: tc,
}
go func(ctx context.Context) {
<-ctx.Done()
srv.Close()
}(ctx)
fingerprint.ApplyTLSFingerprinter(srv)
return srv.ServeTLS(ln, "", "")
}
func (rtr *Router) ListenAndServeMetrics(ctx context.Context, addr string) error {
ln, err := net.Listen("tcp", addr)
if err != nil {
return fmt.Errorf("(metrics) can't bind to tcp %s: %w", addr, err)
}
defer ln.Close()
go func(ctx context.Context) {
<-ctx.Done()
ln.Close()
}(ctx)
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/readyz", readyz)
mux.HandleFunc("/healthz", healthz)
slog.Info("listening", "for", "metrics", "bind", addr)
srv := http.Server{
Addr: addr,
Handler: mux,
ErrorLog: internal.GetFilteredHTTPLogger(),
}
go func(ctx context.Context) {
<-ctx.Done()
srv.Close()
}(ctx)
return srv.Serve(ln)
}
func (rtr *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var host = r.Host
if strings.Contains(host, ":") {
host, _, _ = net.SplitHostPort(host)
}
var h http.Handler
var ok bool
ja4hFP := ja4h.JA4H(r)
slog.Info("got request", "method", r.Method, "host", host, "path", r.URL.Path)
rtr.lock.RLock()
h, ok = rtr.routes[host]
rtr.lock.RUnlock()
if !ok {
unresolvedRequests.Inc()
http.NotFound(w, r) // TODO(Xe): brand this
return
}
r.Header.Set("X-Http-Ja4h-Fingerprint", ja4hFP)
if fp := fingerprint.GetTLSFingerprint(r); fp != nil {
if ja3n := fp.JA3N(); ja3n != nil {
r.Header.Set("X-Tls-Ja3n-Fingerprint", ja3n.String())
}
if ja4 := fp.JA4(); ja4 != nil {
r.Header.Set("X-Tls-Ja4-Fingerprint", ja4.String())
}
}
if tcpFP := fingerprint.GetTCPFingerprint(r); tcpFP != nil {
r.Header.Set("X-Tcp-Ja4t-Fingerprint", tcpFP.String())
}
m := httpsnoop.CaptureMetrics(h, w, r)
requestsPerDomain.WithLabelValues(host, r.Method, fmt.Sprint(m.Code)).Inc()
responseTime.WithLabelValues(host).Observe(float64(m.Duration.Milliseconds()))
slog.Info("request completed", "host", host, "method", r.Method, "response_code", m.Code, "duration_ms", m.Duration.Milliseconds())
}

View File

@@ -0,0 +1,319 @@
package entrypoint
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/TecharoHQ/anubis/cmd/osiris/internal/config"
"github.com/hashicorp/hcl/v2/hclsimple"
)
func loadConfig(t *testing.T, fname string) config.Toplevel {
t.Helper()
var cfg config.Toplevel
if err := hclsimple.DecodeFile(fname, nil, &cfg); err != nil {
t.Fatalf("can't read configuration file %s: %v", fname, err)
}
if err := cfg.Valid(); err != nil {
t.Errorf("configuration file %s is invalid: %v", "./testdata/selfsigned.hcl", err)
}
return cfg
}
func newRouter(t *testing.T, cfg config.Toplevel) *Router {
t.Helper()
rtr, err := NewRouter(cfg)
if err != nil {
t.Fatal(err)
}
return rtr
}
func TestNewRouter(t *testing.T) {
cfg := loadConfig(t, "./testdata/good/selfsigned.hcl")
rtr := newRouter(t, cfg)
srv := httptest.NewServer(rtr)
defer srv.Close()
}
func TestNewRouterFails(t *testing.T) {
cfg := loadConfig(t, "./testdata/good/selfsigned.hcl")
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "test1.internal",
TLS: config.TLS{
Cert: "./testdata/tls/invalid.crt",
Key: "./testdata/tls/invalid.key",
},
Target: cfg.Domains[0].Target,
HealthTarget: cfg.Domains[0].HealthTarget,
})
rtr, err := NewRouter(cfg)
if err == nil {
t.Fatal("wanted an error but got none")
}
srv := httptest.NewServer(rtr)
defer srv.Close()
}
func TestRouterSetConfig(t *testing.T) {
for _, tt := range []struct {
name string
configFname string
mutation func(cfg config.Toplevel) config.Toplevel
err error
}{
{
name: "basic",
configFname: "./testdata/good/selfsigned.hcl",
mutation: func(cfg config.Toplevel) config.Toplevel {
return cfg
},
},
{
name: "all schemes",
configFname: "./testdata/good/selfsigned.hcl",
mutation: func(cfg config.Toplevel) config.Toplevel {
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "http.internal",
TLS: cfg.Domains[0].TLS,
Target: "http://[::1]:3000",
HealthTarget: cfg.Domains[0].HealthTarget,
})
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "https.internal",
TLS: cfg.Domains[0].TLS,
Target: "https://[::1]:3000",
HealthTarget: cfg.Domains[0].HealthTarget,
})
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "h2c.internal",
TLS: cfg.Domains[0].TLS,
Target: "h2c://[::1]:3000",
HealthTarget: cfg.Domains[0].HealthTarget,
})
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "unix.internal",
TLS: cfg.Domains[0].TLS,
Target: "unix://foo.sock",
HealthTarget: cfg.Domains[0].HealthTarget,
})
return cfg
},
},
{
name: "invalid TLS",
configFname: "./testdata/good/selfsigned.hcl",
mutation: func(cfg config.Toplevel) config.Toplevel {
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "test1.internal",
TLS: config.TLS{
Cert: "./testdata/tls/invalid.crt",
Key: "./testdata/tls/invalid.key",
},
Target: cfg.Domains[0].Target,
HealthTarget: cfg.Domains[0].HealthTarget,
})
return cfg
},
err: ErrInvalidTLSKeypair,
},
{
name: "target is not a valid URL",
configFname: "./testdata/good/selfsigned.hcl",
mutation: func(cfg config.Toplevel) config.Toplevel {
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "test1.internal",
TLS: cfg.Domains[0].TLS,
Target: "http://[::1:443",
HealthTarget: cfg.Domains[0].HealthTarget,
})
return cfg
},
err: ErrTargetInvalid,
},
{
name: "invalid target scheme",
configFname: "./testdata/good/selfsigned.hcl",
mutation: func(cfg config.Toplevel) config.Toplevel {
cfg.Domains = append(cfg.Domains, config.Domain{
Name: "test1.internal",
TLS: cfg.Domains[0].TLS,
Target: "foo://",
HealthTarget: cfg.Domains[0].HealthTarget,
})
return cfg
},
err: ErrNoHandler,
},
} {
t.Run(tt.name, func(t *testing.T) {
cfg := loadConfig(t, tt.configFname)
rtr := newRouter(t, cfg)
cfg = tt.mutation(cfg)
if err := rtr.setConfig(cfg); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("got wrong error from rtr.setConfig function")
}
})
}
}
type ackHandler struct {
ack bool
}
func (ah *ackHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ah.ack = true
fmt.Fprintln(w, "OK")
}
func (ah *ackHandler) Reset() {
ah.ack = false
}
func newUnixServer(t *testing.T, h http.Handler) string {
sockName := filepath.Join(t.TempDir(), "s")
ln, err := net.Listen("unix", sockName)
if err != nil {
t.Fatalf("can't listen on %s: %v", sockName, err)
}
t.Cleanup(func() {
ln.Close()
os.Remove(sockName)
})
go func(ctx context.Context) {
srv := &http.Server{
Handler: h,
}
go func() {
<-ctx.Done()
srv.Close()
}()
srv.Serve(ln)
}(t.Context())
return "unix://" + sockName
}
func TestRouterGetCertificate(t *testing.T) {
cfg := loadConfig(t, "./testdata/good/selfsigned.hcl")
rtr := newRouter(t, cfg)
for _, tt := range []struct {
domainName string
err error
}{
{
domainName: "osiris.local.cetacean.club",
},
{
domainName: "whacky-fun.local",
err: ErrNoCert,
},
} {
t.Run(tt.domainName, func(t *testing.T) {
if _, err := rtr.GetCertificate(&tls.ClientHelloInfo{ServerName: tt.domainName}); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("got wrong error from rtr.GetCertificate")
}
})
}
}
func TestRouterServeAllProtocols(t *testing.T) {
cfg := loadConfig(t, "./testdata/good/all_protocols.hcl")
httpAckHandler := &ackHandler{}
httpsAckHandler := &ackHandler{}
h2cAckHandler := &ackHandler{}
unixAckHandler := &ackHandler{}
httpSrv := httptest.NewServer(httpAckHandler)
httpsSrv := httptest.NewTLSServer(httpsAckHandler)
h2cSrv := newH2cServer(t, h2cAckHandler)
unixPath := newUnixServer(t, unixAckHandler)
cfg.Domains[0].Target = httpSrv.URL
cfg.Domains[1].Target = httpsSrv.URL
cfg.Domains[2].Target = strings.ReplaceAll(h2cSrv.URL, "http:", "h2c:")
cfg.Domains[3].Target = unixPath
// enc := json.NewEncoder(os.Stderr)
// enc.SetIndent("", " ")
// enc.Encode(cfg)
rtr := newRouter(t, cfg)
cli := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
t.Run("plain http", func(t *testing.T) {
ln, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
ln.Close()
})
go rtr.HandleHTTP(t.Context(), ln)
serverURL := "http://" + ln.Addr().String()
t.Log(serverURL)
for _, d := range cfg.Domains {
t.Run(d.Name, func(t *testing.T) {
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, serverURL, nil)
if err != nil {
t.Fatal(err)
}
req.Host = d.Name
resp, err := cli.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("wrong status code %d", resp.StatusCode)
}
})
}
})
}

View File

View File

@@ -0,0 +1,15 @@
bind {
http = ":65530"
https = ":65531"
metrics = ":65532"
}
domain "osiris.local.cetacean.club" {
tls {
cert = "./testdata/invalid.crt"
key = "./testdata/invalid.key"
}
target = "http://localhost:3000"
health_target = "http://localhost:9091/healthz"
}

View File

@@ -0,0 +1,46 @@
bind {
http = ":65520"
https = ":65521"
metrics = ":65522"
}
domain "http.internal" {
tls {
cert = "./testdata/selfsigned.crt"
key = "./testdata/selfsigned.key"
}
target = "http://localhost:65510" # XXX(Xe) this is overwritten
health_target = "http://localhost:9091/healthz"
}
domain "https.internal" {
tls {
cert = "./testdata/selfsigned.crt"
key = "./testdata/selfsigned.key"
}
target = "https://localhost:65511" # XXX(Xe) this is overwritten
insecure_skip_verify = true
health_target = "http://localhost:9091/healthz"
}
domain "h2c.internal" {
tls {
cert = "./testdata/selfsigned.crt"
key = "./testdata/selfsigned.key"
}
target = "h2c://localhost:65511" # XXX(Xe) this is overwritten
health_target = "http://localhost:9091/healthz"
}
domain "unix.internal" {
tls {
cert = "./testdata/selfsigned.crt"
key = "./testdata/selfsigned.key"
}
target = "http://localhost:65511" # XXX(Xe) this is overwritten
health_target = "http://localhost:9091/healthz"
}

View File

@@ -0,0 +1,15 @@
bind {
http = ":65530"
https = ":65531"
metrics = ":65532"
}
domain "osiris.local.cetacean.club" {
tls {
cert = "./testdata/selfsigned.crt"
key = "./testdata/selfsigned.key"
}
target = "http://localhost:3000"
health_target = "http://localhost:9091/healthz"
}

View File

@@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBnzCCAVGgAwIBAgIUOLTjSYOjFk00IemtFTC4oEZs988wBQYDK2VwMEUxCzAJ
BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l
dCBXaWRnaXRzIFB0eSBMdGQwHhcNMjUwNzE4MjEyNDIzWhcNMjUwODE3MjEyNDIz
WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwY
SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCowBQYDK2VwAyEAPHphABS15+4VV6R1
vYzBQYIycQmOmlbA8QcfwzuB2VajUzBRMB0GA1UdDgQWBBT2s+MQ4AR6cbK4V0+d
XZnok1orhDAfBgNVHSMEGDAWgBT2s+MQ4AR6cbK4V0+dXZnok1orhDAPBgNVHRMB
Af8EBTADAQH/MAUGAytlcANBAOdoJbRMnHmkEETzVtXP+jkAI9yQNRXujnglApGP
8I5pvIYVgYCgoQrnb4haVWFldHM1T9H698n19e/egfFb+w4=
-----END CERTIFICATE-----

View File

@@ -0,0 +1,3 @@
-----BEGIN PRIVATE KEY-----
MC4CAQAwBQYDK2VwBCIEIBop42tiZ0yzhaKo9NAc0PlAyBsE8NAE0i9Z7s2lgZuR
-----END PRIVATE KEY-----

43
cmd/osiris/main.go Normal file
View File

@@ -0,0 +1,43 @@
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/TecharoHQ/anubis"
"github.com/TecharoHQ/anubis/cmd/osiris/internal/entrypoint"
"github.com/TecharoHQ/anubis/internal"
"github.com/facebookgo/flagenv"
)
var (
configFname = flag.String("config", "./osiris.hcl", "Configuration file (HCL), see docs")
slogLevel = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
versionFlag = flag.Bool("version", false, "if true, show version information then quit")
)
func main() {
flagenv.Parse()
flag.Parse()
if *versionFlag {
fmt.Println("Osiris", anubis.Version)
return
}
internal.InitSlog(*slogLevel)
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
if err := entrypoint.Main(ctx, entrypoint.Options{
ConfigFname: *configFname,
}); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}

15
cmd/osiris/osiris.hcl Normal file
View File

@@ -0,0 +1,15 @@
bind {
http = ":3004"
https = ":3005"
metrics = ":9091"
}
domain "osiris.local.cetacean.club" {
tls {
cert = "./internal/config/testdata/tls/selfsigned.crt"
key = "./internal/config/testdata/tls/selfsigned.key"
}
target = "http://localhost:3000"
health_target = "http://localhost:9091/healthz"
}

View File

@@ -12,7 +12,6 @@ import (
"regexp"
"strings"
"github.com/TecharoHQ/anubis/lib/checker/expression"
"github.com/TecharoHQ/anubis/lib/policy/config"
"sigs.k8s.io/yaml"
@@ -38,11 +37,11 @@ type RobotsRule struct {
}
type AnubisRule struct {
Expression *expression.Config `yaml:"expression,omitempty" json:"expression,omitempty"`
Challenge *config.ChallengeRules `yaml:"challenge,omitempty" json:"challenge,omitempty"`
Weight *config.Weight `yaml:"weight,omitempty" json:"weight,omitempty"`
Name string `yaml:"name" json:"name"`
Action string `yaml:"action" json:"action"`
Expression *config.ExpressionOrList `yaml:"expression,omitempty" json:"expression,omitempty"`
Challenge *config.ChallengeRules `yaml:"challenge,omitempty" json:"challenge,omitempty"`
Weight *config.Weight `yaml:"weight,omitempty" json:"weight,omitempty"`
Name string `yaml:"name" json:"name"`
Action string `yaml:"action" json:"action"`
}
func init() {
@@ -225,11 +224,11 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
}
if userAgent == "*" {
rule.Expression = &expression.Config{
rule.Expression = &config.ExpressionOrList{
All: []string{"true"}, // Always applies
}
} else {
rule.Expression = &expression.Config{
rule.Expression = &config.ExpressionOrList{
All: []string{fmt.Sprintf("userAgent.contains(%q)", userAgent)},
}
}
@@ -250,11 +249,11 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
rule.Name = fmt.Sprintf("%s-global-restriction-%d", *policyName, ruleCounter)
rule.Action = "WEIGH"
rule.Weight = &config.Weight{Adjust: 20} // Increase difficulty significantly
rule.Expression = &expression.Config{
rule.Expression = &config.ExpressionOrList{
All: []string{"true"}, // Always applies
}
} else {
rule.Expression = &expression.Config{
rule.Expression = &config.ExpressionOrList{
All: []string{fmt.Sprintf("userAgent.contains(%q)", userAgent)},
}
}
@@ -286,7 +285,7 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
pathCondition := buildPathCondition(disallow)
conditions = append(conditions, pathCondition)
rule.Expression = &expression.Config{
rule.Expression = &config.ExpressionOrList{
All: conditions,
}

32
docker-bake.hcl Normal file
View File

@@ -0,0 +1,32 @@
variable "ALPINE_VERSION" { default = "3.22" }
variable "GITHUB_SHA" { default = "devel" }
variable "VERSION" { default = "devel-docker" }
group "default" {
targets = [
"osiris",
]
}
target "osiris" {
args = {
ALPINE_VERSION = "3.22"
VERSION = "${VERSION}"
}
context = "."
dockerfile = "./docker/osiris.Dockerfile"
platforms = [
"linux/amd64",
"linux/arm64",
"linux/arm/v7",
"linux/ppc64le",
"linux/riscv64",
]
pull = true
sbom = true
provenance = true
tags = [
"ghcr.io/techarohq/anubis/osiris:${VERSION}",
"ghcr.io/techarohq/anubis/osiris:main"
]
}

30
docker/osiris.Dockerfile Normal file
View File

@@ -0,0 +1,30 @@
ARG ALPINE_VERSION=edge
FROM --platform=${BUILDPLATFORM} alpine:${ALPINE_VERSION} AS build
RUN apk -U add go nodejs git build-base git npm bash zstd brotli gzip
WORKDIR /app
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/root/go go mod download
COPY package.json package-lock.json ./
RUN npm ci
ARG TARGETOS
ARG TARGETARCH
ARG VERSION=devel-docker
COPY . .
RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/root/go GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GOARM=7 go build -gcflags "all=-N -l" -o /app/bin/osiris -ldflags "-s -w -extldflags -static -X github.com/TecharoHQ/anubis.Version=${VERSION}" ./cmd/osiris
FROM alpine:${ALPINE_VERSION} AS run
WORKDIR /app
RUN apk -U add ca-certificates mailcap
COPY --from=build /app/bin/osiris /app/bin/osiris
CMD ["/app/bin/osiris"]
LABEL org.opencontainers.image.source="https://github.com/TecharoHQ/anubis"

View File

@@ -20,9 +20,9 @@ If you rely on Anubis to keep your website safe, please consider sponsoring the
I am waiting to hear back from NLNet on if Anubis was selected for funding or not. Let's hope it is!
## Deprecation warning: `DIFFICULTY`
## Deprecation warning: `DEFAULT_DIFFICULTY`
Anubis v1.20.0 is the last version to support the `DIFFICULTY` flag in the exact way it currently does. In future versions, this will be ineffectual and you should use the [custom threshold system](/docs/admin/configuration/thresholds) instead.
Anubis v1.20.0 is the last version to support the `DEFAULT_DIFFICULTY` flag in the exact way it currently does. In future versions, this will be ineffectual and you should use the [custom threshold system](/docs/admin/configuration/thresholds) instead.
If this becomes an imposition in practice, this will be reverted.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 62 KiB

View File

@@ -1,369 +0,0 @@
---
slug: release/v1.21.1
title: Anubis v1.21.1 is now available!
authors: [xe]
tags: [release]
image: anubis-i18n.webp
---
![](./anubis-i18n.webp)
Hey all!
Recently we released [Anubis v1.21.1: Minfilia Warde (Echo 1)](https://github.com/TecharoHQ/anubis/releases/tag/v1.21.1). This is a fairly meaty release and like [last time](../2025-06-27-release-1.20.0/index.mdx) this blogpost will tell you what you need to know before you update. Kick back, get some popcorn and let's dig into this!
{/* truncate */}
In this release, Anubis becomes internationalized, gains the ability to use system load as input to issuing challenges, finally fixes the "invalid response" after "success" bug, and more! Please read these notes before upgrading as the changes are big enough that administrators should take action to ensure that the upgrade goes smoothly.
This release is brought to you by [FreeCAD](https://www.freecad.org/), an open-source computer aided design tool that lets you design things for the real world.
## What's in this release?
The biggest change is that the ["invalid response" after "success" bug](https://github.com/TecharoHQ/anubis/issues/564) is now finally fixed for good by totally rewriting how [Anubis' challenge issuance flow works](#challenge-flow-v2).
This release gives Anubis the following features:
- [Internationalization support](#internationalization), allowing Anubis to render its messages in the human language you speak.
- Anubis now supports the [`missingHeader`](#missingHeader-function) function to assert the absence of headers in requests.
- Anubis now has the ability to [store data persistently on the server](#persistent-data-storage).
- Anubis can use [the system load average](#load-average-checks) as a factor to determine if it needs to filter traffic or not.
- Add `COOKIE_SECURE` option to set the cookie [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies)
- Sets cookie defaults to use [SameSite: None](https://web.dev/articles/samesite-cookies-explained)
- Allow [Common Crawl](https://commoncrawl.org/) by default so scrapers have less incentive to scrape
- Add `/healthz` metrics route for use in platform-based health checks.
- Start exposing JA4H fingerprints for later use in CEL expressions.
And this release also fixes the following bugs:
- [Challenge issuance has been totally rewritten](#challenge-flow-v2) to finally squash the infamous ["invalid response" after "success" bug](https://github.com/TecharoHQ/anubis/issues/564) for good.
- In order to reduce confusion, the "Success" interstitial that shows up when you pass a proof of work challenge has been removed.
- Don't block Anubis starting up if [Thoth](/docs/admin/thoth/) health checks fail.
- The "Try again" button on the error page has been fixed. Previously it meant "try the solution again" instead of "try the challenge again".
- In certain cases, a user could be stuck with a test cookie that is invalid, locking them out of the service for up to half an hour. This has been fixed with better validation of this case and clearing the cookie.
- "Proof of work" has been removed from the branding due to some users having extremely negative connotations with it.
We try to avoid introducing breaking changes as much as possible, but these are the changes that may be relevant for you as an administrator:
- The [challenge format](#challenge-format-change) has been changed in order to account for [the new challenge issuance flow](#challenge-flow-v2).
- The [systemd service `RuntimeDirectory` has been changed](#breaking-change-systemd-runtimedirectory-change).
### Sponsoring the project
If you rely on Anubis to keep your website safe, please consider sponsoring the project on [GitHub Sponsors](https://github.com/sponsors/Xe) or [Patreon](https://patreon.com/cadey). Funding helps pay hosting bills and offset the time spent on making this project the best it can be. Every little bit helps and when enough money is raised, [I can make Anubis my full-time job](https://github.com/TecharoHQ/anubis/discussions/278).
Once this pie chart is at 100%, I can start to reduce my hours at my day job as most of my needs will be met (pre-tax):
```mermaid
pie title Funding update
"GitHub Sponsors" : 29
"Patreon" : 14
"Remaining" : 56
```
I am waiting to hear back from NLNet on if Anubis was selected for funding or not. Let's hope it is!
## New features
### Internationalization
Anubis now supports localized responses. Locales can be added in [lib/localization/locales/](https://github.com/TecharoHQ/anubis/tree/main/lib/localization/locales). This release includes support for the following languages:
- [Brazilian Portugese](https://github.com/TecharoHQ/anubis/pull/726)
- [Chinese (Simplified)](https://github.com/TecharoHQ/anubis/pull/774)
- [Chinese (Traditional)](https://github.com/TecharoHQ/anubis/pull/759)
- [Czech](https://github.com/TecharoHQ/anubis/pull/849)
- English
- [Estonian](https://github.com/TecharoHQ/anubis/pull/783)
- [Filipino](https://github.com/TecharoHQ/anubis/pull/775)
- [Finnish](https://github.com/TecharoHQ/anubis/pull/863)
- [French](https://github.com/TecharoHQ/anubis/pull/716)
- [German](https://github.com/TecharoHQ/anubis/pull/741)
- [Japanese](https://github.com/TecharoHQ/anubis/pull/772)
- [Icelandic](https://github.com/TecharoHQ/anubis/pull/780)
- [Italian](https://github.com/TecharoHQ/anubis/pull/778)
- [Norwegian](https://github.com/TecharoHQ/anubis/pull/855)
- [Russian](https://github.com/TecharoHQ/anubis/pull/882)
- [Spanish](https://github.com/TecharoHQ/anubis/pull/716)
- [Turkish](https://github.com/TecharoHQ/anubis/pull/751)
If facts or local regulations demand, you can set Anubis default language with the `FORCED_LANGUAGE` environment variable or the `--forced-language` command line argument:
```sh
FORCED_LANGUAGE=de
```
## Big ticket bug fixes
These issues affect every user of Anubis. Administrators should upgrade Anubis as soon as possible to mitigate them.
### Fix event loop thrashing when solving a proof of work challenge
Anubis has a progress bar so that users can have something moving while it works. This gives users more confidence that something is happening and that the website is not being malicious with CPU usage. However, the way it was implemented way back in [#87](https://github.com/TecharoHQ/anubis/pull/87) had a subtle bug:
```js
if (
(nonce > oldNonce) | 1023 && // we've wrapped past 1024
(nonce >> 10) % threads === threadId // and it's our turn
) {
postMessage(nonce);
}
```
The logic here looks fine but is subtly wrong as was reported in [#877](https://github.com/TecharoHQ/anubis/issues/877) by the main Pale Moon developer.
For context, `nonce` is a counter that increments by the worker count every loop. This is intended to spread the load between CPU cores as such:
| Iteration | Worker ID | Nonce |
| :-------- | :-------- | :---- |
| 1 | 0 | 0 |
| 1 | 1 | 1 |
| 2 | 0 | 2 |
| 2 | 1 | 3 |
And so on. This makes the proof of work challenge as fast as it can possibly be so that Anubis quickly goes away and you can enjoy the service it is protecting.
The incorrect part of this is the boolean logic, specifically the part with the bitwise or `|`. I think the intent was to use a logical or (`||`), but this had the effect of making the `postMessage` handler fire on every iteration. The intent of this snippet (as the comment clearly indicates) is to make sure that the main event loop is only updated with the worker status every 1024 iterations per worker. This had the opposite effect, causing a lot of messages to be sent from workers to the parent JavaScript context.
This is bad for the event loop.
Instead, I have ripped out that statement and replaced it with a much simpler increment only counter that fires every 1024 iterations. Additionally, only the first thread communicates back to the parent process. This does mean that in theory the other workers could be ahead of the first thread (posting a message out of a worker has a nonzero cost), but in practice I don't think this will be as much of an issue as the current behaviour is.
The root cause of the stack exhaustion is likely the pressure caused by all of the postMessage futures piling up. Maybe the larger stack size in 64 bit environments is causing this to be fine there, maybe it's some combination of newer hardware in 64 bit systems making this not be as much of a problem due to it being able to handle events fast enough to keep up with the pressure.
Either way, thanks much to [@wolfbeast](https://github.com/wolfbeast) and the Pale Moon community for finding this. This will make Anubis faster for everyone!
### Fix potential memory leak when discovering a solution
In some cases, the parallel solution finder in Anubis could cause all of the worker promises to leak due to the fact the promises were being improperly terminated. A recursion bomb happens in the following scenario:
1. A worker sends a message indicating it found a solution to the proof of work challenge.
2. The `onmessage` handler for that worker calls `terminate()`
3. Inside `terminate()`, the parent process loops through all other workers and calls `w.terminate()` on them.
4. It's possible that terminating a worker could lead to the `onerror` event handler.
5. This would create a recursive loop of `onmessage` -> `terminate` -> `onerror` -> `terminate` -> `onerror` and so on.
This infinite recursion quickly consumes all available stack space, but this has never been noticed in development because all of my computers have at least 64Gi of ram provisioned to them under the axiom paying for more ram is cheaper than paying in my time spent having to work around not having enough ram. Additionally, ia32 has a smaller base stack size, which means that they will run into this issue much sooner than users on other CPU architectures will.
The fix adds a boolean `settled` flag to prevent termination from running more than once.
## Expressions features
Anubis v1.21.1 adds additional [expressions](/docs/admin/configuration/expressions) features so that you can make your request matching even more granular.
### `missingHeader` function
Anubis [expressions](/docs/admin/configuration/expressions) have [a few functions exposed](/docs/admin/configuration/expressions/#functions-exposed-to-anubis-expressions). Anubis v1.21.1 adds the `missingHeader` function, allowing you to assert the _absence_ of a header in requests.
Let's say you're getting a lot of requests from clients that are pretending to be Google Chrome. Google Chrome sends a few signals to web servers, the main one of them is the [`Sec-Ch-Ua`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Sec-CH-UA). Sec-CH-UA is part of Google's [User Agent Client Hints](https://wicg.github.io/ua-client-hints/#sec-ch-ua) proposal, but it being present is a sign that the client is more likely Google Chrome than not. With the `missingHeader` function, you can write a rule to [add weight](/docs/admin/policies/#request-weight) to requests without `Sec-Ch-Ua` that claim to be Google Chrome.
```yaml
# Adds weight clients that claim to be Google Chrome without setting Sec-Ch-Ua
- name: old-chrome
action: WEIGH
weight:
adjust: 10
expression:
all:
- userAgent.matches("Chrome/[1-9][0-9]?\\.0\\.0\\.0")
- missingHeader(headers, "Sec-Ch-Ua")
```
When combined with [weight thresholds](/docs/admin/configuration/thresholds), this allows you to make requests that don't match the signature of Google Chrome more suspicious, which will make them have a more difficult challenge.
### Load average checks
Anubis can dynamically take action [based on the system load average](/docs/admin/configuration/expressions/#using-the-system-load-average), allowing you to write rules like this:
```yaml
## System load based checks.
# If the system is under high load for the last minute, add weight.
- name: high-load-average
action: WEIGH
expression: load_1m >= 10.0 # make sure to end the load comparison in a .0
weight:
adjust: 20
# If it is not for the last 15 minutes, remove weight.
- name: low-load-average
action: WEIGH
expression: load_15m <= 4.0 # make sure to end the load comparison in a .0
weight:
adjust: -10
```
Something to keep in mind about system load average is that it is not aware of the number of cores the system has. If you have a 16 core system that has 16 processes running but none of them is hogging the CPU, then you will get a load average below 16. If you are in doubt, make your "high load" metric at least two times the number of CPU cores and your "low load" metric at least half of the number of CPU cores. For example:
| Kind | Core count | Load threshold |
| --------: | :--------- | :------------- |
| high load | 4 | `8.0` |
| low load | 4 | `2.0` |
| high load | 16 | `32.0` |
| low load | 16 | `8` |
Also keep in mind that this does not account for other kinds of latency like I/O latency or downstream API response latency. A system can have its web applications unresponsive due to high latency from a MySQL server but still have that web application server report a load near or at zero.
:::note
This does not work if you are using Kubernetes.
:::
When combined with [weight thresholds](/docs/admin/configuration/thresholds), this allows you to make incoming sessions "back off" while the server is under high load.
## Challenge flow v2
The main goal of Anubis is to weigh the risks of incoming requests in order to protect upstream resources against abusive clients like badly written scrapers. In order to separate "good" clients (like users wanting to learn from a website's content) from "bad" clients, Anubis issues [challenges](/docs/category/challenges).
Previously the Anubis challenge flow looked like this:
```mermaid
---
title: Old Anubis challenge flow
---
flowchart LR
user(User Browser)
subgraph Anubis
mIC{Challenge?}
ic(Issue Challenge)
rp(Proxy to service)
mIC -->|User needs a challenge| ic
mIC -->|User does not need a challenge| rp
end
target(Target Service)
rp --> target
user --> mIC
ic -->|Pass a challenge| user
target -->|Site data| users
```
In order to issue a challenge, Anubis generated a challenge string based on request metadata that we assumed wouldn't drastically change between requests, including but not limited to:
- The client's User-Agent string.
- The client [`Accept-Language` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Language) value.
- The client's IP address.
Anubis also didn't store any information about challenges so that it can remain lightweight and handle the onslaught of requests from scrapers. The assumption was that the challenge string function was idempotent per client across time. What actually ended up happening was something like this:
```mermaid
---
title: Anubis challenge string idempotency
---
sequenceDiagram
User->>+Anubis: GET /wiki/some-page
Anubis->>+Make Challenge: Generate a challenge string
Make Challenge->>-Anubis: Challenge string: taco salad
Anubis->>-User: HTTP 401 solve a challenge
User->>+Anubis: GET internal-api/pass-challenge
Anubis->>+Make Challenge: Generate a challenge string
Make Challenge->>-Anubis: Challenge string: burrito bar
Anubis->>+User: Error: invalid response
```
Various attempts were made to fix this. All of these ended up failing. Many difficulties were discovered including but not limited to:
- Removing `Accept-Language` from consideration because [Chrome randomizes the contents of `Accept-Language` to reduce fingerprinting](https://github.com/explainers-by-googlers/reduce-accept-language), a behaviour which [causes a lot of confusion](https://www.reddit.com/r/chrome/comments/nhpnez/google_chrome_is_randomly_switching_languages_on/) for users with multiple system languages selected.
- [IPv6 privacy extensions](https://www.internetsociety.org/resources/deploy360/2014/privacy-extensions-for-ipv6-slaac/) mean that each request could be coming from a different IP address (at least one legitimate user in the wild has been observed to have a different IP address per TCP session across an entire `/48`).
- Some [US mobile phone carriers make it too easy for your IP address to drastically change](https://news.ycombinator.com/item?id=32038215) without user input.
- [Happy eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs) means that some requests can come in over IPv4 and some requests can come in over IPv6.
- To make things worse, you can't even assert that users are from the same [BGP autonomous system](<https://en.wikipedia.org/wiki/Autonomous_system_(Internet)>) because some users could have ISPs that are IPv4 only, forcing them to use a different IP address space to get IPv6 internet access. This sounds like it's rare enough, but I personally have to do this even though I pay for 8 gigabit fiber from my ISP and only get IPv4 service from them.
Amusingly enough, the only part of this that has survived is the assertion that a user hasn't changed their `User-Agent` string. Maybe [that one guy that sets his Chrome version to `150`](https://github.com/TecharoHQ/anubis/issues/239) would have issues, but so far I've not seen any evidence that a client randomly changing their user agent between challenge issuance and solving can possibly be legitimate.
As a result, the entire subsystem that generated challenges before had to be ripped out and rewritten from scratch.
It was replaced with a new flow that stores data on the server side, compares that data against what the client responds with, and then checks pass/fail that way:
```mermaid
---
title: New challenge flow
---
sequenceDiagram
User->>+Anubis: GET /wiki/some-page
Anubis->>+Make Challenge: Generate a challenge string
Make Challenge->>+Store: Store info for challenge 1234
Make Challenge->>-Anubis: Challenge string: taco salad, ID 1234
Anubis->>-User: HTTP 401 solve a challenge
User->>+Anubis: GET internal-api/pass-challenge, challenge 1234
Anubis->>+Validate Challenge: verify challenge 1234
Validate Challenge->>+Store: Get info for challenge 1234
Store->>-Validate Challenge: Here you go!
Validate Challenge->>-Anubis: Valid ✅
Anubis->>+User: Here's a cookie to get past Anubis
```
As a result, the [challenge format](#challenge-format-change) had to change. Old cookies will still be validated, but the next minor version (v1.22.0) will include validation to ensure that all challenges are accounted for on the server side. This data is stored in the active [storage backend](/docs/admin/policies/#storage-backends) for up to 30 minutes. This also fixes [#746](https://github.com/TecharoHQ/anubis/issues/746) and other similar instances of this issue.
### Challenge format change
Previously Anubis did no accounting for challenges that it issued. This means that if Anubis restarted during a client, the client would be able to proceed once Anubis came back online.
During the upgrade to v1.21.0 and when v1.21.0 (or later) restarts with the [in-memory storage backend](/docs/admin/policies/#memory), you may see a higher rate of failed challenges than normal. If this persists beyond a few minutes, [open an issue](https://github.com/TecharoHQ/anubis/issues/new).
If you are using the in-memory storage backend, please consider using [a different storage backend](/docs/admin/policies/#storage-backends).
### Storage
Anubis offers a few different storage backends depending on your needs:
| Backend | Description |
| :--------------------------------------- | :------------------------------------------------------------------------------------------------------------- |
| [`memory`](/docs/admin/policies/#memory) | An in-memory hashmap that is cleared when Anubis is restarted. |
| [`bbolt`](/docs/admin/policies/#bbolt) | A memory-mapped key/value store that can persist between Anubis restarts. |
| [`valkey`](/docs/admin/policies/#valkey) | A networked key/value store that can persist between Anubis restarts and coordinate across multiple instances. |
Please review the documentation for each storage method to figure out the one best for your needs. If you aren't sure, consult this diagram:
```mermaid
---
title: What storage backend do I need?
---
flowchart TD
OneInstance{Do you only have
one instance of
Anubis?}
Persistence{Do you have
persistent disk
access in your
environment?}
bbolt[(bbolt)]
memory[(memory)]
valkey[(valkey)]
OneInstance -->|Yes| Persistence
OneInstance -->|No| valkey
Persistence -->|Yes| bbolt
Persistence -->|No| memory
```
## Breaking change: systemd `RuntimeDirectory` change
The following potentially breaking change applies to native installs with systemd only:
Each instance of systemd service template now has a unique `RuntimeDirectory`, as opposed to each instance of the service sharing a `RuntimeDirectory`. This change was made to avoid [the `RuntimeDirectory` getting nuked](https://github.com/TecharoHQ/anubis/issues/748) any time one of the Anubis instances restarts.
If you configured Anubis' unix sockets to listen on `/run/anubis/foo.sock` for instance `anubis@foo`, you will need to configure Anubis to listen on `/run/anubis/foo/foo.sock` and additionally configure your HTTP load balancer as appropriate.
If you need the legacy behaviour, install this [systemd unit dropin](https://www.flatcar.org/docs/latest/setup/systemd/drop-in-units/):
```systemd
# /etc/systemd/system/anubis@.service.d/50-runtimedir.conf
[Service]
RuntimeDirectory=anubis
```
Just keep in mind that this will cause problems when Anubis restarts.
## What's up next?
The biggest things we want to do in the next release (in no particular order):
- A rewrite of bot checking rule configuration syntax to make it less ambiguous.
- [JA4](https://blog.foxio.io/ja4+-network-fingerprinting) (and other forms of) fingerprinting and coordination with [Thoth](/docs/admin/thoth/) to allow clients with high aggregate pass rates through without seeing Anubis at all.
- Advanced heuristics for [users of the unbranded variant of Anubis](/docs/admin/botstopper/).
- Optimize the release flow so that releases can be triggered and executed by continuous integration tools. The ultimate goal is to make it possible to release Anubis in 15 minutes after pressing a single "mint release" button.
- Add "hot reloading" support to Anubis, allowing administrators to update the rules without restarting the service.
- Fix [multiple slash support](https://github.com/TecharoHQ/anubis/issues/754) for web applications that require optional path variables.
- Add weight to "brand new" clients.
- Implement a "maze" feature that tries to get crawlers ensnared in a maze of random links so that clients that are more than 20 links in can be reported to the home base.
- Open [Thoth-based advanced checks](/docs/admin/thoth/) to more users with an easier onboarding flow.
- More smoke tests including for browsers like [Pale Moon](https://www.palemoon.org/).

View File

@@ -13,67 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
<!-- This changes the project to: -->
- The [Thoth client](https://anubis.techaro.lol/docs/admin/thoth) is now public in the repo instead of being an internal package.
## v1.21.3: Minfilia Warde - Echo 3
### Fixes
#### Fixes a problem with nonstandard URLs and redirects
Fixes [GHSA-jhjj-2g64-px7c](https://github.com/TecharoHQ/anubis/security/advisories/GHSA-jhjj-2g64-px7c).
This could allow an attacker to craft an Anubis pass-challenge URL that forces a redirect to nonstandard URLs, such as the `javascript:` scheme which executes arbitrary JavaScript code in a browser context when the user clicks the "Try again" button.
This has been fixed by disallowing any URLs without the scheme `http` or `https`.
Additionally, the "Try again" button has been fixed to completely ignore the user-supplied redirect location. It now redirects to the home page (`/`).
## v1.21.2: Minfilia Warde - Echo 2
This contained an incomplete fix for [GHSA-jhjj-2g64-px7c](https://github.com/TecharoHQ/anubis/security/advisories/GHSA-jhjj-2g64-px7c). Do not use this version.
## v1.21.1: Minfilia Warde - Echo 1
- Expired records are now properly removed from bbolt databases ([#848](https://github.com/TecharoHQ/anubis/pull/848)).
- Fix hanging on service restart ([#853](https://github.com/TecharoHQ/anubis/issues/853))
### Added
Anubis now supports the [`missingHeader`](./admin/configuration/expressions.mdx#missingHeader) to assert the absence of headers in requests.
#### New locales
Anubis now supports these new languages:
- [Czech](https://github.com/TecharoHQ/anubis/pull/849)
- [Finnish](https://github.com/TecharoHQ/anubis/pull/863)
- [Norwegian Bokmål](https://github.com/TecharoHQ/anubis/pull/855)
- [Norwegian Nynorsk](https://github.com/TecharoHQ/anubis/pull/855)
- [Russian](https://github.com/TecharoHQ/anubis/pull/882)
### Fixes
#### Fix ["error: can't get challenge"](https://github.com/TecharoHQ/anubis/issues/869) when details about a challenge can't be found in the server side state
v1.21.0 changed the core challenge flow to maintain information about challenges on the server side instead of only doing them via stateless idempotent generation functions and relying on details to not change. There was a subtle bug introduced in this change: if a client has an unknown challenge ID set in its test cookie, Anubis will clear that cookie and then throw an HTTP 500 error.
This has been fixed by making Anubis throw a new challenge page instead.
#### Fix event loop thrashing when solving a proof of work challenge
Previously the "fast" proof of work solver had a fragment of JavaScript that attempted to only post an update about proof of work progress to the main browser window every 1024 iterations. This fragment of JavaScript was subtly incorrect in a way that passed review but actually made the workers send an update back to the main thread every iteration. This caused a pileup of unhandled async calls (similar to a socket accept() backlog pileup in Unix) that caused stack space exhaustion.
This has been fixed in the following ways:
1. The complicated boolean logic has been totally removed in favour of a worker-local iteration counter.
2. The progress bar is updated by worker `0` instead of all workers.
Hopefully this should limit the event loop thrashing and let ia32 browsers (as well as any environment with a smaller stack size than amd64 and aarch64 seem to have) function normally when processing Anubis proof of work challenges.
#### Fix potential memory leak when discovering a solution
In some cases, the parallel solution finder in Anubis could cause all of the worker promises to leak due to the fact the promises were being improperly terminated. This was fixed by having Anubis debounce worker termination instead of allowing it to potentially recurse infinitely.
Anubis now supports the [`missingHeader`](./admin/configuration/expressions.mdx#missingHeader) to assert the absence of headers in requests.
## v1.21.0: Minfilia Warde

View File

@@ -79,10 +79,6 @@ server {
root "/srv/http/anubistest.techaro.lol";
index index.html;
# Get the visiting IP from the TLS termination server
set_real_ip_from unix:;
real_ip_header X-Real-IP;
# Your normal configuration can go here
# location .php { fastcgi...} etc.
}

20
docs/package-lock.json generated
View File

@@ -5908,9 +5908,9 @@
}
},
"node_modules/brace-expansion": {
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
@@ -6496,16 +6496,16 @@
}
},
"node_modules/compression": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz",
"integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==",
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/compression/-/compression-1.8.0.tgz",
"integrity": "sha512-k6WLKfunuqCYD3t6AsuPGvQWaKwuLLh2/xHNcX4qE+vIfDNXpSqnrhwA7O53R7WVQUnt8dVAIW+YHr7xTgOgGA==",
"license": "MIT",
"dependencies": {
"bytes": "3.1.2",
"compressible": "~2.0.18",
"debug": "2.6.9",
"negotiator": "~0.6.4",
"on-headers": "~1.1.0",
"on-headers": "~1.0.2",
"safe-buffer": "5.2.1",
"vary": "~1.1.2"
},
@@ -13562,9 +13562,9 @@
}
},
"node_modules/on-headers": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz",
"integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz",
"integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==",
"license": "MIT",
"engines": {
"node": ">= 0.8"

View File

@@ -1,7 +0,0 @@
package anubis
import "errors"
var (
ErrMisconfiguration = errors.New("[unexpected] policy: administrator misconfiguration")
)

29
go.mod
View File

@@ -4,12 +4,12 @@ go 1.24.2
require (
github.com/TecharoHQ/thoth-proto v0.4.0
github.com/a-h/templ v0.3.906
github.com/a-h/templ v0.3.920
github.com/cespare/xxhash/v2 v2.3.0
github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456
github.com/gaissmai/bart v0.20.5
github.com/golang-jwt/jwt/v5 v5.2.2
github.com/google/cel-go v0.25.0
github.com/gaissmai/bart v0.22.0
github.com/golang-jwt/jwt/v5 v5.2.3
github.com/google/cel-go v0.26.0
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2
@@ -21,20 +21,20 @@ require (
github.com/redis/go-redis/v9 v9.11.0
github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a
github.com/shirou/gopsutil/v4 v4.25.6
github.com/testcontainers/testcontainers-go v0.37.0
github.com/testcontainers/testcontainers-go v0.38.0
go.etcd.io/bbolt v1.4.2
golang.org/x/net v0.42.0
golang.org/x/text v0.27.0
google.golang.org/grpc v1.73.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apimachinery v0.33.2
k8s.io/apimachinery v0.33.3
sigs.k8s.io/yaml v1.5.0
)
require (
al.essio.dev/pkg/shellescape v1.6.0 // indirect
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 // indirect
cel.dev/expr v0.23.1 // indirect
cel.dev/expr v0.24.0 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/AlekSi/pointer v1.2.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
@@ -47,8 +47,10 @@ require (
github.com/Songmu/gitconfig v0.2.0 // indirect
github.com/TecharoHQ/yeet v0.6.0 // indirect
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e // indirect
github.com/agext/levenshtein v1.2.1 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect
github.com/cavaliergopher/cpio v1.0.1 // indirect
@@ -56,6 +58,8 @@ require (
github.com/cli/browser v1.3.0 // indirect
github.com/cli/go-gh v0.1.0 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
@@ -66,7 +70,7 @@ require (
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/docker v28.0.1+incompatible // indirect
github.com/docker/docker v28.2.2+incompatible // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c // indirect
@@ -91,6 +95,7 @@ require (
github.com/goccy/go-yaml v1.12.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-github/v70 v70.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
@@ -99,6 +104,7 @@ require (
github.com/goreleaser/fileglob v1.3.0 // indirect
github.com/goreleaser/nfpm/v2 v2.42.1 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/hashicorp/hcl/v2 v2.24.0 // indirect
github.com/huandu/xstrings v1.5.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
@@ -109,11 +115,13 @@ require (
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.1.0 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/sys/user v0.1.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
@@ -147,6 +155,7 @@ require (
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zclconf/go-cty v1.16.3 // indirect
gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect

66
go.sum
View File

@@ -2,12 +2,12 @@ al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeX
al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 h1:YhMSc48s25kr7kv31Z8vf7sPUIq5YJva9z1mn/hAt0M=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U=
cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg=
cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w=
github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -40,14 +40,18 @@ github.com/TecharoHQ/yeet v0.6.0 h1:RCBAjr7wIlllsgy0tpvWpLX7jsZgu2tiuBY3RrprcR0=
github.com/TecharoHQ/yeet v0.6.0/go.mod h1:bj2V4Fg8qKQXoiuPZa3HuawrE8g+LsOQv/9q2WyGSsA=
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo=
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e/go.mod h1:3mnrkvGpurZ4ZrTDbYU84xhwXW2TjTKShSwjRi2ihfQ=
github.com/a-h/templ v0.3.906 h1:ZUThc8Q9n04UATaCwaG60pB1AqbulLmYEAMnWV63svg=
github.com/a-h/templ v0.3.906/go.mod h1:FFAu4dI//ESmEN7PQkJ7E7QfnSEMdcnu7QrAY8Dn334=
github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/a-h/templ v0.3.920 h1:IQjjTu4KGrYreHo/ewzSeS8uefecisPayIIc9VflLSE=
github.com/a-h/templ v0.3.920/go.mod h1:FFAu4dI//ESmEN7PQkJ7E7QfnSEMdcnu7QrAY8Dn334=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -77,6 +81,10 @@ github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5
github.com/cli/shurcooL-graphql v0.0.1/go.mod h1:U7gCSuMZP/Qy7kbqkk5PrqXEeDgtfG5K+W+u8weorps=
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
@@ -101,8 +109,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.0.1+incompatible h1:FCHjSRdXhNRFjlHMTv4jUNlIBbTeRjrWfeFuJp7jpo0=
github.com/docker/docker v28.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw=
github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -133,8 +141,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gaissmai/bart v0.20.5 h1:ehoWZWQ7j//qt0K0Zs4i9hpoPpbgqsMQiR8W2QPJh+c=
github.com/gaissmai/bart v0.20.5/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
github.com/gaissmai/bart v0.22.0 h1:+yR2mCpZx8H8GlqA+Icqi7/Iwx2/OUbO4bVbsORK0ns=
github.com/gaissmai/bart v0.22.0/go.mod h1:RpLtt3lWq1BoRz3AAyDAJ7jhLWBkYhVCfi+ximB2t68=
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -175,14 +183,14 @@ github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM=
github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0=
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY=
github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI=
github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI=
github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM=
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -221,6 +229,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1ns
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE=
github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM=
github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo=
github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -274,16 +284,22 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ=
github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo=
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
@@ -373,8 +389,8 @@ github.com/suzuki-shunsuke/pinact v1.6.0 h1:2QvSzREOquwLwKXhF9Hj0AInE/Rl63SZz9dK
github.com/suzuki-shunsuke/pinact v1.6.0/go.mod h1:FDUMck0mmL0mcnNZ23Vjh/aOR5cIdZhF1IIpGksT4dQ=
github.com/suzuki-shunsuke/urfave-cli-help-all v0.0.4 h1:YGHgrVjGTYHY98II6zijXUHP+OyvrzSCvd8m9iUcaK8=
github.com/suzuki-shunsuke/urfave-cli-help-all v0.0.4/go.mod h1:sSi6xaUaHfaqu32ECLeyE7NTMv+ZM5dW0JikhllaalY=
github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg=
github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM=
github.com/testcontainers/testcontainers-go v0.38.0 h1:d7uEapLcv2P8AvH8ahLqDMMxda2W9gQN1nRbHS28HBw=
github.com/testcontainers/testcontainers-go v0.38.0/go.mod h1:C52c9MoHpWO+C4aqmgSU+hxlR5jlEayWtgYrb8Pzz1w=
github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
@@ -395,6 +411,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk=
github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
gitlab.com/digitalxero/go-conventional-commit v1.0.7 h1:8/dO6WWG+98PMhlZowt/YjuiKhqhGlOCwlIV8SqqGh8=
gitlab.com/digitalxero/go-conventional-commit v1.0.7/go.mod h1:05Xc2BFsSyC5tKhK0y+P3bs0AwUtNuTp+mTpbCU/DZ0=
go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I=
@@ -549,12 +567,12 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=
k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY=
k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
mvdan.cc/sh/v3 v3.11.0 h1:q5h+XMDRfUGUedCqFFsjoFjrhwf2Mvtt1rkMvVz0blw=
mvdan.cc/sh/v3 v3.11.0/go.mod h1:LRM+1NjoYCzuq/WZ6y44x14YNAI0NK7FLPeQSaFagGg=
pault.ag/go/debian v0.18.0 h1:nr0iiyOU5QlG1VPnhZLNhnCcHx58kukvBJp+dvaM6CQ=

View File

@@ -0,0 +1,97 @@
package fingerprint
import (
"crypto/md5"
"crypto/tls"
"encoding/hex"
"slices"
"strconv"
)
// TLSFingerprintJA3N represents a JA3N fingerprint
type TLSFingerprintJA3N [md5.Size]byte
func (f TLSFingerprintJA3N) String() string {
return hex.EncodeToString(f[:])
}
func buildJA3N(hello *tls.ClientHelloInfo, sortExtensions bool) TLSFingerprintJA3N {
buf := make([]byte, 0, 256)
{
var sslVersion uint16
var hasGrease bool
for _, v := range hello.SupportedVersions {
if v&greaseMask != greaseValue {
if v > sslVersion {
sslVersion = v
}
} else {
hasGrease = true
}
}
// maximum TLS 1.2 as specified on JA3, as TLS 1.3 is put in SupportedVersions
if slices.Contains(hello.Extensions, extensionSupportedVersions) && hasGrease && sslVersion > tls.VersionTLS12 {
sslVersion = tls.VersionTLS12
}
buf = strconv.AppendUint(buf, uint64(sslVersion), 10)
buf = append(buf, ',')
}
n := 0
for _, cipher := range hello.CipherSuites {
//if !slices.Contains(greaseValues[:], cipher) {
if cipher&greaseMask != greaseValue {
buf = strconv.AppendUint(buf, uint64(cipher), 10)
buf = append(buf, '-')
n = 1
}
}
buf = buf[:len(buf)-n]
buf = append(buf, ',')
n = 0
extensions := hello.Extensions
if sortExtensions {
extensions = slices.Clone(extensions)
slices.Sort(extensions)
}
for _, extension := range extensions {
if extension&greaseMask != greaseValue {
buf = strconv.AppendUint(buf, uint64(extension), 10)
buf = append(buf, '-')
n = 1
}
}
buf = buf[:len(buf)-n]
buf = append(buf, ',')
n = 0
for _, curve := range hello.SupportedCurves {
if curve&greaseMask != greaseValue {
buf = strconv.AppendUint(buf, uint64(curve), 10)
buf = append(buf, '-')
n = 1
}
}
buf = buf[:len(buf)-n]
buf = append(buf, ',')
n = 0
for _, point := range hello.SupportedPoints {
buf = strconv.AppendUint(buf, uint64(point), 10)
buf = append(buf, '-')
n = 1
}
buf = buf[:len(buf)-n]
sum := md5.Sum(buf)
return TLSFingerprintJA3N(sum[:])
}

176
internal/fingerprint/ja4.go Normal file
View File

@@ -0,0 +1,176 @@
package fingerprint
import (
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"fmt"
"slices"
"strconv"
"strings"
)
// TLSFingerprintJA4 represents a JA4 fingerprint
type TLSFingerprintJA4 struct {
A [10]byte
B [6]byte
C [6]byte
}
func (f *TLSFingerprintJA4) String() string {
if f == nil {
return ""
}
return strings.Join([]string{
string(f.A[:]),
hex.EncodeToString(f.B[:]),
hex.EncodeToString(f.C[:]),
}, "_")
}
func buildJA4(hello *tls.ClientHelloInfo) (ja4 TLSFingerprintJA4) {
buf := make([]byte, 0, 36)
hasQuic := false
for _, ext := range hello.Extensions {
if ext == extensionQUICTransportParameters {
hasQuic = true
}
}
switch hasQuic {
case true:
buf = append(buf, 'q')
case false:
buf = append(buf, 't')
}
{
var sslVersion uint16
for _, v := range hello.SupportedVersions {
if v&greaseMask != greaseValue {
if v > sslVersion {
sslVersion = v
}
}
}
switch sslVersion {
case tls.VersionTLS10:
buf = append(buf, '1', '0')
case tls.VersionTLS11:
buf = append(buf, '1', '1')
case tls.VersionTLS12:
buf = append(buf, '1', '2')
case tls.VersionTLS13:
buf = append(buf, '1', '3')
default:
sslVersion -= 0x0201
buf = strconv.AppendUint(buf, uint64(sslVersion>>8), 10)
buf = strconv.AppendUint(buf, uint64(sslVersion&0xff), 10)
}
}
if slices.Contains(hello.Extensions, extensionServerName) && hello.ServerName != "" {
buf = append(buf, 'd')
} else {
buf = append(buf, 'i')
}
ciphers := make([]uint16, 0, len(hello.CipherSuites))
for _, cipher := range hello.CipherSuites {
if cipher&greaseMask != greaseValue {
ciphers = append(ciphers, cipher)
}
}
extensionCount := 0
extensions := make([]uint16, 0, len(hello.Extensions))
for _, extension := range hello.Extensions {
if extension&greaseMask != greaseValue {
extensionCount++
if extension != extensionALPN && extension != extensionServerName {
extensions = append(extensions, extension)
}
}
}
schemes := make([]tls.SignatureScheme, 0, len(hello.SignatureSchemes))
for _, scheme := range hello.SignatureSchemes {
if scheme&greaseMask != greaseValue {
schemes = append(schemes, scheme)
}
}
//TODO: maybe little endian
slices.Sort(ciphers)
slices.Sort(extensions)
//slices.Sort(schemes)
if len(ciphers) < 10 {
buf = append(buf, '0')
buf = strconv.AppendUint(buf, uint64(len(ciphers)), 10)
} else if len(ciphers) > 99 {
buf = append(buf, '9', '9')
} else {
buf = strconv.AppendUint(buf, uint64(len(ciphers)), 10)
}
if extensionCount < 10 {
buf = append(buf, '0')
buf = strconv.AppendUint(buf, uint64(extensionCount), 10)
} else if extensionCount > 99 {
buf = append(buf, '9', '9')
} else {
buf = strconv.AppendUint(buf, uint64(extensionCount), 10)
}
if len(hello.SupportedProtos) > 0 && len(hello.SupportedProtos[0]) > 1 {
buf = append(buf, hello.SupportedProtos[0][0], hello.SupportedProtos[0][len(hello.SupportedProtos[0])-1])
} else {
buf = append(buf, '0', '0')
}
copy(ja4.A[:], buf)
ja4.B = ja4SHA256(uint16SliceToHex(ciphers))
extBuf := uint16SliceToHex(extensions)
if len(schemes) > 0 {
extBuf = append(extBuf, '_')
extBuf = append(extBuf, uint16SliceToHex(schemes)...)
}
ja4.C = ja4SHA256(extBuf)
return ja4
}
func uint16SliceToHex[T ~uint16](in []T) (out []byte) {
if len(in) == 0 {
return out
}
out = slices.Grow(out, hex.EncodedLen(len(in)*2)+len(in))
for _, n := range in {
out = append(out, fmt.Sprintf("%04x", uint16(n))...)
out = append(out, ',')
}
out = out[:len(out)-1]
return out
}
func ja4SHA256(buf []byte) [6]byte {
if len(buf) == 0 {
return [6]byte{0, 0, 0, 0, 0, 0}
}
sum := sha256.Sum256(buf)
return [6]byte(sum[:6])
}

View File

@@ -0,0 +1,46 @@
package fingerprint
import (
"fmt"
"net/http"
"strings"
)
// JA4T represents a TCP fingerprint
type JA4T struct {
Window uint32
Options []uint8
MSS uint16
WindowScale uint8
}
func (j JA4T) String() string {
var sb strings.Builder
// Start with the window size
fmt.Fprintf(&sb, "%d", j.Window)
// Append each option
for i, opt := range j.Options {
if i == 0 {
fmt.Fprint(&sb, "_")
} else {
fmt.Fprint(&sb, "-")
}
fmt.Fprintf(&sb, "%d", opt)
}
// Append MSS and WindowScale
fmt.Fprintf(&sb, "_%d_%d", j.MSS, j.WindowScale)
return sb.String()
}
// GetTCPFingerprint extracts TCP fingerprint from HTTP request context
func GetTCPFingerprint(r *http.Request) *JA4T {
ptr := r.Context().Value(tcpFingerprintKey{})
if fpPtr, ok := ptr.(*JA4T); ok && ptr != nil && fpPtr != nil {
return fpPtr
}
return nil
}

View File

@@ -0,0 +1,106 @@
//go:build freebsd
package fingerprint
import (
"fmt"
"net"
"syscall"
"unsafe"
)
type tcpInfo struct {
State uint8
Options uint8
SndScale uint8
RcvScale uint8
__pad [4]byte
Rto uint32
Ato uint32
SndMss uint32
RcvMss uint32
Unacked uint32
Sacked uint32
Lost uint32
Retrans uint32
Fackets uint32
Last_data_sent uint32
Last_ack_sent uint32
Last_data_recv uint32
Last_ack_recv uint32
Pmtu uint32
Rcv_ssthresh uint32
RTT uint32
RTTvar uint32
Snd_ssthresh uint32
Snd_cwnd uint32
Advmss uint32
Reordering uint32
Rcv_rtt uint32
Rcv_space uint32
Total_retrans uint32
Snd_wnd uint32
// Truncated for brevity — add more fields if needed
}
// AssignTCPFingerprint extracts TCP fingerprint information from a connection
func AssignTCPFingerprint(conn net.Conn) (*JA4T, error) {
tcpConn, ok := conn.(*net.TCPConn)
if !ok {
return nil, fmt.Errorf("not a TCPConn")
}
rawConn, err := tcpConn.SyscallConn()
if err != nil {
return nil, fmt.Errorf("SyscallConn failed: %w", err)
}
var info tcpInfo
var sysErr error
err = rawConn.Control(func(fd uintptr) {
size := uint32(unsafe.Sizeof(info))
_, _, errno := syscall.Syscall6(
syscall.SYS_GETSOCKOPT,
fd,
uintptr(syscall.IPPROTO_TCP),
uintptr(syscall.TCP_INFO),
uintptr(unsafe.Pointer(&info)),
uintptr(unsafe.Pointer(&size)),
0,
)
if errno != 0 {
sysErr = errno
}
})
if err != nil {
return nil, fmt.Errorf("SyscallConn.Control: %w", err)
}
if sysErr != nil {
return nil, fmt.Errorf("getsockopt TCP_INFO: %w", sysErr)
}
fp := &JA4T{
Window: info.Snd_wnd,
MSS: uint16(info.SndMss),
WindowScale: info.SndScale,
}
const (
TCPI_OPT_TIMESTAMPS = 1 << 0
TCPI_OPT_SACK = 1 << 1
TCPI_OPT_WSCALE = 1 << 2
)
if info.Options&TCPI_OPT_SACK != 0 {
fp.Options = append(fp.Options, 4, 1)
}
if info.Options&TCPI_OPT_TIMESTAMPS != 0 {
fp.Options = append(fp.Options, 8, 1)
}
if info.Options&TCPI_OPT_WSCALE != 0 {
fp.Options = append(fp.Options, 3)
}
return fp, nil
}

View File

@@ -0,0 +1,132 @@
//go:build linux
package fingerprint
import (
"fmt"
"net"
"syscall"
"unsafe"
)
type tcpInfo struct {
State uint8
Ca_state uint8
Retransmits uint8
Probes uint8
Backoff uint8
Options uint8
Wnd_scale uint8
Delivery_rate_app_limited uint8
Rto uint32
Ato uint32
SndMss uint32
RcvMss uint32
Unacked uint32
Sacked uint32
Lost uint32
Retrans uint32
Fackets uint32
Last_data_sent uint32
Last_ack_sent uint32
Last_data_recv uint32
Last_ack_recv uint32
PMTU uint32
Rcv_ssthresh uint32
RTT uint32
RTTvar uint32
Snd_ssthresh uint32
Snd_cwnd uint32
Advmss uint32
Reordering uint32
Rcv_rtt uint32
Rcv_space uint32
Total_retrans uint32
Pacing_rate uint64
Max_pacing_rate uint64
Bytes_acked uint64
Bytes_received uint64
Segs_out uint32
Segs_in uint32
Notsent_bytes uint32
Min_rtt uint32
Data_segs_in uint32
Data_segs_out uint32
Delivery_rate uint64
Busy_time uint64
Rwnd_limited uint64
Sndbuf_limited uint64
Delivered uint32
Delivered_ce uint32
Bytes_sent uint64
Bytes_retrans uint64
DSACK_dups uint32
Reord_seen uint32
Rcv_ooopack uint32
Snd_wnd uint32
}
// AssignTCPFingerprint extracts TCP fingerprint information from a connection
func AssignTCPFingerprint(conn net.Conn) (*JA4T, error) {
tcpConn, ok := conn.(*net.TCPConn)
if !ok {
return nil, fmt.Errorf("not a TCPConn")
}
rawConn, err := tcpConn.SyscallConn()
if err != nil {
return nil, fmt.Errorf("SyscallConn failed: %w", err)
}
var info tcpInfo
var sysErr error
err = rawConn.Control(func(fd uintptr) {
size := uint32(unsafe.Sizeof(info))
_, _, errno := syscall.Syscall6(
syscall.SYS_GETSOCKOPT,
fd,
uintptr(syscall.IPPROTO_TCP),
uintptr(syscall.TCP_INFO),
uintptr(unsafe.Pointer(&info)),
uintptr(unsafe.Pointer(&size)),
0,
)
if errno != 0 {
sysErr = errno
}
})
if err != nil {
return nil, fmt.Errorf("SyscallConn.Control: %w", err)
}
if sysErr != nil {
return nil, fmt.Errorf("getsockopt TCP_INFO: %w", sysErr)
}
fp := &JA4T{
Window: info.Snd_wnd,
MSS: uint16(info.SndMss),
}
const (
TCPI_OPT_TIMESTAMPS = 1 << 0
TCPI_OPT_SACK = 1 << 1
TCPI_OPT_WSCALE = 1 << 2
)
if info.Options&TCPI_OPT_SACK != 0 {
fp.Options = append(fp.Options, 4, 1)
}
if info.Options&TCPI_OPT_TIMESTAMPS != 0 {
fp.Options = append(fp.Options, 8, 1)
}
if info.Options&TCPI_OPT_WSCALE != 0 {
fp.Options = append(fp.Options, 3)
fp.WindowScale = info.Wnd_scale
}
return fp, nil
}

View File

@@ -0,0 +1,11 @@
//go:build !linux && !freebsd
package fingerprint
import "net"
// AssignTCPFingerprint is not supported on this platform
func AssignTCPFingerprint(conn net.Conn) (*JA4T, error) {
// Not supported on macOS and other platforms
return &JA4T{}, nil
}

110
internal/fingerprint/tls.go Normal file
View File

@@ -0,0 +1,110 @@
package fingerprint
import (
"context"
"crypto/tls"
"log/slog"
"net"
"net/http"
"sync/atomic"
)
// ApplyTLSFingerprinter configures a TLS server to capture TLS fingerprints
func ApplyTLSFingerprinter(server *http.Server) {
if server.TLSConfig == nil {
return
}
server.TLSConfig = server.TLSConfig.Clone()
getConfigForClient := server.TLSConfig.GetConfigForClient
if getConfigForClient == nil {
getConfigForClient = func(info *tls.ClientHelloInfo) (*tls.Config, error) {
return nil, nil
}
}
server.TLSConfig.GetConfigForClient = func(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
ja3n, ja4 := buildTLSFingerprint(clientHello)
ptr := clientHello.Context().Value(tlsFingerprintKey{})
if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
fpPtr.ja3n.Store(&ja3n)
fpPtr.ja4.Store(&ja4)
}
return getConfigForClient(clientHello)
}
server.ConnContext = func(ctx context.Context, c net.Conn) context.Context {
ctx = context.WithValue(ctx, tlsFingerprintKey{}, &TLSFingerprint{})
if tc, ok := c.(*tls.Conn); ok {
tcpFP, err := AssignTCPFingerprint(tc.NetConn())
if err == nil {
ctx = context.WithValue(ctx, tcpFingerprintKey{}, tcpFP)
} else {
slog.Debug("ja4t error", "err", err)
}
}
return ctx
}
}
type tcpFingerprintKey struct{}
type tlsFingerprintKey struct{}
// TLSFingerprint represents TLS fingerprint data
type TLSFingerprint struct {
ja3n atomic.Pointer[TLSFingerprintJA3N]
ja4 atomic.Pointer[TLSFingerprintJA4]
}
// JA3N returns the JA3N fingerprint
func (f *TLSFingerprint) JA3N() *TLSFingerprintJA3N {
return f.ja3n.Load()
}
// JA4 returns the JA4 fingerprint
func (f *TLSFingerprint) JA4() *TLSFingerprintJA4 {
return f.ja4.Load()
}
const greaseMask = 0x0F0F
const greaseValue = 0x0a0a
// TLS extension numbers
const (
extensionServerName uint16 = 0
extensionStatusRequest uint16 = 5
extensionSupportedCurves uint16 = 10 // supported_groups in TLS 1.3, see RFC 8446, Section 4.2.7
extensionSupportedPoints uint16 = 11
extensionSignatureAlgorithms uint16 = 13
extensionALPN uint16 = 16
extensionSCT uint16 = 18
extensionExtendedMasterSecret uint16 = 23
extensionSessionTicket uint16 = 35
extensionPreSharedKey uint16 = 41
extensionEarlyData uint16 = 42
extensionSupportedVersions uint16 = 43
extensionCookie uint16 = 44
extensionPSKModes uint16 = 45
extensionCertificateAuthorities uint16 = 47
extensionSignatureAlgorithmsCert uint16 = 50
extensionKeyShare uint16 = 51
extensionQUICTransportParameters uint16 = 57
extensionRenegotiationInfo uint16 = 0xff01
extensionECHOuterExtensions uint16 = 0xfd00
extensionEncryptedClientHello uint16 = 0xfe0d
)
func buildTLSFingerprint(hello *tls.ClientHelloInfo) (TLSFingerprintJA3N, TLSFingerprintJA4) {
return TLSFingerprintJA3N(buildJA3N(hello, true)), buildJA4(hello)
}
// GetTLSFingerprint extracts TLS fingerprint from HTTP request context
func GetTLSFingerprint(r *http.Request) *TLSFingerprint {
ptr := r.Context().Value(tlsFingerprintKey{})
if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
return fpPtr
}
return nil
}

View File

@@ -10,11 +10,11 @@ import (
"time"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/policy/checker"
iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
)
func (c *Client) ASNCheckerFor(asns []uint32) checker.Interface {
func (c *Client) ASNCheckerFor(asns []uint32) checker.Impl {
asnMap := map[uint32]struct{}{}
var sb strings.Builder
fmt.Fprintln(&sb, "ASNChecker")

View File

@@ -5,12 +5,12 @@ import (
"net/http/httptest"
"testing"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/lib/policy/checker"
iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
)
var _ checker.Interface = &thoth.ASNChecker{}
var _ checker.Impl = &thoth.ASNChecker{}
func TestASNChecker(t *testing.T) {
cli := loadSecrets(t)

View File

@@ -9,11 +9,11 @@ import (
"strings"
"time"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/policy/checker"
iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
)
func (c *Client) GeoIPCheckerFor(countries []string) checker.Interface {
func (c *Client) GeoIPCheckerFor(countries []string) checker.Impl {
countryMap := map[string]struct{}{}
var sb strings.Builder
fmt.Fprintln(&sb, "GeoIPChecker")

View File

@@ -5,11 +5,11 @@ import (
"net/http/httptest"
"testing"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/lib/policy/checker"
)
var _ checker.Interface = &thoth.GeoIPChecker{}
var _ checker.Impl = &thoth.GeoIPChecker{}
func TestGeoIPChecker(t *testing.T) {
cli := loadSecrets(t)

View File

@@ -4,8 +4,8 @@ import (
"os"
"testing"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/TecharoHQ/anubis/lib/thoth/thothmock"
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/internal/thoth/thothmock"
"github.com/joho/godotenv"
)

View File

@@ -4,7 +4,7 @@ import (
"context"
"testing"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/TecharoHQ/anubis/internal/thoth"
)
func WithMockThoth(t *testing.T) context.Context {

View File

@@ -28,17 +28,15 @@ import (
"github.com/TecharoHQ/anubis/internal/dnsbl"
"github.com/TecharoHQ/anubis/internal/ogtags"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/checker"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
// checker implementations
_ "github.com/TecharoHQ/anubis/lib/checker/all"
// challenge implementations
_ "github.com/TecharoHQ/anubis/lib/challenge/all"
_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
_ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork"
)
var (
@@ -104,10 +102,6 @@ func (s *Server) challengeFor(r *http.Request) (*challenge.Challenge, error) {
ckie := ckies[0]
chall, err := j.Get(r.Context(), "challenge:"+ckie.Value)
if err != nil {
if errors.Is(err, store.ErrNotFound) {
return s.issueChallenge(r.Context(), r)
}
return nil, err
}
@@ -386,23 +380,6 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
lg := internal.GetRequestLogger(r)
localizer := localization.GetLocalizer(r)
redir := r.FormValue("redir")
redirURL, err := url.ParseRequestURI(redir)
if err != nil {
lg.Error("invalid redirect", "err", err)
s.respondWithStatus(w, r, localizer.T("invalid_redirect"), http.StatusBadRequest)
return
}
switch redirURL.Scheme {
case "", "http", "https":
// allowed
default:
lg.Error("XSS attempt blocked, invalid redirect scheme", "scheme", redirURL.Scheme)
s.respondWithStatus(w, r, localizer.T("invalid_redirect"), http.StatusBadRequest)
return
}
// Adjust cookie path if base prefix is not empty
cookiePath := "/"
if anubis.BasePrefix != "" {
@@ -417,6 +394,13 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
return
}
redir := r.FormValue("redir")
redirURL, err := url.ParseRequestURI(redir)
if err != nil {
lg.Error("invalid redirect", "err", err)
s.respondWithError(w, r, localizer.T("invalid_redirect"))
return
}
// used by the path checker rule
r.URL = redirURL
@@ -551,7 +535,7 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
if matches {
return cr("threshold/"+t.Name, t.Action, weight), &policy.Bot{
Challenge: t.Challenge,
Rules: &checker.Any{},
Rules: &checker.List{},
}, nil
}
}
@@ -562,6 +546,6 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
ReportAs: s.policy.DefaultDifficulty,
Algorithm: config.DefaultAlgorithm,
},
Rules: &checker.Any{},
Rules: &checker.List{},
}, nil
}

View File

@@ -1,10 +1,8 @@
package lib
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
@@ -17,9 +15,9 @@ import (
"github.com/TecharoHQ/anubis"
"github.com/TecharoHQ/anubis/data"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/internal/thoth/thothmock"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/thoth/thothmock"
)
func init() {
@@ -738,230 +736,3 @@ func TestStripBasePrefixFromRequest(t *testing.T) {
})
}
}
// TestChallengeFor_ErrNotFound makes sure that users with invalid challenge IDs
// in the test cookie don't get rejected by the database lookup failing.
func TestChallengeFor_ErrNotFound(t *testing.T) {
pol := loadPolicies(t, "testdata/aggressive_403.yaml", 0)
ckieExpiration := 10 * time.Minute
const wrongCookie = "wrong cookie"
srv := spawnAnubis(t, Options{
Next: http.NewServeMux(),
Policy: pol,
CookieDomain: "127.0.0.1",
CookieExpiration: ckieExpiration,
})
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.Header.Set("X-Real-IP", "127.0.0.1")
req.Header.Set("User-Agent", "CHALLENGE")
req.AddCookie(&http.Cookie{Name: anubis.TestCookieName, Value: wrongCookie})
w := httptest.NewRecorder()
srv.maybeReverseProxyOrPage(w, req)
resp := w.Result()
defer resp.Body.Close()
body := new(strings.Builder)
_, err := io.Copy(body, resp.Body)
if err != nil {
t.Fatalf("reading body should not fail: %v", err)
}
t.Run("make sure challenge page is issued", func(t *testing.T) {
if !strings.Contains(body.String(), "anubis_challenge") {
t.Error("should get a challenge page")
}
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("should get a 401 Unauthorized, got: %d", resp.StatusCode)
}
})
t.Run("make sure that the body is not an error page", func(t *testing.T) {
if strings.Contains(body.String(), "reject.webp") {
t.Error("should not get an internal server error")
}
})
t.Run("make sure new test cookie is issued", func(t *testing.T) {
found := false
for _, cookie := range resp.Cookies() {
if cookie.Name == anubis.TestCookieName {
if cookie.Value == wrongCookie {
t.Error("a new challenge cookie should be issued")
}
found = true
}
}
if !found {
t.Error("a new test cookie should be set")
}
})
}
func TestPassChallengeXSS(t *testing.T) {
pol := loadPolicies(t, "", anubis.DefaultDifficulty)
srv := spawnAnubis(t, Options{
Next: http.NewServeMux(),
Policy: pol,
})
ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
defer ts.Close()
cli := httpClient(t)
chall := makeChallenge(t, ts, cli)
testCases := []struct {
name string
redir string
}{
{
name: "javascript alert",
redir: "javascript:alert('xss')",
},
{
name: "vbscript",
redir: "vbscript:msgbox(\"XSS\")",
},
{
name: "data url",
redir: "data:text/html;base64,PHNjcmlwdD5hbGVydCgneHNzJyk8L3NjcmlwdD4=",
},
}
t.Run("with test cookie", func(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
nonce := 0
elapsedTime := 420
calculated := ""
calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce)
calculated = internal.SHA256sum(calcString)
req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
q := req.URL.Query()
q.Set("response", calculated)
q.Set("nonce", fmt.Sprint(nonce))
q.Set("redir", tc.redir)
q.Set("elapsedTime", fmt.Sprint(elapsedTime))
req.URL.RawQuery = q.Encode()
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
for _, ckie := range cli.Jar.Cookies(u) {
if ckie.Name == anubis.TestCookieName {
req.AddCookie(ckie)
}
}
resp, err := cli.Do(req)
if err != nil {
t.Fatalf("can't do request: %v", err)
}
body, _ := io.ReadAll(resp.Body)
if bytes.Contains(body, []byte(tc.redir)) {
t.Log(string(body))
t.Error("found XSS in HTML body")
}
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("wanted status %d, got %d. body: %s", http.StatusBadRequest, resp.StatusCode, body)
}
})
}
})
t.Run("no test cookie", func(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
nonce := 0
elapsedTime := 420
calculated := ""
calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce)
calculated = internal.SHA256sum(calcString)
req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
q := req.URL.Query()
q.Set("response", calculated)
q.Set("nonce", fmt.Sprint(nonce))
q.Set("redir", tc.redir)
q.Set("elapsedTime", fmt.Sprint(elapsedTime))
req.URL.RawQuery = q.Encode()
resp, err := cli.Do(req)
if err != nil {
t.Fatalf("can't do request: %v", err)
}
body, _ := io.ReadAll(resp.Body)
if bytes.Contains(body, []byte(tc.redir)) {
t.Log(string(body))
t.Error("found XSS in HTML body")
}
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("wanted status %d, got %d. body: %s", http.StatusBadRequest, resp.StatusCode, body)
}
})
}
})
}
func TestXForwardedForNoDoubleComma(t *testing.T) {
var h http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Forwarded-For", r.Header.Get("X-Forwarded-For"))
fmt.Fprintln(w, "OK")
})
h = internal.XForwardedForToXRealIP(h)
h = internal.XForwardedForUpdate(false, h)
pol := loadPolicies(t, "testdata/permissive.yaml", 4)
srv := spawnAnubis(t, Options{
Next: h,
Policy: pol,
})
ts := httptest.NewServer(srv)
t.Cleanup(ts.Close)
req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("X-Real-Ip", "10.0.0.1")
resp, err := ts.Client().Do(req)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("response status is wrong, wanted %d but got: %s", http.StatusOK, resp.Status)
}
if xff := resp.Header.Get("X-Forwarded-For"); strings.HasPrefix(xff, ",,") {
t.Errorf("X-Forwarded-For has two leading commas: %q", xff)
}
}

View File

@@ -1,6 +0,0 @@
package all
import (
_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
_ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork"
)

View File

@@ -1,35 +0,0 @@
package checker
import (
"fmt"
"net/http"
"strings"
"github.com/TecharoHQ/anubis/internal"
)
type All []Interface
func (a All) Check(r *http.Request) (bool, error) {
for _, c := range a {
match, err := c.Check(r)
if err != nil {
return match, err
}
if !match {
return false, err // no match
}
}
return true, nil // match
}
func (a All) Hash() string {
var sb strings.Builder
for _, c := range a {
fmt.Fprintln(&sb, c.Hash())
}
return internal.FastHash(sb.String())
}

View File

@@ -1,10 +0,0 @@
// Package all imports all of the standard checker types.
package all
import (
_ "github.com/TecharoHQ/anubis/lib/checker/expression"
_ "github.com/TecharoHQ/anubis/lib/checker/headerexists"
_ "github.com/TecharoHQ/anubis/lib/checker/headermatches"
_ "github.com/TecharoHQ/anubis/lib/checker/path"
_ "github.com/TecharoHQ/anubis/lib/checker/remoteaddress"
)

View File

@@ -1,70 +0,0 @@
package checker
import (
"net/http"
"testing"
)
func TestAll_Check(t *testing.T) {
tests := []struct {
name string
checkers []MockChecker
want bool
wantErr bool
}{
{
name: "All match",
checkers: []MockChecker{
{Result: true, Err: nil},
{Result: true, Err: nil},
},
want: true,
wantErr: false,
},
{
name: "One not match",
checkers: []MockChecker{
{Result: true, Err: nil},
{Result: false, Err: nil},
},
want: false,
wantErr: false,
},
{
name: "No match",
checkers: []MockChecker{
{Result: false, Err: nil},
{Result: false, Err: nil},
},
want: false,
wantErr: false,
},
{
name: "Error encountered",
checkers: []MockChecker{
{Result: true, Err: nil},
{Result: false, Err: http.ErrNotSupported},
},
want: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var all All
for _, mc := range tt.checkers {
all = append(all, mc)
}
got, err := all.Check(nil)
if (err != nil) != tt.wantErr {
t.Errorf("All.Check() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("All.Check() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -1,35 +0,0 @@
package checker
import (
"fmt"
"net/http"
"strings"
"github.com/TecharoHQ/anubis/internal"
)
type Any []Interface
func (a Any) Check(r *http.Request) (bool, error) {
for _, c := range a {
match, err := c.Check(r)
if err != nil {
return match, err
}
if match {
return true, err // match
}
}
return false, nil // no match
}
func (a Any) Hash() string {
var sb strings.Builder
for _, c := range a {
fmt.Fprintln(&sb, c.Hash())
}
return internal.FastHash(sb.String())
}

View File

@@ -1,83 +0,0 @@
package checker
import (
"net/http"
"testing"
)
type MockChecker struct {
Result bool
Err error
}
func (m MockChecker) Check(r *http.Request) (bool, error) {
return m.Result, m.Err
}
func (m MockChecker) Hash() string {
return "mock-hash"
}
func TestAny_Check(t *testing.T) {
tests := []struct {
name string
checkers []MockChecker
want bool
wantErr bool
}{
{
name: "All match",
checkers: []MockChecker{
{Result: true, Err: nil},
{Result: true, Err: nil},
},
want: true,
wantErr: false,
},
{
name: "One match",
checkers: []MockChecker{
{Result: false, Err: nil},
{Result: true, Err: nil},
},
want: true,
wantErr: false,
},
{
name: "No match",
checkers: []MockChecker{
{Result: false, Err: nil},
{Result: false, Err: nil},
},
want: false,
wantErr: false,
},
{
name: "Error encountered",
checkers: []MockChecker{
{Result: false, Err: nil},
{Result: false, Err: http.ErrNotSupported},
},
want: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var any Any
for _, mc := range tt.checkers {
any = append(any, mc)
}
got, err := any.Check(nil)
if (err != nil) != tt.wantErr {
t.Errorf("Any.Check() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("Any.Check() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -1,17 +0,0 @@
// Package checker defines the Checker interface and a helper utility to avoid import cycles.
package checker
import (
"errors"
"net/http"
)
var (
ErrUnparseableConfig = errors.New("checker: config is unparseable")
ErrInvalidConfig = errors.New("checker: config is invalid")
)
type Interface interface {
Check(*http.Request) (matches bool, err error)
Hash() string
}

View File

@@ -1,43 +0,0 @@
package expression
import (
"context"
"encoding/json"
"errors"
"github.com/TecharoHQ/anubis/lib/checker"
)
func init() {
checker.Register("expression", Factory{})
}
type Factory struct{}
func (f Factory) Build(ctx context.Context, data json.RawMessage) (checker.Interface, error) {
var fc = &Config{}
if err := json.Unmarshal([]byte(data), fc); err != nil {
return nil, errors.Join(checker.ErrUnparseableConfig, err)
}
if err := fc.Valid(); err != nil {
return nil, errors.Join(checker.ErrInvalidConfig, err)
}
return New(fc)
}
func (f Factory) Valid(ctx context.Context, data json.RawMessage) error {
var fc = &Config{}
if err := json.Unmarshal([]byte(data), fc); err != nil {
return err
}
if err := fc.Valid(); err != nil {
return err
}
return nil
}

View File

@@ -1,32 +0,0 @@
package headerexists
import (
"net/http"
"strings"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/checker"
)
func New(key string) checker.Interface {
return headerExistsChecker{
header: strings.TrimSpace(http.CanonicalHeaderKey(key)),
hash: internal.FastHash(key),
}
}
type headerExistsChecker struct {
header, hash string
}
func (hec headerExistsChecker) Check(r *http.Request) (bool, error) {
if r.Header.Get(hec.header) != "" {
return true, nil
}
return false, nil
}
func (hec headerExistsChecker) Hash() string {
return hec.hash
}

View File

@@ -1,57 +0,0 @@
package headerexists
import (
"encoding/json"
"fmt"
"net/http"
"testing"
)
func TestChecker(t *testing.T) {
fac := Factory{}
for _, tt := range []struct {
name string
header string
reqHeader string
ok bool
}{
{
name: "match",
header: "Authorization",
reqHeader: "Authorization",
ok: true,
},
{
name: "not_match",
header: "Authorization",
reqHeader: "Authentication",
},
} {
t.Run(tt.name, func(t *testing.T) {
hec, err := fac.Build(t.Context(), json.RawMessage(fmt.Sprintf("%q", tt.header)))
if err != nil {
t.Fatal(err)
}
t.Log(hec.Hash())
r, err := http.NewRequest(http.MethodGet, "/", nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
r.Header.Set(tt.reqHeader, "hunter2")
ok, err := hec.Check(r)
if tt.ok != ok {
t.Errorf("ok: %v, wanted: %v", ok, tt.ok)
}
if err != nil {
t.Errorf("err: %v", err)
}
})
}
}

View File

@@ -1,40 +0,0 @@
package headerexists
import (
"context"
"encoding/json"
"fmt"
"net/http"
"github.com/TecharoHQ/anubis/lib/checker"
)
type Factory struct{}
func (f Factory) Build(ctx context.Context, data json.RawMessage) (checker.Interface, error) {
var headerName string
if err := json.Unmarshal([]byte(data), &headerName); err != nil {
return nil, fmt.Errorf("%w: want string", checker.ErrUnparseableConfig)
}
if err := f.Valid(ctx, data); err != nil {
return nil, err
}
return New(http.CanonicalHeaderKey(headerName)), nil
}
func (Factory) Valid(ctx context.Context, data json.RawMessage) error {
var headerName string
if err := json.Unmarshal([]byte(data), &headerName); err != nil {
return fmt.Errorf("%w: want string", checker.ErrUnparseableConfig)
}
if headerName == "" {
return fmt.Errorf("%w: string must not be empty", checker.ErrInvalidConfig)
}
return nil
}

View File

@@ -1,60 +0,0 @@
package headerexists
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestFactoryGood(t *testing.T) {
files, err := os.ReadDir("./testdata/good")
if err != nil {
t.Fatal(err)
}
fac := Factory{}
for _, fname := range files {
t.Run(fname.Name(), func(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "good", fname.Name()))
if err != nil {
t.Fatal(err)
}
if err := fac.Valid(t.Context(), json.RawMessage(data)); err != nil {
t.Fatal(err)
}
})
}
}
func TestFactoryBad(t *testing.T) {
files, err := os.ReadDir("./testdata/bad")
if err != nil {
t.Fatal(err)
}
fac := Factory{}
for _, fname := range files {
t.Run(fname.Name(), func(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "bad", fname.Name()))
if err != nil {
t.Fatal(err)
}
t.Run("Build", func(t *testing.T) {
if _, err := fac.Build(t.Context(), json.RawMessage(data)); err == nil {
t.Fatal(err)
}
})
t.Run("Valid", func(t *testing.T) {
if err := fac.Valid(t.Context(), json.RawMessage(data)); err == nil {
t.Fatal(err)
}
})
})
}
}

View File

@@ -1 +0,0 @@
""

View File

@@ -1 +0,0 @@
{}

View File

@@ -1 +0,0 @@
"Authorization"

View File

@@ -1,46 +0,0 @@
package headermatches
import (
"context"
"encoding/json"
"net/http"
"regexp"
"github.com/TecharoHQ/anubis/lib/checker"
)
type Checker struct {
header string
regexp *regexp.Regexp
hash string
}
func (c *Checker) Check(r *http.Request) (bool, error) {
if c.regexp.MatchString(r.Header.Get(c.header)) {
return true, nil
}
return false, nil
}
func (c *Checker) Hash() string {
return c.hash
}
func New(key, valueRex string) (checker.Interface, error) {
fc := fileConfig{
Header: key,
ValueRegex: valueRex,
}
if err := fc.Valid(); err != nil {
return nil, err
}
data, err := json.Marshal(fc)
if err != nil {
return nil, err
}
return Factory{}.Build(context.Background(), json.RawMessage(data))
}

View File

@@ -1,98 +0,0 @@
package headermatches
import (
"encoding/json"
"errors"
"net/http"
"testing"
)
func TestChecker(t *testing.T) {
}
func TestHeaderMatchesChecker(t *testing.T) {
fac := Factory{}
for _, tt := range []struct {
err error
name string
header string
rexStr string
reqHeaderKey string
reqHeaderValue string
ok bool
}{
{
name: "match",
header: "Cf-Worker",
rexStr: ".*",
reqHeaderKey: "Cf-Worker",
reqHeaderValue: "true",
ok: true,
err: nil,
},
{
name: "not_match",
header: "Cf-Worker",
rexStr: "false",
reqHeaderKey: "Cf-Worker",
reqHeaderValue: "true",
ok: false,
err: nil,
},
{
name: "not_present",
header: "Cf-Worker",
rexStr: "foobar",
reqHeaderKey: "Something-Else",
reqHeaderValue: "true",
ok: false,
err: nil,
},
{
name: "invalid_regex",
rexStr: "a(b",
err: ErrInvalidRegex,
},
} {
t.Run(tt.name, func(t *testing.T) {
fc := fileConfig{
Header: tt.header,
ValueRegex: tt.rexStr,
}
data, err := json.Marshal(fc)
if err != nil {
t.Fatal(err)
}
hmc, err := fac.Build(t.Context(), json.RawMessage(data))
if err != nil && !errors.Is(err, tt.err) {
t.Fatalf("creating HeaderMatchesChecker failed")
}
if tt.err != nil && hmc == nil {
return
}
t.Log(hmc.Hash())
r, err := http.NewRequest(http.MethodGet, "/", nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
r.Header.Set(tt.reqHeaderKey, tt.reqHeaderValue)
ok, err := hmc.Check(r)
if tt.ok != ok {
t.Errorf("ok: %v, wanted: %v", ok, tt.ok)
}
if err != nil && tt.err != nil && !errors.Is(err, tt.err) {
t.Errorf("err: %v, wanted: %v", err, tt.err)
}
})
}
}

View File

@@ -1,44 +0,0 @@
package headermatches
import (
"errors"
"fmt"
"regexp"
)
var (
ErrNoHeader = errors.New("headermatches: no header is configured")
ErrNoValueRegex = errors.New("headermatches: no value regex is configured")
ErrInvalidRegex = errors.New("headermatches: value regex is invalid")
)
type fileConfig struct {
Header string `json:"header" yaml:"header"`
ValueRegex string `json:"value_regex" yaml:"value_regex"`
}
func (fc fileConfig) String() string {
return fmt.Sprintf("header=%q value_regex=%q", fc.Header, fc.ValueRegex)
}
func (fc fileConfig) Valid() error {
var errs []error
if fc.Header == "" {
errs = append(errs, ErrNoHeader)
}
if fc.ValueRegex == "" {
errs = append(errs, ErrNoValueRegex)
}
if _, err := regexp.Compile(fc.ValueRegex); err != nil {
errs = append(errs, ErrInvalidRegex, err)
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}

View File

@@ -1,55 +0,0 @@
package headermatches
import (
"errors"
"testing"
)
func TestFileConfigValid(t *testing.T) {
for _, tt := range []struct {
name, description string
in fileConfig
err error
}{
{
name: "simple happy",
description: "the most common usecase",
in: fileConfig{
Header: "User-Agent",
ValueRegex: ".*",
},
},
{
name: "no header",
description: "Header must be set, it is not",
in: fileConfig{
ValueRegex: ".*",
},
err: ErrNoHeader,
},
{
name: "no value regex",
description: "ValueRegex must be set, it is not",
in: fileConfig{
Header: "User-Agent",
},
err: ErrNoValueRegex,
},
{
name: "invalid regex",
description: "the user wrote an invalid value regular expression",
in: fileConfig{
Header: "User-Agent",
ValueRegex: "[a-z",
},
err: ErrInvalidRegex,
},
} {
t.Run(tt.name, func(t *testing.T) {
if err := tt.in.Valid(); !errors.Is(err, tt.err) {
t.Log(tt.description)
t.Fatal(err)
}
})
}
}

View File

@@ -1,66 +0,0 @@
package headermatches
import (
"context"
"encoding/json"
"errors"
"net/http"
"regexp"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/checker"
)
func init() {
checker.Register("header_matches", Factory{})
checker.Register("user_agent", Factory{defaultHeader: "User-Agent"})
}
type Factory struct {
defaultHeader string
}
func (f Factory) Build(ctx context.Context, data json.RawMessage) (checker.Interface, error) {
var fc fileConfig
if f.defaultHeader != "" {
fc.Header = f.defaultHeader
}
if err := json.Unmarshal([]byte(data), &fc); err != nil {
return nil, errors.Join(checker.ErrUnparseableConfig, err)
}
if err := fc.Valid(); err != nil {
return nil, errors.Join(checker.ErrInvalidConfig, err)
}
valueRex, err := regexp.Compile(fc.ValueRegex)
if err != nil {
return nil, errors.Join(ErrInvalidRegex, err)
}
return &Checker{
header: http.CanonicalHeaderKey(fc.Header),
regexp: valueRex,
hash: internal.FastHash(fc.String()),
}, nil
}
func (f Factory) Valid(ctx context.Context, data json.RawMessage) error {
var fc fileConfig
if f.defaultHeader != "" {
fc.Header = f.defaultHeader
}
if err := json.Unmarshal([]byte(data), &fc); err != nil {
return err
}
if err := fc.Valid(); err != nil {
return err
}
return nil
}

View File

@@ -1,52 +0,0 @@
package headermatches
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestFactoryGood(t *testing.T) {
files, err := os.ReadDir("./testdata/good")
if err != nil {
t.Fatal(err)
}
fac := Factory{}
for _, fname := range files {
t.Run(fname.Name(), func(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "good", fname.Name()))
if err != nil {
t.Fatal(err)
}
if err := fac.Valid(t.Context(), json.RawMessage(data)); err != nil {
t.Fatal(err)
}
})
}
}
func TestFactoryBad(t *testing.T) {
files, err := os.ReadDir("./testdata/bad")
if err != nil {
t.Fatal(err)
}
fac := Factory{}
for _, fname := range files {
t.Run(fname.Name(), func(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "bad", fname.Name()))
if err != nil {
t.Fatal(err)
}
if err := fac.Valid(t.Context(), json.RawMessage(data)); err == nil {
t.Fatal(err)
}
})
}
}

View File

@@ -1 +0,0 @@
}

View File

@@ -1,4 +0,0 @@
{
"header": "User-Agent",
"value_regex": "a(b"
}

View File

@@ -1,3 +0,0 @@
{
"value_regex": "PaleMoon"
}

View File

@@ -1,3 +0,0 @@
{
"header": "User-Agent"
}

View File

@@ -1 +0,0 @@
{}

View File

@@ -1,4 +0,0 @@
{
"header": "User-Agent",
"value_regex": "PaleMoon"
}

View File

@@ -1,35 +0,0 @@
package headermatches
import (
"context"
"encoding/json"
"github.com/TecharoHQ/anubis/lib/checker"
)
func ValidUserAgent(valueRex string) error {
fc := fileConfig{
Header: "User-Agent",
ValueRegex: valueRex,
}
return fc.Valid()
}
func NewUserAgent(valueRex string) (checker.Interface, error) {
fc := fileConfig{
Header: "User-Agent",
ValueRegex: valueRex,
}
if err := fc.Valid(); err != nil {
return nil, err
}
data, err := json.Marshal(fc)
if err != nil {
return nil, err
}
return Factory{}.Build(context.Background(), json.RawMessage(data))
}

View File

@@ -1,37 +0,0 @@
package path
import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/TecharoHQ/anubis"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/checker"
)
func New(rexStr string) (checker.Interface, error) {
rex, err := regexp.Compile(strings.TrimSpace(rexStr))
if err != nil {
return nil, fmt.Errorf("%w: regex %s failed parse: %w", anubis.ErrMisconfiguration, rexStr, err)
}
return &Checker{rex, internal.FastHash(rexStr)}, nil
}
type Checker struct {
regexp *regexp.Regexp
hash string
}
func (c *Checker) Check(r *http.Request) (bool, error) {
if c.regexp.MatchString(r.URL.Path) {
return true, nil
}
return false, nil
}
func (c *Checker) Hash() string {
return c.hash
}

View File

@@ -1,90 +0,0 @@
package path
import (
"encoding/json"
"errors"
"net/http"
"testing"
)
func TestChecker(t *testing.T) {
fac := Factory{}
for _, tt := range []struct {
err error
name string
rexStr string
reqPath string
ok bool
}{
{
name: "match",
rexStr: "^/api/.*",
reqPath: "/api/v1/users",
ok: true,
err: nil,
},
{
name: "not_match",
rexStr: "^/api/.*",
reqPath: "/static/index.html",
ok: false,
err: nil,
},
{
name: "wildcard_match",
rexStr: ".*\\.json$",
reqPath: "/data/config.json",
ok: true,
err: nil,
},
{
name: "wildcard_not_match",
rexStr: ".*\\.json$",
reqPath: "/data/config.yaml",
ok: false,
err: nil,
},
{
name: "invalid_regex",
rexStr: "a(b",
err: ErrInvalidRegex,
},
} {
t.Run(tt.name, func(t *testing.T) {
fc := fileConfig{
Regex: tt.rexStr,
}
data, err := json.Marshal(fc)
if err != nil {
t.Fatal(err)
}
pc, err := fac.Build(t.Context(), json.RawMessage(data))
if err != nil && !errors.Is(err, tt.err) {
t.Fatalf("creating PathChecker failed")
}
if tt.err != nil && pc == nil {
return
}
t.Log(pc.Hash())
r, err := http.NewRequest(http.MethodGet, tt.reqPath, nil)
if err != nil {
t.Fatalf("can't make request: %v", err)
}
ok, err := pc.Check(r)
if tt.ok != ok {
t.Errorf("ok: %v, wanted: %v", ok, tt.ok)
}
if err != nil && tt.err != nil && !errors.Is(err, tt.err) {
t.Errorf("err: %v, wanted: %v", err, tt.err)
}
})
}
}

Some files were not shown because too many files have changed in this diff Show More