Compare commits

..

5 Commits

Author SHA1 Message Date
Xe Iaso
0510aad9ae docs: clarify that log filters remove lines if the filter matches
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-09-12 16:55:35 +00:00
Xe Iaso
a7a5e0d5c7 feat(lib): add log filtering rules
Closes #942

Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-09-12 16:51:36 +00:00
Xe Iaso
53516738c1 feat(lib/logging): add slog filter handler, move slog init logic here
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-09-12 16:50:22 +00:00
Xe Iaso
e0d2136ee6 feat(config): add support for log filters
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-09-12 16:50:22 +00:00
Xe Iaso
10f05938ac feat(expressions): add CEL log filtering calls
Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-09-12 16:50:22 +00:00
28 changed files with 489 additions and 448 deletions

View File

@@ -1,7 +1,4 @@
acs
Actorified
actorifiedstore
actorify
Aibrew
alibaba
alrest
@@ -160,7 +157,6 @@ ifm
Imagesift
imgproxy
impressum
inbox
inp
internets
IPTo

View File

@@ -12,14 +12,12 @@ permissions:
jobs:
ssh:
if: github.repository == 'TecharoHQ/anubis'
runs-on: alrest-techarohq
runs-on: ubuntu-24.04
strategy:
matrix:
host:
- riscv64
- ppc64le
- aarch64-4k
- aarch64-16k
- ubuntu@riscv64.techaro.lol
- ci@ppc64le.techaro.lol
steps:
- name: Checkout code
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0

View File

@@ -31,6 +31,7 @@ import (
"github.com/TecharoHQ/anubis/data"
"github.com/TecharoHQ/anubis/internal"
libanubis "github.com/TecharoHQ/anubis/lib"
"github.com/TecharoHQ/anubis/lib/logging"
botPolicy "github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/thoth"
@@ -56,7 +57,6 @@ var (
forcedLanguage = flag.String("forced-language", "", "if set, this language is being used instead of the one from the request's Accept-Language header")
hs512Secret = flag.String("hs512-secret", "", "secret used to sign JWTs, uses ed25519 if not set")
cookieSecure = flag.Bool("cookie-secure", true, "if true, sets the secure flag on Anubis cookies")
cookieSameSite = flag.String("cookie-same-site", "None", "sets the same site option on Anubis cookies, will auto-downgrade None to Lax if cookie-secure is false. Valid values are None, Lax, Strict, and Default.")
ed25519PrivateKeyHex = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned")
ed25519PrivateKeyHexFile = flag.String("ed25519-private-key-hex-file", "", "file name containing value for ed25519-private-key-hex")
metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
@@ -144,22 +144,6 @@ func parseBindNetFromAddr(address string) (string, string) {
return "", address
}
func parseSameSite(s string) (http.SameSite) {
switch strings.ToLower(s) {
case "none":
return http.SameSiteNoneMode
case "lax":
return http.SameSiteLaxMode
case "strict":
return http.SameSiteStrictMode
case "default":
return http.SameSiteDefaultMode
default:
log.Fatalf("invalid cookie same-site mode: %s, valid values are None, Lax, Strict, and Default", s)
}
return http.SameSiteDefaultMode
}
func setupListener(network string, address string) (net.Listener, string) {
formattedAddress := ""
@@ -267,7 +251,7 @@ func main() {
return
}
internal.InitSlog(*slogLevel)
slog.SetDefault(slog.New(logging.Init(*slogLevel)))
internal.SetHealth("anubis", healthv1.HealthCheckResponse_NOT_SERVING)
if *healthcheck {
@@ -449,7 +433,6 @@ func main() {
WebmasterEmail: *webmasterEmail,
OpenGraph: policy.OpenGraph,
CookieSecure: *cookieSecure,
CookieSameSite: parseSameSite(*cookieSameSite),
PublicUrl: *publicUrl,
JWTRestrictionHeader: *jwtRestrictionHeader,
DifficultyInJWT: *difficultyInJWT,
@@ -465,7 +448,10 @@ func main() {
h = internal.XForwardedForUpdate(*xffStripPrivate, h)
h = internal.JA4H(h)
srv := http.Server{Handler: h, ErrorLog: internal.GetFilteredHTTPLogger()}
srv := http.Server{
Handler: h,
ErrorLog: logging.StdlibLogger(s.GetLogger("http-server").Handler(), slog.LevelDebug),
}
listener, listenerUrl := setupListener(*bindNetwork, *bind)
slog.Info(
"listening",
@@ -525,7 +511,10 @@ func metricsServer(ctx context.Context, done func()) {
}
})
srv := http.Server{Handler: mux, ErrorLog: internal.GetFilteredHTTPLogger()}
srv := http.Server{
Handler: mux,
ErrorLog: logging.StdlibLogger(slog.With("subsystem", "metrics-server").Handler(), slog.LevelDebug),
}
listener, metricsUrl := setupListener(*metricsBindNetwork, *metricsBind)
slog.Debug("listening for metrics", "url", metricsUrl)

View File

@@ -10,7 +10,7 @@ import (
"path/filepath"
"strings"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/logging"
"github.com/facebookgo/flagenv"
)
@@ -28,7 +28,7 @@ func main() {
flagenv.Parse()
flag.Parse()
internal.InitSlog(*slogLevel)
slog.SetDefault(slog.New(logging.Init(*slogLevel)))
koDockerRepo := strings.TrimSuffix(*dockerRepo, "/"+filepath.Base(*dockerRepo))

View File

@@ -132,6 +132,28 @@ dnsbl: false
# <!-- ... -->
# Logging settings for Anubis
logging:
# CEL log filters. Note that this is a very powerful feature and it is very easy to get
# yourself into trouble with this. Avoid using log filters unless you are running into
# circumstances like https://github.com/TecharoHQ/anubis/issues/942. This has a nonzero
# impact on logging, which spirals out into a more than zero impact on Anubis'
# performance and memory usage.
filters:
# Every filter must have a name and an expression. You can use the same expression
# syntax as you can with bots or thresholds.
#
# If the expression returns `true`, then the log line is filtered _out_.
- name: "http-stdlib"
# Log lines where the message starts with "http:" are filtered out.
expression: msg.startsWith("http:")
- name: "context-canceled"
# Log lines relating to context cancellation are filtered out.
expression: msg.contains("context canceled")
- name: "http-pipelining"
# Log lines relating to HTTP/1.1 pipelining being improperly handled are filtered out.
expression: msg.contains("Unsolicited response received on idle HTTP channel")
# Open Graph passthrough configuration, see here for more information:
# https://anubis.techaro.lol/docs/admin/configuration/open-graph/
openGraph:

View File

@@ -13,15 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
<!-- This changes the project to: -->
- Add `COOKIE_SAME_SITE_MODE` to force anubis cookies SameSite value, and downgrade automatically from `None` to `Lax` if cookie is insecure.
- Fix lock convoy problem in decaymap ([#1103](https://github.com/TecharoHQ/anubis/issues/1103)).
- Fix lock convoy problem in bbolt by implementing the actor pattern ([#1103](https://github.com/TecharoHQ/anubis/issues/1103)).
- Document missing environment variables in installation guide: `SLOG_LEVEL`, `COOKIE_PREFIX`, `FORCED_LANGUAGE`, and `TARGET_DISABLE_KEEPALIVE` ([#1086](https://github.com/TecharoHQ/anubis/pull/1086)).
- Add validation warning when persistent storage is used without setting signing keys.
- Fixed `robots2policy` to properly group consecutive user agents into `any:` instead of only processing the last one ([#925](https://github.com/TecharoHQ/anubis/pull/925)).
- Fix lock convoy problem in decaymap ([#1103](https://github.com/TecharoHQ/anubis/issues/1103))
- [Log filtering](./admin/configuration/expressions.mdx#log-filtering) rules have been added. This allows users to write custom log filtering logic.
- Document missing environment variables in installation guide: `SLOG_LEVEL`, `COOKIE_PREFIX`, `FORCED_LANGUAGE`, and `TARGET_DISABLE_KEEPALIVE` ([#1086](https://github.com/TecharoHQ/anubis/pull/1086))
- Add validation warning when persistent storage is used without setting signing keys
- Fixed `robots2policy` to properly group consecutive user agents into `any:` instead of only processing the last one ([#925](https://github.com/TecharoHQ/anubis/pull/925))
- Add the [`s3api` storage backend](./admin/policies.mdx#s3api) to allow Anubis to use S3 API compatible object storage as its storage backend.
- Make `cmd/containerbuild` support commas for separating elements of the `--docker-tags` argument as well as newlines.
- Add the `DIFFICULTY_IN_JWT` option, which allows one to add the `difficulty` field in the JWT claims which indicates the difficulty of the token ([#1063](https://github.com/TecharoHQ/anubis/pull/1063)).
- Add the `DIFFICULTY_IN_JWT` option, which allows one to add the `difficulty` field in the JWT claims which indicates the difficulty of the token ([#1063](https://github.com/TecharoHQ/anubis/pull/1063))
- Ported the client-side JS to TypeScript to avoid egregious errors in the future.
- Fixes concurrency problems with very old browsers ([#1082](https://github.com/TecharoHQ/anubis/issues/1082)).

View File

@@ -3,7 +3,6 @@
Anubis supports multiple challenge methods:
- [Meta Refresh](./metarefresh.mdx)
- [Preact](./preact.mdx)
- [Proof of Work](./proof-of-work.mdx)
Read the documentation to know which method is best for you.

View File

@@ -99,6 +99,10 @@ For this rule, if a request comes in matching [the signature of the `go get` com
Anubis exposes the following variables to expressions:
### Bot expressions
Bot expressions are used for evaluating [bot rules](../policies.mdx#bot-policies).
| Name | Type | Explanation | Example |
| :-------------- | :-------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------- |
| `headers` | `map[string, string]` | The [headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers) of the request being processed. | `{"User-Agent": "Mozilla/5.0 Gecko/20100101 Firefox/137.0"}` |
@@ -182,6 +186,23 @@ Something to keep in mind about system load average is that it is not aware of t
Also keep in mind that this does not account for other kinds of latency like I/O latency. A system can have its web applications unresponsive due to high latency from a MySQL server but still have that web application server report a load near or at zero.
### Log filtering
Log filters are run on every time Anubis logs data. These are high throughput filters and should be written with care.
| Name | Type | Explanation | Example |
| :------ | :-------------------- | :----------------------------------------------------------------------------------------------------- | --------------------------------------- |
| `time` | Timestamp | The time that the log line was emitted. | `2025-08-18T06:45:38-04:00` |
| `msg` | `string` | The text-based message for the given log line. | `"invalid response"` |
| `level` | `string` | The [log level](https://pkg.go.dev/log/slog#Level) for the log message. | `"INFO"` |
| `attrs` | `map[string, string]` | The key -> value attributes for the given log line. Note that this is an expensive variable to access. | `{"err": "internal: the sun exploded"}` |
:::note
When you define a log filter, anything matching that filter is _removed_. Any remaining logs are sent through to the system journal or standard error.
:::
## Functions exposed to Anubis expressions
Anubis expressions can be augmented with the following functions:

View File

@@ -69,7 +69,6 @@ Anubis uses these environment variables for configuration:
| `COOKIE_PARTITIONED` | `false` | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe. |
| `COOKIE_PREFIX` | `anubis-cookie` | The prefix used for browser cookies created by Anubis. Useful for customization or avoiding conflicts with other applications. |
| `COOKIE_SECURE` | `true` | If set to `true`, enables the [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies), meaning that the cookies will only be transmitted over HTTPS. If Anubis is used in an unsecure context (plain HTTP), this will be need to be set to false |
| `COOKIE_SAME_SITE` | `None` | Controls the cookies [`SameSite` attribute](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie#samesitesamesite-value). Allowed: `None`, `Lax`, `Strict`, `Default`. `None` permits cross-site use but modern browsers require it to be **Secure**—so if `COOKIE_SECURE=false` or you serve over plain HTTP, use `Lax` (recommended) or `Strict` or the cookie will be rejected. `Default` uses the Go runtimes `SameSiteDefaultMode`. `None` will be downgraded to `Lax` automatically if cookie is set NOT to be secure. |
| `DIFFICULTY` | `4` | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses. |
| `DIFFICULTY_IN_JWT` | `false` | If set to `true`, adds the `difficulty` field into JWT claims, which indicates the difficulty the token has been generated. This may be useful for statistics and debugging. |
| `ED25519_PRIVATE_KEY_HEX` | unset | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. **Required when using persistent storage backends** (like bbolt) to ensure challenges survive service restarts. When running multiple instances on the same base domain, the key must be the same across all instances. See below for details. |

View File

@@ -123,6 +123,10 @@ remote_addresses:
Anubis has support for showing imprint / impressum information. This is defined in the `impressum` block of your configuration. See [Imprint / Impressum configuration](./configuration/impressum.mdx) for more information.
## Logging
Anubis has support for configuring log filtering using expressions. See the [log filters](./configuration/expressions.mdx#log-filters) of the [expression](./configuration/expressions.mdx) documentation for more information.
## Storage backends
Anubis needs to store temporary data in order to determine if a user is legitimate or not. Administrators should choose a storage backend based on their infrastructure needs. Each backend has its own advantages and disadvantages.

View File

@@ -1,107 +0,0 @@
// Package actorify lets you transform a parallel operation into a serialized
// operation via the Actor pattern[1].
//
// [1]: https://en.wikipedia.org/wiki/Actor_model
package actorify
import (
"context"
"errors"
)
func z[Z any]() Z {
var z Z
return z
}
var (
// ErrActorDied is returned when the actor inbox or reply channel was closed.
ErrActorDied = errors.New("actorify: the actor inbox or reply channel was closed")
)
// Handler is a function alias for the underlying logic the Actor should call.
type Handler[Input, Output any] func(ctx context.Context, input Input) (Output, error)
// Actor is a serializing wrapper that runs a function in a background goroutine.
// Whenever the Call method is invoked, a message is sent to the actor's inbox and then
// the callee waits for a response. Depending on how busy the actor is, this may take
// a moment.
type Actor[Input, Output any] struct {
handler Handler[Input, Output]
inbox chan *message[Input, Output]
}
type message[Input, Output any] struct {
ctx context.Context
arg Input
reply chan reply[Output]
}
type reply[Output any] struct {
output Output
err error
}
// New constructs a new Actor and starts its background thread. Cancel the context and you cancel
// the Actor.
func New[Input, Output any](ctx context.Context, handler Handler[Input, Output]) *Actor[Input, Output] {
result := &Actor[Input, Output]{
handler: handler,
inbox: make(chan *message[Input, Output], 32),
}
go result.handle(ctx)
return result
}
func (a *Actor[Input, Output]) handle(ctx context.Context) {
for {
select {
case <-ctx.Done():
close(a.inbox)
return
case msg, ok := <-a.inbox:
if !ok {
if msg.reply != nil {
close(msg.reply)
}
return
}
result, err := a.handler(msg.ctx, msg.arg)
reply := reply[Output]{
output: result,
err: err,
}
msg.reply <- reply
}
}
}
// Call calls the Actor with a given Input and returns the handler's Output.
//
// This only works with unary functions by design. If you need to have more inputs, define
// a struct type to use as a container.
func (a *Actor[Input, Output]) Call(ctx context.Context, input Input) (Output, error) {
replyCh := make(chan reply[Output])
a.inbox <- &message[Input, Output]{
arg: input,
reply: replyCh,
}
select {
case reply, ok := <-replyCh:
if !ok {
return z[Output](), ErrActorDied
}
return reply.output, reply.err
case <-ctx.Done():
return z[Output](), context.Cause(ctx)
}
}

View File

@@ -1,31 +1,10 @@
package internal
import (
"fmt"
"log"
"log/slog"
"net/http"
"os"
"strings"
)
func InitSlog(level string) {
var programLevel slog.Level
if err := (&programLevel).UnmarshalText([]byte(level)); err != nil {
fmt.Fprintf(os.Stderr, "invalid log level %s: %v, using info\n", level, err)
programLevel = slog.LevelInfo
}
leveler := &slog.LevelVar{}
leveler.Set(programLevel)
h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
AddSource: true,
Level: leveler,
})
slog.SetDefault(slog.New(h))
}
func GetRequestLogger(base *slog.Logger, r *http.Request) *slog.Logger {
host := r.Host
if host == "" {
@@ -44,27 +23,3 @@ func GetRequestLogger(base *slog.Logger, r *http.Request) *slog.Logger {
"x-real-ip", r.Header.Get("X-Real-Ip"),
)
}
// ErrorLogFilter is used to suppress "context canceled" logs from the http server when a request is canceled (e.g., when a client disconnects).
type ErrorLogFilter struct {
Unwrap *log.Logger
}
func (elf *ErrorLogFilter) Write(p []byte) (n int, err error) {
logMessage := string(p)
if strings.Contains(logMessage, "context canceled") {
return len(p), nil // Suppress the log by doing nothing
}
if strings.Contains(logMessage, "Unsolicited response received on idle HTTP channel") {
return len(p), nil
}
if elf.Unwrap != nil {
return elf.Unwrap.Writer().Write(p)
}
return len(p), nil
}
func GetFilteredHTTPLogger() *log.Logger {
stdErrLogger := log.New(os.Stderr, "", log.LstdFlags) // essentially what the default logger is.
return log.New(&ErrorLogFilter{Unwrap: stdErrLogger}, "", 0)
}

View File

@@ -1,82 +0,0 @@
package internal
import (
"bytes"
"log"
"log/slog"
"net/http"
"strings"
"testing"
)
func TestErrorLogFilter(t *testing.T) {
var buf bytes.Buffer
destLogger := log.New(&buf, "", 0)
errorFilterWriter := &ErrorLogFilter{Unwrap: destLogger}
testErrorLogger := log.New(errorFilterWriter, "", 0)
// Test Case 1: Suppressed message
suppressedMessage := "http: proxy error: context canceled"
testErrorLogger.Println(suppressedMessage)
if buf.Len() != 0 {
t.Errorf("Suppressed message was written to output. Output: %q", buf.String())
}
buf.Reset()
// Test Case 2: Allowed message
allowedMessage := "http: another error occurred"
testErrorLogger.Println(allowedMessage)
output := buf.String()
if !strings.Contains(output, allowedMessage) {
t.Errorf("Allowed message was not written to output. Output: %q", output)
}
if !strings.HasSuffix(output, "\n") {
t.Errorf("Allowed message output is missing newline. Output: %q", output)
}
buf.Reset()
// Test Case 3: Partially matching message (should be suppressed)
partiallyMatchingMessage := "Some other log before http: proxy error: context canceled and after"
testErrorLogger.Println(partiallyMatchingMessage)
if buf.Len() != 0 {
t.Errorf("Partially matching message was written to output. Output: %q", buf.String())
}
buf.Reset()
}
func TestGetRequestLogger(t *testing.T) {
// Test case 1: Normal request with Host header
req1, _ := http.NewRequest("GET", "http://example.com/test", nil)
req1.Host = "example.com"
logger := slog.Default()
reqLogger := GetRequestLogger(logger, req1)
// We can't easily test the actual log output without setting up a test handler,
// but we can verify the function doesn't panic and returns a logger
if reqLogger == nil {
t.Error("GetRequestLogger returned nil")
}
// Test case 2: Subrequest auth mode with X-Forwarded-Host
req2, _ := http.NewRequest("GET", "http://test.com/auth", nil)
req2.Host = ""
req2.Header.Set("X-Forwarded-Host", "original-site.com")
reqLogger2 := GetRequestLogger(logger, req2)
if reqLogger2 == nil {
t.Error("GetRequestLogger returned nil for X-Forwarded-Host case")
}
// Test case 3: No host information available
req3, _ := http.NewRequest("GET", "http://test.com/nohost", nil)
req3.Host = ""
reqLogger3 := GetRequestLogger(logger, req3)
if reqLogger3 == nil {
t.Error("GetRequestLogger returned nil for no host case")
}
}

View File

@@ -78,6 +78,10 @@ type Server struct {
logger *slog.Logger
}
func (s *Server) GetLogger(subsystem string) *slog.Logger {
return s.logger.With("subsystem", subsystem)
}
func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
// return ED25519 key if HS512 is not set
if len(s.hs512Secret) == 0 {

View File

@@ -299,7 +299,6 @@ func TestCookieSettings(t *testing.T) {
CookieDomain: "127.0.0.1",
CookiePartitioned: true,
CookieSecure: true,
CookieSameSite: http.SameSiteNoneMode,
CookieExpiration: anubis.CookieDefaultExpirationTime,
})
@@ -340,65 +339,6 @@ func TestCookieSettings(t *testing.T) {
if ckie.Secure != srv.opts.CookieSecure {
t.Errorf("wanted secure flag %v, got: %v", srv.opts.CookieSecure, ckie.Secure)
}
if ckie.SameSite != srv.opts.CookieSameSite {
t.Errorf("wanted same site option %v, got: %v", srv.opts.CookieSameSite, ckie.SameSite)
}
}
func TestCookieSettingsSameSiteNoneModeDowngradedToLaxWhenUnsecure(t *testing.T) {
pol := loadPolicies(t, "testdata/zero_difficulty.yaml", 0)
srv := spawnAnubis(t, Options{
Next: http.NewServeMux(),
Policy: pol,
CookieDomain: "127.0.0.1",
CookiePartitioned: true,
CookieSecure: false,
CookieSameSite: http.SameSiteNoneMode,
CookieExpiration: anubis.CookieDefaultExpirationTime,
})
ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
defer ts.Close()
cli := httpClient(t)
chall := makeChallenge(t, ts, cli)
resp := handleChallengeZeroDifficulty(t, ts, cli, chall)
if resp.StatusCode != http.StatusFound {
resp.Write(os.Stderr)
t.Errorf("wanted %d, got: %d", http.StatusFound, resp.StatusCode)
}
var ckie *http.Cookie
for _, cookie := range resp.Cookies() {
t.Logf("%#v", cookie)
if cookie.Name == anubis.CookieName {
ckie = cookie
break
}
}
if ckie == nil {
t.Errorf("Cookie %q not found", anubis.CookieName)
return
}
if ckie.Domain != "127.0.0.1" {
t.Errorf("cookie domain is wrong, wanted 127.0.0.1, got: %s", ckie.Domain)
}
if ckie.Partitioned != srv.opts.CookiePartitioned {
t.Errorf("wanted partitioned flag %v, got: %v", srv.opts.CookiePartitioned, ckie.Partitioned)
}
if ckie.Secure != srv.opts.CookieSecure {
t.Errorf("wanted secure flag %v, got: %v", srv.opts.CookieSecure, ckie.Secure)
}
if ckie.SameSite != http.SameSiteLaxMode {
t.Errorf("wanted same site Lax option %v, got: %v", http.SameSiteLaxMode, ckie.SameSite)
}
}
func TestCheckDefaultDifficultyMatchesPolicy(t *testing.T) {

View File

@@ -43,7 +43,6 @@ type Options struct {
OpenGraph config.OpenGraph
ServeRobotsTXT bool
CookieSecure bool
CookieSameSite http.SameSite
Logger *slog.Logger
PublicUrl string
JWTRestrictionHeader string
@@ -121,6 +120,14 @@ func New(opts Options) (*Server, error) {
logger: opts.Logger,
}
if opts.Policy.Logging != nil {
var err error
result.logger, err = opts.Policy.ApplyLogFilters(opts.Logger)
if err != nil {
return nil, fmt.Errorf("can't create log filters: %w", err)
}
}
mux := http.NewServeMux()
xess.Mount(mux)

View File

@@ -56,8 +56,6 @@ func (s *Server) SetCookie(w http.ResponseWriter, cookieOpts CookieOpts) {
var domain = s.opts.CookieDomain
var name = anubis.CookieName
var path = "/"
var sameSite = s.opts.CookieSameSite
if cookieOpts.Name != "" {
name = cookieOpts.Name
}
@@ -74,15 +72,11 @@ func (s *Server) SetCookie(w http.ResponseWriter, cookieOpts CookieOpts) {
cookieOpts.Expiry = s.opts.CookieExpiration
}
if s.opts.CookieSameSite == http.SameSiteNoneMode && !s.opts.CookieSecure {
sameSite = http.SameSiteLaxMode
}
http.SetCookie(w, &http.Cookie{
Name: name,
Value: cookieOpts.Value,
Expires: time.Now().Add(cookieOpts.Expiry),
SameSite: sameSite,
SameSite: http.SameSiteNoneMode,
Domain: domain,
Secure: s.opts.CookieSecure,
Partitioned: s.opts.CookiePartitioned,
@@ -94,8 +88,6 @@ func (s *Server) ClearCookie(w http.ResponseWriter, cookieOpts CookieOpts) {
var domain = s.opts.CookieDomain
var name = anubis.CookieName
var path = "/"
var sameSite = s.opts.CookieSameSite
if cookieOpts.Name != "" {
name = cookieOpts.Name
}
@@ -107,16 +99,13 @@ func (s *Server) ClearCookie(w http.ResponseWriter, cookieOpts CookieOpts) {
domain = etld
}
}
if s.opts.CookieSameSite == http.SameSiteNoneMode && !s.opts.CookieSecure {
sameSite = http.SameSiteLaxMode
}
http.SetCookie(w, &http.Cookie{
Name: name,
Value: "",
MaxAge: -1,
Expires: time.Now().Add(-1 * time.Minute),
SameSite: sameSite,
SameSite: http.SameSiteNoneMode,
Partitioned: s.opts.CookiePartitioned,
Domain: domain,
Secure: s.opts.CookieSecure,

67
lib/logging/filter.go Normal file
View File

@@ -0,0 +1,67 @@
package logging
import (
"context"
"log/slog"
)
// Filterer is the shape of any type that can perform log filtering. This takes
// the context of the log filtering call and the log record to be filtered.
type Filterer interface {
Filter(ctx context.Context, r slog.Record) bool
}
// FilterFunc lets you make inline log filters with plain functions.
type FilterFunc func(ctx context.Context, r *slog.Record) bool
// Filter implements Filterer for FilterFunc.
func (ff FilterFunc) Filter(ctx context.Context, r *slog.Record) bool {
return ff(ctx, r)
}
// FilterHandler wraps a slog Handler with one or more filters, enabling administrators
// to customize the logging subsystem of Anubis.
type FilterHandler struct {
next slog.Handler
filters []Filterer
}
// NewFilterHandler creates a new filtering handler with the given base handler and filters.
func NewFilterHandler(handler slog.Handler, filters ...Filterer) *FilterHandler {
return &FilterHandler{
next: handler,
filters: filters,
}
}
// Enabled passes through to the upstream slog Handler.
func (h *FilterHandler) Enabled(ctx context.Context, level slog.Level) bool {
return h.next.Enabled(ctx, level)
}
// Handle implements slog.Handler and applies all filters before delegating to the base handler.
func (h *FilterHandler) Handle(ctx context.Context, r slog.Record) error {
// Apply all filters - if any filter returns false, skip the log
for _, filter := range h.filters {
if !filter.Filter(ctx, r) {
return nil // Skip this log record
}
}
return h.next.Handle(ctx, r)
}
// WithAttrs implements slog.Handler.
func (h *FilterHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return &FilterHandler{
next: h.next.WithAttrs(attrs),
filters: h.filters,
}
}
// WithGroup implements slog.Handler.
func (h *FilterHandler) WithGroup(name string) slog.Handler {
return &FilterHandler{
next: h.next.WithGroup(name),
filters: h.filters,
}
}

24
lib/logging/logging.go Normal file
View File

@@ -0,0 +1,24 @@
package logging
import (
"fmt"
"log/slog"
"os"
)
func Init(level string) slog.Handler {
var programLevel slog.Level
if err := (&programLevel).UnmarshalText([]byte(level)); err != nil {
fmt.Fprintf(os.Stderr, "invalid log level %s: %v, using info\n", level, err)
programLevel = slog.LevelInfo
}
leveler := &slog.LevelVar{}
leveler.Set(programLevel)
h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
AddSource: true,
Level: leveler,
})
return h
}

36
lib/logging/stdlib.go Normal file
View File

@@ -0,0 +1,36 @@
package logging
import (
"bytes"
"context"
"log"
"log/slog"
"time"
)
// handlerWriter is an io.Writer that calls a Handler.
// It is used to link the default log.Logger to the default slog.Logger.
//
// Adapted from https://cs.opensource.google/go/go/+/refs/tags/go1.24.5:src/log/slog/logger.go;l=62
type handlerWriter struct {
h slog.Handler
level slog.Leveler
}
func (w *handlerWriter) Write(buf []byte) (int, error) {
level := w.level.Level()
if !w.h.Enabled(context.Background(), level) {
return 0, nil
}
var pc uintptr
// Remove final newline.
origLen := len(buf) // Report that the entire buf was written.
buf = bytes.TrimSuffix(buf, []byte{'\n'})
r := slog.NewRecord(time.Now(), level, string(buf), pc)
return origLen, w.h.Handle(context.Background(), r)
}
func StdlibLogger(next slog.Handler, level slog.Level) *log.Logger {
return log.New(&handlerWriter{h: next, level: level}, "", log.LstdFlags)
}

View File

@@ -326,6 +326,7 @@ func (sc StatusCodes) Valid() error {
type fileConfig struct {
Bots []BotOrImport `json:"bots"`
DNSBL bool `json:"dnsbl"`
Logging *Logging `json:"logging"`
OpenGraph openGraphFileConfig `json:"openGraph,omitempty"`
Impressum *Impressum `json:"impressum,omitempty"`
StatusCodes StatusCodes `json:"status_codes"`
@@ -368,6 +369,12 @@ func (c *fileConfig) Valid() error {
}
}
if c.Logging != nil {
if err := c.Logging.Valid(); err != nil {
errs = append(errs, err)
}
}
if len(errs) != 0 {
return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
}
@@ -401,6 +408,7 @@ func Load(fin io.Reader, fname string) (*Config, error) {
ConsiderHost: c.OpenGraph.ConsiderHost,
Override: c.OpenGraph.Override,
},
Logging: c.Logging,
StatusCodes: c.StatusCodes,
Store: c.Store,
}
@@ -441,6 +449,12 @@ func Load(fin io.Reader, fname string) (*Config, error) {
result.Impressum = c.Impressum
}
if c.Logging != nil {
if err := c.Logging.Valid(); err != nil {
validationErrs = append(validationErrs, err)
}
}
if len(c.Thresholds) == 0 {
c.Thresholds = DefaultThresholds
}
@@ -465,6 +479,7 @@ type Config struct {
Bots []BotConfig
Thresholds []Threshold
DNSBL bool
Logging *Logging
Impressum *Impressum
OpenGraph OpenGraph
StatusCodes StatusCodes

View File

@@ -0,0 +1,49 @@
package config
import (
"errors"
"fmt"
)
type Logging struct {
Filters []LogFilter `json:"filters,omitempty" yaml:"filters,omitempty"`
}
func (l *Logging) Valid() error {
var errs []error
for _, lf := range l.Filters {
if err := lf.Valid(); err != nil {
errs = append(errs, err)
}
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}
type LogFilter struct {
Name string `json:"name" yaml:"name"`
Expression ExpressionOrList `json:"expression" yaml:"expression"`
}
func (lf LogFilter) Valid() error {
var errs []error
if lf.Name == "" {
errs = append(errs, fmt.Errorf("%w: log filter has no name", ErrMissingValue))
}
if err := lf.Expression.Valid(); err != nil {
errs = append(errs, err)
}
if len(errs) != 0 {
return fmt.Errorf("log filter %q is not valid: %w", lf.Name, errors.Join(errs...))
}
return nil
}

View File

@@ -0,0 +1,114 @@
package expressions
import (
"context"
"fmt"
"log/slog"
"time"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
timestamp "google.golang.org/protobuf/types/known/timestamppb"
)
var (
filterExecutionTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "anubis",
Subsystem: "slog",
Name: "filter_execution_time_nanoseconds",
Help: "How long each log filter took to execute (nanoseconds)",
Buckets: []float64{10, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000, 2000000, 5000000, 10000000}, // 10 nanoseconds to 10 milliseconds
}, []string{"name"})
)
func LogFilter(opts ...cel.EnvOption) (*cel.Env, error) {
return New(
// Slog record metadata
cel.Variable("time", cel.TimestampType),
cel.Variable("msg", cel.StringType),
cel.Variable("level", cel.StringType),
cel.Variable("attrs", cel.MapType(cel.StringType, cel.StringType)),
)
}
func NewFilter(lg *slog.Logger, name, src string) (*Filter, error) {
env, err := LogFilter()
if err != nil {
return nil, fmt.Errorf("logging: can't create CEL env: %w", err)
}
program, err := Compile(env, src)
if err != nil {
return nil, fmt.Errorf("logging: can't compile expression: Compile(%q): %w", src, err)
}
return &Filter{
program: program,
name: name,
src: src,
log: lg.With("filter", name),
}, nil
}
type Filter struct {
program cel.Program
name string
src string
log *slog.Logger
}
func (f Filter) Filter(ctx context.Context, r slog.Record) bool {
t0 := time.Now()
result, _, err := f.program.ContextEval(ctx, &Record{
Record: r,
})
if err != nil {
f.log.Error("error executing log filter", "err", err, "src", f.src)
return false
}
dur := time.Since(t0)
filterExecutionTime.WithLabelValues(f.name).Observe(float64(dur.Nanoseconds()))
//f.log.Debug("filter execution", "dur", dur.Nanoseconds())
if val, ok := result.(types.Bool); ok {
return !bool(val)
}
return false
}
type Record struct {
slog.Record
attrs map[string]string
}
func (r *Record) Parent() cel.Activation { return nil }
func (r *Record) ResolveName(name string) (any, bool) {
switch name {
case "time":
return &timestamp.Timestamp{Seconds: r.Time.Unix()}, true
case "msg":
return r.Message, true
case "level":
return r.Level.String(), true
case "attrs":
if r.attrs == nil {
attrs := map[string]string{}
r.Attrs(func(attr slog.Attr) bool {
attrs[attr.Key] = attr.Value.String()
return true
})
r.attrs = attrs
return attrs, true
}
return r.attrs, true
default:
return nil, false
}
}

View File

@@ -0,0 +1,65 @@
package expressions
import (
"context"
"fmt"
"io"
"log/slog"
"strings"
"testing"
"time"
)
func BenchmarkFilter(b *testing.B) {
log := slog.New(slog.NewTextHandler(io.Discard, nil))
filter, err := NewFilter(log, "benchmark", `msg == "hello"`)
if err != nil {
b.Fatalf("NewFilter() error = %v", err)
}
record := slog.NewRecord(time.Now(), slog.LevelInfo, "hello", 0)
record.AddAttrs(slog.String("foo", "bar"))
ctx := context.Background()
b.ReportAllocs()
for b.Loop() {
filter.Filter(ctx, record)
}
}
func BenchmarkFilterAttributes(b *testing.B) {
for _, numAttrs := range []int{1, 2, 4, 8, 16, 32} {
b.Run(fmt.Sprintf("%d_attributes", numAttrs), func(b *testing.B) {
log := slog.New(slog.NewTextHandler(io.Discard, nil))
var sb strings.Builder
sb.WriteString(`msg == "hello" && "foo" in attrs`)
attrs := make([]slog.Attr, numAttrs)
for i := range numAttrs {
key := fmt.Sprintf("foo%d", i)
val := "bar"
attrs[i] = slog.String(key, val)
}
filter, err := NewFilter(log, "benchmark", sb.String())
if err != nil {
b.Fatalf("NewFilter() error = %v", err)
}
record := slog.NewRecord(time.Now(), slog.LevelInfo, "hello", 0)
record.AddAttrs(attrs...)
ctx := context.Background()
b.ResetTimer()
b.ReportAllocs()
for b.Loop() {
filter.Filter(ctx, record)
}
})
}
}

View File

@@ -8,8 +8,10 @@ import (
"log/slog"
"sync/atomic"
"github.com/TecharoHQ/anubis/lib/logging"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/policy/expressions"
"github.com/TecharoHQ/anubis/lib/store"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/prometheus/client_golang/prometheus"
@@ -35,17 +37,42 @@ type ParsedConfig struct {
Thresholds []*Threshold
DNSBL bool
Impressum *config.Impressum
Logging *config.Logging
OpenGraph config.OpenGraph
DefaultDifficulty int
StatusCodes config.StatusCodes
Store store.Interface
}
func (pc *ParsedConfig) ApplyLogFilters(base *slog.Logger) (*slog.Logger, error) {
var errs []error
var filters []logging.Filterer
for _, f := range pc.Logging.Filters {
filter, err := expressions.NewFilter(base, f.Name, f.Expression.String())
if err != nil {
errs = append(errs, fmt.Errorf("filter %s invalid: %w", f.Name, err))
continue
}
filters = append(filters, filter)
}
result := slog.New(logging.NewFilterHandler(base.Handler(), filters...))
slog.SetDefault(result)
if len(errs) != 0 {
return nil, errors.Join(errs...)
}
return result, nil
}
func newParsedConfig(orig *config.Config) *ParsedConfig {
return &ParsedConfig{
orig: orig,
OpenGraph: orig.OpenGraph,
StatusCodes: orig.StatusCodes,
Logging: orig.Logging,
}
}

View File

@@ -1,82 +0,0 @@
package store
import (
"context"
"time"
"github.com/TecharoHQ/anubis/internal/actorify"
)
type unit struct{}
type ActorifiedStore struct {
Interface
deleteActor *actorify.Actor[string, unit]
getActor *actorify.Actor[string, []byte]
setActor *actorify.Actor[*actorSetReq, unit]
cancel context.CancelFunc
}
type actorSetReq struct {
key string
value []byte
expiry time.Duration
}
func NewActorifiedStore(backend Interface) *ActorifiedStore {
ctx, cancel := context.WithCancel(context.Background())
result := &ActorifiedStore{
Interface: backend,
cancel: cancel,
}
result.deleteActor = actorify.New(ctx, result.actorDelete)
result.getActor = actorify.New(ctx, backend.Get)
result.setActor = actorify.New(ctx, result.actorSet)
return result
}
func (a *ActorifiedStore) Close() { a.cancel() }
func (a *ActorifiedStore) Delete(ctx context.Context, key string) error {
if _, err := a.deleteActor.Call(ctx, key); err != nil {
return err
}
return nil
}
func (a *ActorifiedStore) Get(ctx context.Context, key string) ([]byte, error) {
return a.getActor.Call(ctx, key)
}
func (a *ActorifiedStore) Set(ctx context.Context, key string, value []byte, expiry time.Duration) error {
if _, err := a.setActor.Call(ctx, &actorSetReq{
key: key,
value: value,
expiry: expiry,
}); err != nil {
return err
}
return nil
}
func (a *ActorifiedStore) actorDelete(ctx context.Context, key string) (unit, error) {
if err := a.Interface.Delete(ctx, key); err != nil {
return unit{}, err
}
return unit{}, nil
}
func (a *ActorifiedStore) actorSet(ctx context.Context, req *actorSetReq) (unit, error) {
if err := a.Interface.Set(ctx, req.key, req.value, req.expiry); err != nil {
return unit{}, err
}
return unit{}, nil
}

View File

@@ -48,7 +48,7 @@ func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface
go result.cleanupThread(ctx)
return store.NewActorifiedStore(result), nil
return result, nil
}
// Valid parses and validates the bbolt store Config or returns

View File

@@ -4,37 +4,30 @@ set -euo pipefail
[ ! -z "${DEBUG:-}" ] && set -x
if [ "$#" -ne 1 ]; then
echo "Usage: rigging.sh <user@host>"
echo "Usage: rigging.sh <user@host>"
fi
declare -A Hosts
Hosts["riscv64"]="ubuntu@riscv64.techaro.lol" # GOARCH=riscv64 GOOS=linux
Hosts["ppc64le"]="ci@ppc64le.techaro.lol" # GOARCH=ppc64le GOOS=linux
Hosts["aarch64-4k"]="rocky@192.168.2.52" # GOARCH=arm64 GOOS=linux 4k page size
Hosts["aarch64-16k"]="ci@192.168.2.28" # GOARCH=arm64 GOOS=linux 16k page size
CIRunnerImage="ghcr.io/techarohq/anubis/ci-runner:latest"
RunID=${GITHUB_RUN_ID:-$(uuidgen)}
RunFolder="anubis/runs/${RunID}"
Target="${Hosts["$1"]}"
Target="${1}"
ssh "${Target}" uname -av
ssh "${Target}" mkdir -p "${RunFolder}"
git archive HEAD | ssh "${Target}" tar xC "${RunFolder}"
ssh "${Target}" <<EOF
ssh "${Target}" << EOF
set -euo pipefail
set -x
mkdir -p anubis/cache/{go,go-build,node}
mkdir -p "anubis/cache/{go,go-build,node}"
podman pull ${CIRunnerImage}
podman run --rm -it \
-v "\$HOME/${RunFolder}:/app/anubis:z" \
-v "\$HOME/anubis/cache/go:/root/go:z" \
-v "\$HOME/anubis/cache/go-build:/root/.cache/go-build:z" \
-v "\$HOME/anubis/cache/node:/root/.npm:z" \
-v "\$HOME/${RunFolder}:/app/anubis" \
-v "\$HOME/anubis/cache/go:/root/go" \
-v "\$HOME/anubis/cache/go-build:/root/.cache/go-build" \
-v "\$HOME/anubis/cache/node:/root/.npm" \
-w /app/anubis \
${CIRunnerImage} \
sh /app/anubis/test/ssh-ci/in-container.sh
ssh "${Target}" rm -rf "${RunFolder}"
EOF
EOF