feat(lib): use new challenge creation flow

Previously Anubis constructed challenge strings from request metadata.
This was a good idea in spirit, but has turned out to be a very bad idea
in practice. This new flow reuses the Store facility to dynamically
create challenge values with completely random data.

This is a fairly big rewrite of how Anubis processes challenges. Right
now it defaults to using the in-memory storage backend, but on-disk
(boltdb) and valkey-based adaptors will come soon.

Signed-off-by: Xe Iaso <me@xeiaso.net>

lib/anubis.go

@@ -1,8 +1,9 @@
 package lib
 
 import (
+	"context"
 	"crypto/ed25519"
-	"crypto/sha256"
+	"crypto/rand"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -16,6 +17,7 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/google/cel-go/common/types"
+	"github.com/google/uuid"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -30,6 +32,7 @@ import (
 	"github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/checker"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/TecharoHQ/anubis/lib/store"
 
 	// challenge implementations
 	_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
@@ -72,6 +75,7 @@ type Server struct {
 	ed25519Priv ed25519.PrivateKey
 	hs512Secret []byte
 	opts        Options
+	store       store.Interface
 }
 
 func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
@@ -87,23 +91,50 @@ func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
 	}
 }
 
-func (s *Server) challengeFor(r *http.Request, difficulty int) string {
-	var fp [32]byte
-	if len(s.hs512Secret) == 0 {
-		fp = sha256.Sum256(s.ed25519Priv.Public().(ed25519.PublicKey)[:])
-	} else {
-		fp = sha256.Sum256(s.hs512Secret)
+func (s *Server) challengeFor(r *http.Request) (*challenge.Challenge, error) {
+	ckies := r.CookiesNamed(anubis.TestCookieName)
+
+	j := store.JSON[challenge.Challenge]{Underlying: s.store}
+
+	for _, ckie := range ckies {
+		chall, err := j.Get(r.Context(), "challenge:"+ckie.Value)
+		if err != nil {
+			return nil, err
+		}
+
+		return &chall, nil
 	}
 
-	challengeData := fmt.Sprintf(
-		"X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
-		r.Header.Get("X-Real-Ip"),
-		r.UserAgent(),
-		time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
-		fp,
-		difficulty,
-	)
-	return internal.FastHash(challengeData)
+	return s.issueChallenge(r.Context(), r)
+}
+
+func (s *Server) issueChallenge(ctx context.Context, r *http.Request) (*challenge.Challenge, error) {
+	id, err := uuid.NewV7()
+	if err != nil {
+		return nil, err
+	}
+
+	var randomData = make([]byte, 256)
+	if _, err := rand.Read(randomData); err != nil {
+		return nil, err
+	}
+
+	chall := challenge.Challenge{
+		ID:         id.String(),
+		RandomData: fmt.Sprintf("%x", randomData),
+		IssuedAt:   time.Now(),
+		Metadata: map[string]string{
+			"User-Agent": r.Header.Get("User-Agent"),
+			"X-Real-Ip":  r.Header.Get("X-Real-Ip"),
+		},
+	}
+
+	j := store.JSON[challenge.Challenge]{Underlying: s.store}
+	if err := j.Set(ctx, "challenge:"+id.String(), chall, 5*time.Minute); err != nil {
+		return nil, err
+	}
+
+	return &chall, err
 }
 
 func (s *Server) maybeReverseProxyHttpStatusOnly(w http.ResponseWriter, r *http.Request) {
@@ -309,15 +340,30 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	lg = lg.With("check_result", cr)
-	chal := s.challengeFor(r, rule.Challenge.Difficulty)
 
-	s.SetCookie(w, CookieOpts{Host: r.Host, Name: anubis.TestCookieName, Value: chal})
+	chall, err := s.challengeFor(r)
+	if err != nil {
+		lg.Error("failed to fetch or issue challenge", "err", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		err := encoder.Encode(struct {
+			Error string `json:"error"`
+		}{
+			Error: fmt.Sprintf("%s \"makeChallenge\"", localizer.T("internal_server_error")),
+		})
+		if err != nil {
+			lg.Error("failed to encode error response", "err", err)
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+		return
+	}
+
+	s.SetCookie(w, CookieOpts{Host: r.Host, Name: anubis.TestCookieName, Value: chall.ID})
 
 	err = encoder.Encode(struct {
 		Rules     *config.ChallengeRules `json:"rules"`
 		Challenge string                 `json:"challenge"`
 	}{
-		Challenge: chal,
+		Challenge: chall.RandomData,
 		Rules:     rule.Challenge,
 	})
 	if err != nil {
@@ -325,7 +371,7 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
-	lg.Debug("made challenge", "challenge", chal, "rules", rule.Challenge, "cr", cr)
+	lg.Debug("made challenge", "challenge", chall, "rules", rule.Challenge, "cr", cr)
 	challengesIssued.WithLabelValues("api").Inc()
 }
 
@@ -384,10 +430,16 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	challengeStr := s.challengeFor(r, rule.Challenge.Difficulty)
+	chall, err := s.challengeFor(r)
+	if err != nil {
+		lg.Error("check failed", "err", err)
+		s.respondWithError(w, r, fmt.Sprintf("%s: %s", localizer.T("internal_server_error"), rule.Challenge.Algorithm))
+	}
+
 	in := &challenge.ValidateInput{
-		Challenge: challengeStr,
+		Challenge: chall,
 		Rule:      rule,
+		Store:     s.store,
 	}
 
 	if err := impl.Validate(r, lg, in); err != nil {
@@ -409,7 +461,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
 
 	// generate JWT cookie
 	tokenString, err := s.signJWT(jwt.MapClaims{
-		"challenge":  challengeStr,
+		"challenge":  chall.ID,
 		"method":     rule.Challenge.Algorithm,
 		"policyRule": rule.Hash(),
 		"action":     string(cr.Rule),

lib/challenge/challenge.go

@@ -1,65 +1,11 @@
 package challenge
 
-import (
-	"log/slog"
-	"net/http"
-	"sort"
-	"sync"
+import "time"
 
-	"github.com/TecharoHQ/anubis/lib/policy"
-	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/a-h/templ"
-)
-
-var (
-	registry map[string]Impl = map[string]Impl{}
-	regLock  sync.RWMutex
-)
-
-func Register(name string, impl Impl) {
-	regLock.Lock()
-	defer regLock.Unlock()
-
-	registry[name] = impl
-}
-
-func Get(name string) (Impl, bool) {
-	regLock.RLock()
-	defer regLock.RUnlock()
-	result, ok := registry[name]
-	return result, ok
-}
-
-func Methods() []string {
-	regLock.RLock()
-	defer regLock.RUnlock()
-	var result []string
-	for method := range registry {
-		result = append(result, method)
-	}
-	sort.Strings(result)
-	return result
-}
-
-type IssueInput struct {
-	Impressum *config.Impressum
-	Rule      *policy.Bot
-	Challenge string
-	OGTags    map[string]string
-}
-
-type ValidateInput struct {
-	Rule      *policy.Bot
-	Challenge string
-}
-
-type Impl interface {
-	// Setup registers any additional routes with the Impl for assets or API routes.
-	Setup(mux *http.ServeMux)
-
-	// Issue a new challenge to the user, called by the Anubis.
-	Issue(r *http.Request, lg *slog.Logger, in *IssueInput) (templ.Component, error)
-
-	// Validate a challenge, making sure that it passes muster.
-	Validate(r *http.Request, lg *slog.Logger, in *ValidateInput) error
+// Challenge is the metadata about a single challenge issuance.
+type Challenge struct {
+	ID         string            `json:"id"`         // UUID identifying the challenge
+	RandomData string            `json:"randomData"` // The random data the client processes
+	IssuedAt   time.Time         `json:"issuedAt"`   // When the challenge was issued
+	Metadata   map[string]string `json:"metadata"`   // Challenge metadata such as IP address and user agent
 }

lib/challenge/challengetest/challengetest.go

@@ -0,0 +1,23 @@
+package challengetest
+
+import (
+	"testing"
+	"time"
+
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/challenge"
+	"github.com/google/uuid"
+)
+
+func New(t *testing.T) *challenge.Challenge {
+	t.Helper()
+
+	id := uuid.Must(uuid.NewV7())
+	randomData := internal.SHA256sum(time.Now().String())
+
+	return &challenge.Challenge{
+		ID:         id.String(),
+		RandomData: randomData,
+		IssuedAt:   time.Now(),
+	}
+}

lib/challenge/challengetest/challengetest_test.go

@@ -0,0 +1,7 @@
+package challengetest
+
+import "testing"
+
+func TestNew(t *testing.T) {
+	_ = New(t)
+}

lib/challenge/interface.go

@@ -0,0 +1,68 @@
+package challenge
+
+import (
+	"log/slog"
+	"net/http"
+	"sort"
+	"sync"
+
+	"github.com/TecharoHQ/anubis/lib/policy"
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/TecharoHQ/anubis/lib/store"
+	"github.com/a-h/templ"
+)
+
+var (
+	registry map[string]Impl = map[string]Impl{}
+	regLock  sync.RWMutex
+)
+
+func Register(name string, impl Impl) {
+	regLock.Lock()
+	defer regLock.Unlock()
+
+	registry[name] = impl
+}
+
+func Get(name string) (Impl, bool) {
+	regLock.RLock()
+	defer regLock.RUnlock()
+	result, ok := registry[name]
+	return result, ok
+}
+
+func Methods() []string {
+	regLock.RLock()
+	defer regLock.RUnlock()
+	var result []string
+	for method := range registry {
+		result = append(result, method)
+	}
+	sort.Strings(result)
+	return result
+}
+
+type IssueInput struct {
+	Impressum *config.Impressum
+	Rule      *policy.Bot
+	Challenge *Challenge
+	OGTags    map[string]string
+	Store     store.Interface
+}
+
+type ValidateInput struct {
+	Rule      *policy.Bot
+	Challenge *Challenge
+	Store     store.Interface
+}
+
+type Impl interface {
+	// Setup registers any additional routes with the Impl for assets or API routes.
+	Setup(mux *http.ServeMux)
+
+	// Issue a new challenge to the user, called by the Anubis.
+	Issue(r *http.Request, lg *slog.Logger, in *IssueInput) (templ.Component, error)
+
+	// Validate a challenge, making sure that it passes muster.
+	Validate(r *http.Request, lg *slog.Logger, in *ValidateInput) error
+}

lib/challenge/metarefresh/metarefresh.go

@@ -31,11 +31,11 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *challenge.IssueInput)
 
 	q := u.Query()
 	q.Set("redir", r.URL.String())
-	q.Set("challenge", in.Challenge)
+	q.Set("challenge", in.Challenge.RandomData)
 	u.RawQuery = q.Encode()
 
 	loc := localization.GetLocalizer(r)
-	component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), page(in.Challenge, u.String(), in.Rule.Challenge.Difficulty, loc), in.Impressum, in.Challenge, in.Rule.Challenge, in.OGTags, loc)
+	component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), page(u.String(), in.Rule.Challenge.Difficulty, loc), in.Impressum, in.Challenge.RandomData, in.Rule.Challenge, in.OGTags, loc)
 
 	if err != nil {
 		return nil, fmt.Errorf("can't render page: %w", err)
@@ -45,12 +45,10 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *challenge.IssueInput)
 }
 
 func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *challenge.ValidateInput) error {
-	wantChallenge := in.Challenge
-
 	gotChallenge := r.FormValue("challenge")
 
-	if subtle.ConstantTimeCompare([]byte(wantChallenge), []byte(gotChallenge)) != 1 {
-		return challenge.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", challenge.ErrFailed, wantChallenge, gotChallenge))
+	if subtle.ConstantTimeCompare([]byte(in.Challenge.RandomData), []byte(gotChallenge)) != 1 {
+		return challenge.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", challenge.ErrFailed, in.Challenge.RandomData, gotChallenge))
 	}
 
 	return nil

lib/challenge/metarefresh/metarefresh.templ

@@ -7,7 +7,7 @@ import (
 	"github.com/TecharoHQ/anubis/lib/localization"
 )
 
-templ page(challenge, redir string, difficulty int, loc *localization.SimpleLocalizer) {
+templ page(redir string, difficulty int, loc *localization.SimpleLocalizer) {
 	<div class="centered-div">
 		<img id="image" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" + anubis.Version }/>
 		<img style="display:none;" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version }/>

lib/challenge/metarefresh/metarefresh_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.898
+// templ: version: v0.3.906
 package metarefresh
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.
@@ -15,7 +15,7 @@ import (
 	"github.com/TecharoHQ/anubis/lib/localization"
 )
 
-func page(challenge, redir string, difficulty int, loc *localization.SimpleLocalizer) templ.Component {
+func page(redir string, difficulty int, loc *localization.SimpleLocalizer) templ.Component {
 	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
 		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
 		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {

lib/challenge/proofofwork/proofofwork.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/TecharoHQ/anubis/internal"
-	"github.com/TecharoHQ/anubis/lib/challenge"
 	chall "github.com/TecharoHQ/anubis/lib/challenge"
 	"github.com/TecharoHQ/anubis/lib/localization"
 	"github.com/TecharoHQ/anubis/web"
@@ -31,7 +30,7 @@ func (i *Impl) Setup(mux *http.ServeMux) {
 
 func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *chall.IssueInput) (templ.Component, error) {
 	loc := localization.GetLocalizer(r)
-	component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), web.Index(loc), in.Impressum, in.Challenge, in.Rule.Challenge, in.OGTags, loc)
+	component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), web.Index(loc), in.Impressum, in.Challenge.RandomData, in.Rule.Challenge, in.OGTags, loc)
 	if err != nil {
 		return nil, fmt.Errorf("can't render page: %w", err)
 	}
@@ -39,9 +38,9 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *chall.IssueInput) (te
 	return component, nil
 }
 
-func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *challenge.ValidateInput) error {
+func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *chall.ValidateInput) error {
 	rule := in.Rule
-	challenge := in.Challenge
+	challenge := in.Challenge.RandomData
 
 	nonceStr := r.FormValue("nonce")
 	if nonceStr == "" {

lib/challenge/proofofwork/proofofwork_test.go

@@ -127,15 +127,22 @@ func TestBasic(t *testing.T) {
 			i.Setup(http.NewServeMux())
 
 			inp := &challenge.IssueInput{
-				Rule:      bot,
-				Challenge: cs.challengeStr,
+				Rule: bot,
+				Challenge: &challenge.Challenge{
+					RandomData: cs.challengeStr,
+				},
 			}
 
 			if _, err := i.Issue(cs.req, lg, inp); err != nil {
 				t.Errorf("can't issue challenge: %v", err)
 			}
 
-			if err := i.Validate(cs.req, lg, &challenge.ValidateInput{Rule: bot, Challenge: cs.challengeStr}); !errors.Is(err, cs.err) {
+			if err := i.Validate(cs.req, lg, &challenge.ValidateInput{
+				Rule: bot,
+				Challenge: &challenge.Challenge{
+					RandomData: cs.challengeStr,
+				},
+			}); !errors.Is(err, cs.err) {
 				t.Errorf("got wrong error from Validate, got %v but wanted %v", err, cs.err)
 			}
 		})

lib/config.go

@@ -110,6 +110,7 @@ func New(opts Options) (*Server, error) {
 		opts:        opts,
 		DNSBLCache:  decaymap.New[string, dnsbl.DroneBLResponse](),
 		OGTags:      ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph),
+		store:       opts.Policy.Store,
 	}
 
 	mux := http.NewServeMux()

lib/http.go

@@ -128,7 +128,12 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 	}
 
 	challengesIssued.WithLabelValues("embedded").Add(1)
-	challengeStr := s.challengeFor(r, rule.Challenge.Difficulty)
+	chall, err := s.challengeFor(r)
+	if err != nil {
+		lg.Error("can't get challenge", "err", "err")
+		s.respondWithError(w, r, fmt.Sprintf("%s: %s", localizer.T("internal_server_error"), rule.Challenge.Algorithm))
+		return
+	}
 
 	var ogTags map[string]string = nil
 	if s.opts.OpenGraph.Enabled {
@@ -140,7 +145,7 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 	}
 
 	s.SetCookie(w, CookieOpts{
-		Value:  challengeStr,
+		Value:  chall.ID,
 		Host:   r.Host,
 		Path:   "/",
 		Name:   anubis.TestCookieName,
@@ -157,8 +162,9 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 	in := &challenge.IssueInput{
 		Impressum: s.policy.Impressum,
 		Rule:      rule,
-		Challenge: challengeStr,
+		Challenge: chall,
 		OGTags:    ogTags,
+		Store:     s.store,
 	}
 
 	component, err := impl.Issue(r, lg, in)

lib/store/decaymap.go

@@ -1,61 +0,0 @@
-package store
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/TecharoHQ/anubis/decaymap"
-)
-
-type decayMapStore struct {
-	store *decaymap.Impl[string, []byte]
-}
-
-func (d *decayMapStore) Delete(_ context.Context, key string) error {
-	if !d.store.Delete(key) {
-		return fmt.Errorf("%w: %q", ErrNotFound, key)
-	}
-
-	return nil
-}
-
-func (d *decayMapStore) Get(_ context.Context, key string) ([]byte, error) {
-	result, ok := d.store.Get(key)
-	if !ok {
-		return nil, fmt.Errorf("%w: %q", ErrNotFound, key)
-	}
-
-	return result, nil
-}
-
-func (d *decayMapStore) Set(_ context.Context, key string, value []byte, expiry time.Duration) error {
-	d.store.Set(key, value, expiry)
-	return nil
-}
-
-func (d *decayMapStore) cleanupThread(ctx context.Context) {
-	t := time.NewTicker(5 * time.Minute)
-	defer t.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-t.C:
-			d.store.Cleanup()
-		}
-	}
-}
-
-// NewDecayMapStore creates a simple in-memory store. This will not scale
-// to multiple Anubis instances.
-func NewDecayMapStore(ctx context.Context) Interface {
-	result := &decayMapStore{
-		store: decaymap.New[string, []byte](),
-	}
-
-	go result.cleanupThread(ctx)
-
-	return result
-}

web/index_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.898
+// templ: version: v0.3.906
 package web
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.