chore(devcontainer): move playwright to its own devcontainer service

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/excludes.txt

@@ -84,7 +84,6 @@
 ^\Q.github/workflows/spelling.yml\E$
 ^data/crawlers/
 ^docs/blog/tags\.yml$
-^docs/docs/user/known-instances.md$
 ^docs/manifest/.*$
 ^docs/static/\.nojekyll$
 ^lib/policy/config/testdata/bad/unparseable\.json$

.github/actions/spelling/expect.txt

@@ -20,7 +20,7 @@ bbolt
 bdba
 berr
 bingbot
-Bitcoin
+bitcoin
 bitrate
 blogging
 Bluesky
@@ -31,7 +31,6 @@ botstopper
 BPort
 Brightbot
 broked
-byteslice
 Bytespider
 cachebuster
 cachediptoasn
@@ -131,6 +130,7 @@ Hashcash
 hashrate
 headermap
 healthcheck
+hebis
 hec
 hmc
 hostable
@@ -250,6 +250,7 @@ RUnlock
 runtimedir
 sas
 sasl
+Scumm
 searchbot
 searx
 sebest
@@ -262,8 +263,10 @@ shellcheck
 Sidetrade
 simprint
 sitemap
+skopeo
 sls
 sni
+Sourceware
 Spambot
 sparkline
 spyderbot
@@ -286,6 +289,7 @@ techarohq
 templ
 templruntime
 testarea
+testdb
 Thancred
 thoth
 thothmock

docs/docs/admin/installation.mdx

@@ -60,7 +60,7 @@ Anubis uses these environment variables for configuration:
 
 | Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | :----------------------------- | :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints (everything starting with `/.within.website/x/anubis/`). For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                                                                                                                                             |
+| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints. For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                                                                                                                                                                                                     |
 | `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                                                                                                                                                                                                                 |
 | `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                                                                                                                                                                                                              |
 | `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See this [stackoverflow explanation of cookies](https://stackoverflow.com/a/1063760) for more information.<br/><br/>Note that unlike `REDIRECT_DOMAINS`, you should never include a port number in this variable.                                                                                  |

docs/docs/admin/policies.mdx

@@ -290,7 +290,8 @@ When Anubis opens a bbolt database, it takes an exclusive lock on that database.
 The `bbolt` backend takes the following configuration options:
 
 | Name     | Type   | Example            | Description                                                                                                                       |
-| :----- | :--- | :----------------- | :--------------------------------------------------------------------------------------------------------------------------- |
+| :------- | :----- | :----------------- | :-------------------------------------------------------------------------------------------------------------------------------- |
+| `bucket` | string | `anubis`           | The bbolt bucket that Anubis should place all its data into. If this is not set, then Anubis will default to the bucket `anubis`. |
 | `path`   | path   | `/data/anubis.bdb` | The filesystem path for the Anubis bbolt database. Anubis requires write access to the folder containing the bbolt database.      |
 
 Example:

docs/docs/user/known-instances.md

@@ -46,10 +46,6 @@ This page contains a non-exhaustive list with all websites using Anubis.
 - https://wiki.koha-community.org/
 - https://extensions.typo3.org/
 - https://ebird.org/
-- https://fabulous.systems/
-- https://coinhoards.org/
-- https://pluralpedia.org/
-- https://git.aya.so/
 - <details>
   <summary>FreeCAD</summary>
   - https://forum.freecad.org/
@@ -87,10 +83,3 @@ This page contains a non-exhaustive list with all websites using Anubis.
   - https://karla.hds.hebis.de/
   - and many more (see https://www.hebis.de/dienste/hebis-discovery-system/)
   </details>
-- <details>
-  <summary>Duke University</summary>
-  - https://repository.duke.edu/
-  - https://archives.lib.duke.edu/
-  - https://find.library.duke.edu/
-  - https://nicholas.duke.edu/
-  </details>

lib/store/bbolt/bbolt.go

@@ -2,6 +2,7 @@ package bbolt
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -11,85 +12,52 @@ import (
 	"go.etcd.io/bbolt"
 )
 
-// Sentinel error values used for testing and in admin-visible error messages.
 var (
 	ErrBucketDoesNotExist = errors.New("bbolt: bucket does not exist")
 	ErrNotExists          = errors.New("bbolt: value does not exist in store")
 )
 
-// Store implements store.Interface backed by bbolt[1].
+type Item struct {
-//
+	Data    []byte    `json:"data"`
-// In essence, bbolt is a hierarchical key/value store with a twist: every value
+	Expires time.Time `json:"expires"`
-// needs to belong to a bucket. Buckets can contain an infinite number of
+}
-// buckets. As such, Anubis nests values in buckets. Each value in the store
+
-// is given its own bucket with two keys:
-//
-// 1. data - The raw data, usually in JSON
-// 2. expiry - The expiry time formatted as a time.RFC3339Nano timestamp string
-//
-// When Anubis stores a new bit of data, it creates a new bucket for that value.
-// This allows the cleanup phase to iterate over every bucket in the database and
-// only scan the expiry times without having to decode the entire record.
-//
-// bbolt is not suitable for environments where multiple instance of Anubis need
-// to read from and write to the same backend store. For that, use the valkey
-// storage backend.
-//
-// [1]: https://github.com/etcd-io/bbolt
 type Store struct {
+	bucket []byte
 	bdb    *bbolt.DB
 }
 
-// Delete a key from the datastore. If the key does not exist, return an error.
 func (s *Store) Delete(ctx context.Context, key string) error {
 	return s.bdb.Update(func(tx *bbolt.Tx) error {
-		if tx.Bucket([]byte(key)) == nil {
+		bkt := tx.Bucket(s.bucket)
+		if bkt == nil {
+			return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
+		}
+
+		if bkt.Get([]byte(key)) == nil {
 			return fmt.Errorf("%w: %q", ErrNotExists, key)
 		}
 
-		return tx.DeleteBucket([]byte(key))
+		return bkt.Delete([]byte(key))
 	})
 }
 
-// Get a value from the datastore.
-//
-// Because each value is stored in its own bucket with data and expiry keys,
-// two get operations are required:
-//
-// 1. Get the expiry key, parse as time.RFC3339Nano. If the key has expired, run deletion in the background and return a "key not found" error.
-// 2. Get the data key, copy into the result byteslice, return it.
 func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
-	var result []byte
+	var i Item
 
 	if err := s.bdb.View(func(tx *bbolt.Tx) error {
-		itemBucket := tx.Bucket([]byte(key))
+		bkt := tx.Bucket(s.bucket)
-		if itemBucket == nil {
+		if bkt == nil {
+			return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
+		}
+
+		bucketData := bkt.Get([]byte(key))
+		if bucketData == nil {
 			return fmt.Errorf("%w: %q", store.ErrNotFound, key)
 		}
 
-		expiryStr := itemBucket.Get([]byte("expiry"))
+		if err := json.Unmarshal(bucketData, &i); err != nil {
-		if expiryStr == nil {
+			return fmt.Errorf("%w: %w", store.ErrCantDecode, err)
-			return fmt.Errorf("[unexpected] %w: %q (expiry is nil)", store.ErrNotFound, key)
-		}
-
-		expiry, err := time.Parse(time.RFC3339Nano, string(expiryStr))
-		if err != nil {
-			return fmt.Errorf("[unexpected] %w: %w", store.ErrCantDecode, err)
-		}
-
-		if time.Now().After(expiry) {
-			go s.Delete(context.Background(), key)
-			return fmt.Errorf("%w: %q", store.ErrNotFound, key)
-		}
-
-		dataStr := itemBucket.Get([]byte("data"))
-		if dataStr == nil {
-			return fmt.Errorf("[unexpected] %w: %q (data is nil)", store.ErrNotFound, key)
-		}
-
-		result = make([]byte, len(dataStr))
-		if n := copy(result, dataStr); n != len(dataStr) {
-			return fmt.Errorf("[unexpected] %w: %d bytes copied of %d", store.ErrCantDecode, n, len(dataStr))
 		}
 
 		return nil
@@ -97,28 +65,32 @@ func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
 		return nil, err
 	}
 
-	return result, nil
+	if time.Now().After(i.Expires) {
+		go s.Delete(context.Background(), key)
+		return nil, fmt.Errorf("%w: %q", store.ErrNotFound, key)
+	}
+
+	return i.Data, nil
 }
 
-// Set a value into the store with a given expiry.
 func (s *Store) Set(ctx context.Context, key string, value []byte, expiry time.Duration) error {
-	expires := time.Now().Add(expiry)
+	i := Item{
+		Data:    value,
+		Expires: time.Now().Add(expiry),
+	}
+
+	data, err := json.Marshal(i)
+	if err != nil {
+		return fmt.Errorf("%w: %w", store.ErrCantEncode, err)
+	}
 
 	return s.bdb.Update(func(tx *bbolt.Tx) error {
-		valueBkt, err := tx.CreateBucketIfNotExists([]byte(key))
+		bkt := tx.Bucket(s.bucket)
-		if err != nil {
+		if bkt == nil {
-			return fmt.Errorf("%w: %w: %q (create bucket)", store.ErrCantEncode, err, key)
+			return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
 		}
 
-		if err := valueBkt.Put([]byte("expiry"), []byte(expires.Format(time.RFC3339Nano))); err != nil {
+		return bkt.Put([]byte(key), data)
-			return fmt.Errorf("%w: %q (expiry)", store.ErrCantEncode, key)
-		}
-
-		if err := valueBkt.Put([]byte("data"), value); err != nil {
-			return fmt.Errorf("%w: %q (data)", store.ErrCantEncode, key)
-		}
-
-		return nil
 	})
 }
 
@@ -126,28 +98,31 @@ func (s *Store) cleanup(ctx context.Context) error {
 	now := time.Now()
 
 	return s.bdb.Update(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(key []byte, valueBkt *bbolt.Bucket) error {
+		bkt := tx.Bucket(s.bucket)
-			var expiry time.Time
+		if bkt == nil {
-			var err error
+			return fmt.Errorf("cache bucket %q does not exist", string(s.bucket))
-
-			expiryStr := valueBkt.Get([]byte("expiry"))
-			if expiryStr == nil {
-				slog.Warn("while running cleanup, expiry is not set somehow, file a bug?", "key", string(key))
-				return nil
 		}
 
-			expiry, err = time.Parse(time.RFC3339Nano, string(expiryStr))
+		return bkt.ForEach(func(k, v []byte) error {
-			if err != nil {
+			var i Item
-				return fmt.Errorf("[unexpected] %w in bucket %q: %w", store.ErrCantDecode, string(key), err)
+
+			data := bkt.Get(k)
+			if data == nil {
+				return fmt.Errorf("%s in Cache bucket does not exist???", string(k))
 			}
 
-			if now.After(expiry) {
+			if err := json.Unmarshal(data, &i); err != nil {
-				return valueBkt.DeleteBucket(key)
+				return fmt.Errorf("can't unmarshal data at key %s: %w", string(k), err)
+			}
+
+			if now.After(i.Expires) {
+				return bkt.Delete(k)
 			}
 
 			return nil
 		})
 	})
+
 }
 
 func (s *Store) cleanupThread(ctx context.Context) {

lib/store/bbolt/bbolt_test.go

@@ -13,6 +13,7 @@ func TestImpl(t *testing.T) {
 	t.Log(path)
 	data, err := json.Marshal(Config{
 		Path:   path,
+		Bucket: "anubis",
 	})
 	if err != nil {
 		t.Fatal(err)

lib/store/bbolt/factory.go

@@ -21,12 +21,8 @@ func init() {
 	store.Register("bbolt", Factory{})
 }
 
-// Factory builds new instances of the bbolt storage backend according to
-// configuration passed via a json.RawMessage.
 type Factory struct{}
 
-// Build parses and validates the bbolt storage backend Config and creates
-// a new instance of it.
 func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface, error) {
 	var config Config
 	if err := json.Unmarshal([]byte(data), &config); err != nil {
@@ -37,13 +33,28 @@ func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface
 		return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
 	}
 
+	if config.Bucket == "" {
+		config.Bucket = "anubis"
+	}
+
 	bdb, err := bbolt.Open(config.Path, 0600, nil)
 	if err != nil {
 		return nil, fmt.Errorf("can't open bbolt database %s: %w", config.Path, err)
 	}
 
+	if err := bdb.Update(func(tx *bbolt.Tx) error {
+		if _, err := tx.CreateBucketIfNotExists([]byte(config.Bucket)); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		return nil, fmt.Errorf("can't create bbolt bucket %q: %w", config.Bucket, err)
+	}
+
 	result := &Store{
 		bdb:    bdb,
+		bucket: []byte(config.Bucket),
 	}
 
 	go result.cleanupThread(ctx)
@@ -51,8 +62,6 @@ func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface
 	return result, nil
 }
 
-// Valid parses and validates the bbolt store Config or returns
-// an error.
 func (Factory) Valid(data json.RawMessage) error {
 	var config Config
 	if err := json.Unmarshal([]byte(data), &config); err != nil {
@@ -66,13 +75,11 @@ func (Factory) Valid(data json.RawMessage) error {
 	return nil
 }
 
-// Config is the bbolt storage backend configuration.
 type Config struct {
-	// Path is the filesystem path of the database. The folder must be writable to Anubis.
 	Path   string `json:"path"`
+	Bucket string `json:"bucket,omitempty"`
 }
 
-// Valid validates the configuration including checking if its containing folder is writable.
 func (c Config) Valid() error {
 	var errs []error
 
@@ -83,7 +90,6 @@ func (c Config) Valid() error {
 		if err := os.WriteFile(filepath.Join(dir, ".test-file"), []byte(""), 0600); err != nil {
 			errs = append(errs, ErrCantWriteToPath)
 		}
-		os.Remove(filepath.Join(dir, ".test-file"))
 	}
 
 	if len(errs) != 0 {

lib/store/bbolt/factory_test.go

@@ -3,6 +3,7 @@ package bbolt
 import (
 	"encoding/json"
 	"errors"
+	"path/filepath"
 	"testing"
 )
 
@@ -26,6 +27,13 @@ func TestFactoryValid(t *testing.T) {
 				cfg:  Config{},
 				err:  ErrMissingPath,
 			},
+			{
+				name: "unwritable folder",
+				cfg: Config{
+					Path: filepath.Join("/", "testdb"),
+				},
+				err: ErrCantWriteToPath,
+			},
 		} {
 			t.Run(tt.name, func(t *testing.T) {
 				data, err := json.Marshal(tt.cfg)

lib/store/storetest/storetest.go

@@ -38,9 +38,7 @@ func Common(t *testing.T, f store.Factory, config json.RawMessage) {
 
 				val, err := s.Get(t.Context(), t.Name())
 				if errors.Is(err, store.ErrNotFound) {
-					t.Errorf("wanted %s to exist in store but it does not: %v", t.Name(), err)
+					t.Errorf("wanted %s to exist in store but it does not", t.Name())
-				} else if err != nil {
-					t.Error(err)
 				}
 
 				if !bytes.Equal(val, []byte(t.Name())) {