chore(devcontainer): move playwright to its own devcontainer service

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/excludes.txt

@@ -84,7 +84,6 @@
 ^\Q.github/workflows/spelling.yml\E$
 ^data/crawlers/
 ^docs/blog/tags\.yml$
-^docs/docs/user/known-instances.md$
 ^docs/manifest/.*$
 ^docs/static/\.nojekyll$
 ^lib/policy/config/testdata/bad/unparseable\.json$

.github/actions/spelling/expect.txt

@@ -20,7 +20,7 @@ bbolt
 bdba
 berr
 bingbot
-Bitcoin
+bitcoin
 bitrate
 blogging
 Bluesky
@@ -31,7 +31,6 @@ botstopper
 BPort
 Brightbot
 broked
-byteslice
 Bytespider
 cachebuster
 cachediptoasn
@@ -131,6 +130,7 @@ Hashcash
 hashrate
 headermap
 healthcheck
+hebis
 hec
 hmc
 hostable
@@ -250,6 +250,7 @@ RUnlock
 runtimedir
 sas
 sasl
+Scumm
 searchbot
 searx
 sebest
@@ -262,8 +263,10 @@ shellcheck
 Sidetrade
 simprint
 sitemap
+skopeo
 sls
 sni
+Sourceware
 Spambot
 sparkline
 spyderbot
@@ -286,6 +289,7 @@ techarohq
 templ
 templruntime
 testarea
+testdb
 Thancred
 thoth
 thothmock

docs/docs/admin/installation.mdx

@@ -60,14 +60,14 @@ Anubis uses these environment variables for configuration:
 
 | Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | :----------------------------- | :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints (everything starting with `/.within.website/x/anubis/`). For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                                                                                                                                             |
+| `BASE_PREFIX`                  | unset                   | If set, adds a global prefix to all Anubis endpoints. For example, setting this to `/myapp` would make Anubis accessible at `/myapp/` instead of `/`. This is useful when running Anubis behind a reverse proxy that routes based on path prefixes.                                                                                                                                                                                                                                                     |
 | `BIND`                         | `:8923`                 | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock`                                                                                                                                                                                                                                                                                                                                                                                                 |
 | `BIND_NETWORK`                 | `tcp`                   | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports.                                                                                                                                                                                                                                                                                                                                                              |
 | `COOKIE_DOMAIN`                | unset                   | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See this [stackoverflow explanation of cookies](https://stackoverflow.com/a/1063760) for more information.<br/><br/>Note that unlike `REDIRECT_DOMAINS`, you should never include a port number in this variable.                                                                                  |
 | `COOKIE_DYNAMIC_DOMAIN`        | false                   | If set to true, automatically set cookie domain fields based on the hostname of the request. EG: if you are making a request to `anubis.techaro.lol`, the Anubis cookie will be valid for any subdomain of `techaro.lol`.                                                                                                                                                                                                                                                                               |
 | `COOKIE_EXPIRATION_TIME`       | `168h`                  | The amount of time the authorization cookie is valid for.                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `COOKIE_PARTITIONED`           | `false`                 | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe.                                                                                                                                                                                                                                                                          |
-| `COOKIE_SECURE`                | `true`                  | If set to `true`, enables the [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies), meaning that the cookies will only be transmitted over HTTPS. If Anubis is used in an unsecure context (plain HTTP), this will be need to be set to false                                                                                                                                                                                                   |
+| `COOKIE_SECURE`           | `true`                 | If set to `true`, enables the [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies), meaning that the cookies will only be transmitted over HTTPS. If Anubis is used in an unsecure context (plain HTTP), this will be need to be set to false                                                                                                                                                                                                                                                                            |
 | `DIFFICULTY`                   | `4`                     | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses.                                                                                                                                                                                                                                                                                                                                                                                                  |
 | `ED25519_PRIVATE_KEY_HEX`      | unset                   | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details.                                                                                                                                                                                                                                                                                                              |
 | `ED25519_PRIVATE_KEY_HEX_FILE` | unset                   | Path to a file containing the hex-encoded ed25519 private key. Only one of this or its sister option may be set.                                                                                                                                                                                                                                                                                                                                                                                        |

docs/docs/admin/policies.mdx

@@ -289,9 +289,10 @@ When Anubis opens a bbolt database, it takes an exclusive lock on that database.
 
 The `bbolt` backend takes the following configuration options:
 
-| Name   | Type | Example            | Description                                                                                                                  |
+| Name     | Type   | Example            | Description                                                                                                                       |
-| :----- | :--- | :----------------- | :--------------------------------------------------------------------------------------------------------------------------- |
+| :------- | :----- | :----------------- | :-------------------------------------------------------------------------------------------------------------------------------- |
-| `path` | path | `/data/anubis.bdb` | The filesystem path for the Anubis bbolt database. Anubis requires write access to the folder containing the bbolt database. |
+| `bucket` | string | `anubis`           | The bbolt bucket that Anubis should place all its data into. If this is not set, then Anubis will default to the bucket `anubis`. |
+| `path`   | path   | `/data/anubis.bdb` | The filesystem path for the Anubis bbolt database. Anubis requires write access to the folder containing the bbolt database.      |
 
 Example:
 

docs/docs/user/known-instances.md

@@ -46,10 +46,6 @@ This page contains a non-exhaustive list with all websites using Anubis.
 - https://wiki.koha-community.org/
 - https://extensions.typo3.org/
 - https://ebird.org/
-- https://fabulous.systems/
-- https://coinhoards.org/
-- https://pluralpedia.org/
-- https://git.aya.so/
 - <details>
   <summary>FreeCAD</summary>
   - https://forum.freecad.org/
@@ -87,10 +83,3 @@ This page contains a non-exhaustive list with all websites using Anubis.
   - https://karla.hds.hebis.de/
   - and many more (see https://www.hebis.de/dienste/hebis-discovery-system/)
   </details>
-- <details>
-  <summary>Duke University</summary>
-  - https://repository.duke.edu/
-  - https://archives.lib.duke.edu/
-  - https://find.library.duke.edu/
-  - https://nicholas.duke.edu/
-  </details>

lib/store/bbolt/bbolt.go

@@ -2,6 +2,7 @@ package bbolt
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -11,85 +12,52 @@ import (
 	"go.etcd.io/bbolt"
 )
 
-// Sentinel error values used for testing and in admin-visible error messages.
 var (
 	ErrBucketDoesNotExist = errors.New("bbolt: bucket does not exist")
 	ErrNotExists          = errors.New("bbolt: value does not exist in store")
 )
 
-// Store implements store.Interface backed by bbolt[1].
+type Item struct {
-//
+	Data    []byte    `json:"data"`
-// In essence, bbolt is a hierarchical key/value store with a twist: every value
+	Expires time.Time `json:"expires"`
-// needs to belong to a bucket. Buckets can contain an infinite number of
+}
-// buckets. As such, Anubis nests values in buckets. Each value in the store
+
-// is given its own bucket with two keys:
+type Store struct {
-//
+	bucket []byte
-// 1. data - The raw data, usually in JSON
+	bdb    *bbolt.DB
-// 2. expiry - The expiry time formatted as a time.RFC3339Nano timestamp string
-//
-// When Anubis stores a new bit of data, it creates a new bucket for that value.
-// This allows the cleanup phase to iterate over every bucket in the database and
-// only scan the expiry times without having to decode the entire record.
-//
-// bbolt is not suitable for environments where multiple instance of Anubis need
-// to read from and write to the same backend store. For that, use the valkey
-// storage backend.
-//
-// [1]: https://github.com/etcd-io/bbolt
-type Store struct {
-	bdb *bbolt.DB
 }
 
-// Delete a key from the datastore. If the key does not exist, return an error.
 func (s *Store) Delete(ctx context.Context, key string) error {
 	return s.bdb.Update(func(tx *bbolt.Tx) error {
-		if tx.Bucket([]byte(key)) == nil {
+		bkt := tx.Bucket(s.bucket)
+		if bkt == nil {
+			return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
+		}
+
+		if bkt.Get([]byte(key)) == nil {
 			return fmt.Errorf("%w: %q", ErrNotExists, key)
 		}
 
-		return tx.DeleteBucket([]byte(key))
+		return bkt.Delete([]byte(key))
 	})
 }
 
-// Get a value from the datastore.
-//
-// Because each value is stored in its own bucket with data and expiry keys,
-// two get operations are required:
-//
-// 1. Get the expiry key, parse as time.RFC3339Nano. If the key has expired, run deletion in the background and return a "key not found" error.
-// 2. Get the data key, copy into the result byteslice, return it.
 func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
-	var result []byte
+	var i Item
 
 	if err := s.bdb.View(func(tx *bbolt.Tx) error {
-		itemBucket := tx.Bucket([]byte(key))
+		bkt := tx.Bucket(s.bucket)
-		if itemBucket == nil {
+		if bkt == nil {
+			return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
+		}
+
+		bucketData := bkt.Get([]byte(key))
+		if bucketData == nil {
 			return fmt.Errorf("%w: %q", store.ErrNotFound, key)
 		}
 
-		expiryStr := itemBucket.Get([]byte("expiry"))
+		if err := json.Unmarshal(bucketData, &i); err != nil {
-		if expiryStr == nil {
+			return fmt.Errorf("%w: %w", store.ErrCantDecode, err)
-			return fmt.Errorf("[unexpected] %w: %q (expiry is nil)", store.ErrNotFound, key)
-		}
-
-		expiry, err := time.Parse(time.RFC3339Nano, string(expiryStr))
-		if err != nil {
-			return fmt.Errorf("[unexpected] %w: %w", store.ErrCantDecode, err)
-		}
-
-		if time.Now().After(expiry) {
-			go s.Delete(context.Background(), key)
-			return fmt.Errorf("%w: %q", store.ErrNotFound, key)
-		}
-
-		dataStr := itemBucket.Get([]byte("data"))
-		if dataStr == nil {
-			return fmt.Errorf("[unexpected] %w: %q (data is nil)", store.ErrNotFound, key)
-		}
-
-		result = make([]byte, len(dataStr))
-		if n := copy(result, dataStr); n != len(dataStr) {
-			return fmt.Errorf("[unexpected] %w: %d bytes copied of %d", store.ErrCantDecode, n, len(dataStr))
 		}
 
 		return nil
@@ -97,28 +65,32 @@ func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
 		return nil, err
 	}
 
-	return result, nil
+	if time.Now().After(i.Expires) {
+		go s.Delete(context.Background(), key)
+		return nil, fmt.Errorf("%w: %q", store.ErrNotFound, key)
+	}
+
+	return i.Data, nil
 }
 
-// Set a value into the store with a given expiry.
 func (s *Store) Set(ctx context.Context, key string, value []byte, expiry time.Duration) error {
-	expires := time.Now().Add(expiry)
+	i := Item{
+		Data:    value,
+		Expires: time.Now().Add(expiry),
+	}
+
+	data, err := json.Marshal(i)
+	if err != nil {
+		return fmt.Errorf("%w: %w", store.ErrCantEncode, err)
+	}
 
 	return s.bdb.Update(func(tx *bbolt.Tx) error {
-		valueBkt, err := tx.CreateBucketIfNotExists([]byte(key))
+		bkt := tx.Bucket(s.bucket)
-		if err != nil {
+		if bkt == nil {
-			return fmt.Errorf("%w: %w: %q (create bucket)", store.ErrCantEncode, err, key)
+			return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
 		}
 
-		if err := valueBkt.Put([]byte("expiry"), []byte(expires.Format(time.RFC3339Nano))); err != nil {
+		return bkt.Put([]byte(key), data)
-			return fmt.Errorf("%w: %q (expiry)", store.ErrCantEncode, key)
-		}
-
-		if err := valueBkt.Put([]byte("data"), value); err != nil {
-			return fmt.Errorf("%w: %q (data)", store.ErrCantEncode, key)
-		}
-
-		return nil
 	})
 }
 
@@ -126,28 +98,31 @@ func (s *Store) cleanup(ctx context.Context) error {
 	now := time.Now()
 
 	return s.bdb.Update(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(key []byte, valueBkt *bbolt.Bucket) error {
+		bkt := tx.Bucket(s.bucket)
-			var expiry time.Time
+		if bkt == nil {
-			var err error
+			return fmt.Errorf("cache bucket %q does not exist", string(s.bucket))
+		}
 
-			expiryStr := valueBkt.Get([]byte("expiry"))
+		return bkt.ForEach(func(k, v []byte) error {
-			if expiryStr == nil {
+			var i Item
-				slog.Warn("while running cleanup, expiry is not set somehow, file a bug?", "key", string(key))
+
-				return nil
+			data := bkt.Get(k)
+			if data == nil {
+				return fmt.Errorf("%s in Cache bucket does not exist???", string(k))
 			}
 
-			expiry, err = time.Parse(time.RFC3339Nano, string(expiryStr))
+			if err := json.Unmarshal(data, &i); err != nil {
-			if err != nil {
+				return fmt.Errorf("can't unmarshal data at key %s: %w", string(k), err)
-				return fmt.Errorf("[unexpected] %w in bucket %q: %w", store.ErrCantDecode, string(key), err)
 			}
 
-			if now.After(expiry) {
+			if now.After(i.Expires) {
-				return valueBkt.DeleteBucket(key)
+				return bkt.Delete(k)
 			}
 
 			return nil
 		})
 	})
+
 }
 
 func (s *Store) cleanupThread(ctx context.Context) {

lib/store/bbolt/bbolt_test.go

@@ -12,7 +12,8 @@ func TestImpl(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "db")
 	t.Log(path)
 	data, err := json.Marshal(Config{
-		Path: path,
+		Path:   path,
+		Bucket: "anubis",
 	})
 	if err != nil {
 		t.Fatal(err)

lib/store/bbolt/factory.go

@@ -21,12 +21,8 @@ func init() {
 	store.Register("bbolt", Factory{})
 }
 
-// Factory builds new instances of the bbolt storage backend according to
-// configuration passed via a json.RawMessage.
 type Factory struct{}
 
-// Build parses and validates the bbolt storage backend Config and creates
-// a new instance of it.
 func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface, error) {
 	var config Config
 	if err := json.Unmarshal([]byte(data), &config); err != nil {
@@ -37,13 +33,28 @@ func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface
 		return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
 	}
 
+	if config.Bucket == "" {
+		config.Bucket = "anubis"
+	}
+
 	bdb, err := bbolt.Open(config.Path, 0600, nil)
 	if err != nil {
 		return nil, fmt.Errorf("can't open bbolt database %s: %w", config.Path, err)
 	}
 
+	if err := bdb.Update(func(tx *bbolt.Tx) error {
+		if _, err := tx.CreateBucketIfNotExists([]byte(config.Bucket)); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		return nil, fmt.Errorf("can't create bbolt bucket %q: %w", config.Bucket, err)
+	}
+
 	result := &Store{
-		bdb: bdb,
+		bdb:    bdb,
+		bucket: []byte(config.Bucket),
 	}
 
 	go result.cleanupThread(ctx)
@@ -51,8 +62,6 @@ func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface
 	return result, nil
 }
 
-// Valid parses and validates the bbolt store Config or returns
-// an error.
 func (Factory) Valid(data json.RawMessage) error {
 	var config Config
 	if err := json.Unmarshal([]byte(data), &config); err != nil {
@@ -66,13 +75,11 @@ func (Factory) Valid(data json.RawMessage) error {
 	return nil
 }
 
-// Config is the bbolt storage backend configuration.
 type Config struct {
-	// Path is the filesystem path of the database. The folder must be writable to Anubis.
+	Path   string `json:"path"`
-	Path string `json:"path"`
+	Bucket string `json:"bucket,omitempty"`
 }
 
-// Valid validates the configuration including checking if its containing folder is writable.
 func (c Config) Valid() error {
 	var errs []error
 
@@ -83,7 +90,6 @@ func (c Config) Valid() error {
 		if err := os.WriteFile(filepath.Join(dir, ".test-file"), []byte(""), 0600); err != nil {
 			errs = append(errs, ErrCantWriteToPath)
 		}
-		os.Remove(filepath.Join(dir, ".test-file"))
 	}
 
 	if len(errs) != 0 {

lib/store/bbolt/factory_test.go

@@ -3,6 +3,7 @@ package bbolt
 import (
 	"encoding/json"
 	"errors"
+	"path/filepath"
 	"testing"
 )
 
@@ -26,6 +27,13 @@ func TestFactoryValid(t *testing.T) {
 				cfg:  Config{},
 				err:  ErrMissingPath,
 			},
+			{
+				name: "unwritable folder",
+				cfg: Config{
+					Path: filepath.Join("/", "testdb"),
+				},
+				err: ErrCantWriteToPath,
+			},
 		} {
 			t.Run(tt.name, func(t *testing.T) {
 				data, err := json.Marshal(tt.cfg)

lib/store/storetest/storetest.go

@@ -38,9 +38,7 @@ func Common(t *testing.T, f store.Factory, config json.RawMessage) {
 
 				val, err := s.Get(t.Context(), t.Name())
 				if errors.Is(err, store.ErrNotFound) {
-					t.Errorf("wanted %s to exist in store but it does not: %v", t.Name(), err)
+					t.Errorf("wanted %s to exist in store but it does not", t.Name())
-				} else if err != nil {
-					t.Error(err)
 				}
 
 				if !bytes.Equal(val, []byte(t.Name())) {