fix(lib/store/bbolt): gracefully handle the obsolete anubis bucket in cleanup

Signed-off-by: Xe Iaso <me@xeiaso.net>

.devcontainer/devcontainer.json

@@ -7,8 +7,7 @@
   "workspaceFolder": "/workspace/anubis",
   "postStartCommand": "npm ci && go mod download",
   "features": {
-    "ghcr.io/xe/devcontainer-features/ko:1.1.0": {},
-    "ghcr.io/devcontainers/features/github-cli:1": {}
+    "ghcr.io/xe/devcontainer-features/ko:1.1.0": {}
   },
   "initializeCommand": "mkdir -p ${localEnv:HOME}${localEnv:USERPROFILE}/.local/share/atuin",
   "customizations": {

.github/actions/spelling/expect.txt

@@ -22,6 +22,7 @@ berr
 bingbot
 Bitcoin
 bitrate
+blogging
 Bluesky
 blueskybot
 boi
@@ -70,7 +71,6 @@ DDOS
 Debian
 debrpm
 decaymap
-devcontainers
 Diffbot
 discordapp
 discordbot
@@ -120,7 +120,6 @@ goland
 gomod
 goodbot
 googlebot
-gopsutil
 govulncheck
 goyaml
 GPG
@@ -194,7 +193,6 @@ Mojeek
 mojeekbot
 mozilla
 nbf
-nepeat
 netsurf
 nginx
 nicksnyder
@@ -261,7 +259,6 @@ Semrush
 Seo
 setsebool
 shellcheck
-shirou
 Sidetrade
 simprint
 sitemap

README.md

@@ -41,13 +41,6 @@ Anubis is brought to you by sponsors and donors like:
 <a href="https://wildbase.xyz/">
   <img src="./docs/static/img/sponsors/wildbase-logo.webp" alt="Wildbase" height="64">
 </a>
-<a href="https://emma.pet">
-  <img
-    src="./docs/static/img/sponsors/nepeat-logo.webp"
-    alt="Cat eyes over the word Emma in a serif font"
-    height="64"
-  />
-</a>
 
 ## Overview
 

VERSION

@@ -1 +1 @@
-1.21.0-pre1
+1.20.0

cmd/anubis/main.go

@@ -231,6 +231,20 @@ func makeReverseProxy(target string, targetSNI string, targetHost string, insecu
 	return rp, nil
 }
 
+func startDecayMapCleanup(ctx context.Context, s *libanubis.Server) {
+	ticker := time.NewTicker(1 * time.Hour)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			s.CleanupDecayMap()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func main() {
 	flagenv.Parse()
 	flag.Parse()
@@ -407,6 +421,7 @@ func main() {
 		wg.Add(1)
 		go metricsServer(ctx, wg.Done)
 	}
+	go startDecayMapCleanup(ctx, s)
 
 	var h http.Handler
 	h = s

data/botPolicies.yaml

@@ -74,25 +74,6 @@ bots:
     weight:
       adjust: 10
 
-  ## System load based checks.
-  # If the system is under high load, add weight.
-  - name: high-load-average
-    action: WEIGH
-    expression: load_1m >= 10.0 # make sure to end the load comparison in a .0
-    weight:
-      adjust: 20
-
-  ## If your backend service is running on the same operating system as Anubis,
-  ## you can uncomment this rule to make the challenge easier when the system is
-  ## under low load.
-  ##
-  ## If it is not, remove weight.
-  # - name: low-load-average
-  #   action: WEIGH
-  #   expression: load_15m <= 4.0 # make sure to end the load comparison in a .0
-  #   weight:
-  #     adjust: -10
-
   # Generic catchall rule
   - name: generic-browser
     user_agent_regex: >-

docs/docs/CHANGELOG.md

@@ -13,81 +13,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- This changes the project to: -->
 
-## v1.21.0: Minfilia Warde
-
-> Please, be at ease. You are among friends here.
-
-In this release, Anubis becomes internationalized, gains the ability to use system load as input to issuing challenges,
-
-### Big ticket changes
-
-The biggest change is that the ["invalid response" after "success" bug](https://github.com/TecharoHQ/anubis/issues/564) is now finally fixed for good by totally rewriting how Anubis' challenge issuance flow works. Instead of generating challenge strings from request metadata (under the assumption that the values being compared against are stable), Anubis now generates random data for each challenge. This data is stored in the active [storage backend](./admin/policies.mdx#storage-backends) for up to 30 minutes. This also fixes [#746](https://github.com/TecharoHQ/anubis/issues/746) and other similar instances of this issue.
-
-In order to reduce confusion, the "Success" interstitial that shows up when you pass a proof of work challenge has been removed.
-
-#### Storage
-
-Anubis now is able to store things persistently [in memory](./admin/policies.mdx#memory), [on the disk](./admin/policies.mdx#bbolt), or [in Valkey](./admin/policies.mdx#valkey) (this includes other compatible software). By default Anubis uses the in-memory backend. If you have an environment with mutable storage (even if it is temporary), be sure to configure the [`bbolt`](./admin/policies.mdx#bbolt) storage backend.
-
-Anubis now supports localized responses. Locales can be added in [lib/localization/locales/](https://github.com/TecharoHQ/anubis/tree/main/lib/localization/locales). This release includes support for the following languages:
-
-- [Brazilian Portugese](https://github.com/TecharoHQ/anubis/pull/726)
-- [Chinese (Traditional)](https://github.com/TecharoHQ/anubis/pull/759)
-- English
-- [French](https://github.com/TecharoHQ/anubis/pull/716)
-- [German](https://github.com/TecharoHQ/anubis/pull/741)
-- [Spanish](https://github.com/TecharoHQ/anubis/pull/716)
-- [Turkish](https://github.com/TecharoHQ/anubis/pull/751)
-
-If facts or local regulations demand, you can set Anubis default language with the `FORCE_LANGUAGE` environment variable:
-
-```sh
-FORCE_LANGUAGE=de
-```
-
-Anubis can dynamically take action [based on the system load average](./admin/configuration/expressions.mdx#using-the-system-load-average), allowing you to write rules like this:
-
-```yaml
-## System load based checks.
-# If the system is under high load for the last minute, add weight.
-- name: high-load-average
-  action: WEIGH
-  expression: load_1m >= 10.0 # make sure to end the load comparison in a .0
-  weight:
-    adjust: 20
-
-# If it is not for the last 15 minutes, remove weight.
-- name: low-load-average
-  action: WEIGH
-  expression: load_15m <= 4.0 # make sure to end the load comparison in a .0
-  weight:
-    adjust: -10
-```
-
-Something to keep in mind about system load average is that it is not aware of the number of cores the system has. If you have a 16 core system that has 16 processes running but none of them is hogging the CPU, then you will get a load average below 16. If you are in doubt, make your "high load" metric at least two times the number of CPU cores and your "low load" metric at least half of the number of CPU cores. For example:
-
-|      Kind | Core count | Load threshold |
-| --------: | :--------- | :------------- |
-| high load | 4          | `8.0`          |
-|  low load | 4          | `2.0`          |
-| high load | 16         | `32.0`         |
-|  low load | 16         | `8`            |
-
-Also keep in mind that this does not account for other kinds of latency like I/O latency. A system can have its web applications unresponsive due to high latency from a MySQL server but still have that web application server report a load near or at zero.
-
-### Other features and fixes
-
-There are a bunch of other assorted features and fixes too:
-
 - Add `COOKIE_SECURE` option to set the cookie [Secure flag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Cookies#block_access_to_your_cookies)
 - Sets cookie defaults to use [SameSite: None](https://web.dev/articles/samesite-cookies-explained)
 - Determine the `BIND_NETWORK`/`--bind-network` value from the bind address ([#677](https://github.com/TecharoHQ/anubis/issues/677)).
+- Implement localization system. Find locale files in lib/localization/locales/.
 - Implement a [development container](https://containers.dev/) manifest to make contributions easier.
 - Fix dynamic cookie domains functionality ([#731](https://github.com/TecharoHQ/anubis/pull/731))
 - Add option for custom cookie prefix ([#732](https://github.com/TecharoHQ/anubis/pull/732))
-- Make the [Open Graph](./admin/configuration/open-graph.mdx) subsystem and DNSBL subsystem use [storage backends](./admin/policies.mdx#storage-backends) instead of storing everything in memory by default.
+- Add translation for German language ([#741](https://github.com/TecharoHQ/anubis/pull/741))
+- Remove the "Success" interstitial after a proof of work challenge is concluded.
+- Anubis now has the concept of [storage backends](./admin/policies.mdx#storage-backends). These allow you to change how Anubis stores temporary data (in memory, on the disk, or in Valkey). If you run Anubis in an environment where you have a low amount of memory available for Anubis (eg: less than 64 megabytes), be sure to configure the [`bbolt`](./admin/policies.mdx#bbolt) storage backend.
+- The challenge issuance and validation process has been rewritten from scratch. Instead of generating challenge strings from request metadata (under the assumption that the values being compared against are stable), Anubis now generates random data for each challenge. This data is stored in the active [storage backend](./admin/policies.mdx#storage-backends) for up to 30 minutes. Fixes [#564](https://github.com/TecharoHQ/anubis/issues/564), [#746](https://github.com/TecharoHQ/anubis/issues/746), and other similar instances of this issue.
+- Add option for forcing a specific language ([#742](https://github.com/TecharoHQ/anubis/pull/742))
+- Add translation for Turkish language ([#751](https://github.com/TecharoHQ/anubis/pull/751))
 - Allow [Common Crawl](https://commoncrawl.org/) by default so scrapers have less incentive to scrape
-- The [bbolt storage backend](./admin/policies.mdx#bbolt) now runs its cleanup every hour instead of every five minutes.
 
 ### Potentially breaking changes
 
@@ -95,7 +34,7 @@ The following potentially breaking change applies to native installs with system
 
 Each instance of systemd service template now has a unique `RuntimeDirectory`, as opposed to each instance of the service sharing a `RuntimeDirectory`. This change was made to avoid [the `RuntimeDirectory` getting nuked any time one of the Anubis instances restarts](https://github.com/TecharoHQ/anubis/issues/748).
 
-If you configured Anubis' unix sockets to listen on `/run/anubis/foo.sock` for instance `anubis@foo`, you will need to configure Anubis to listen on `/run/anubis/foo/foo.sock` and additionally configure your HTTP load balancer as appropriate.
+If you configured Anubis' unix sockets to listen on `/run/anubis/foo.sock` for instance `anubis@foo`, you will need to configure Anubis to listen on `/run/anubis/foo/sock` and additionally configure your HTTP load balancer as appropriate.
 
 If you need the legacy behaviour, install this [systemd unit dropin](https://www.flatcar.org/docs/latest/setup/systemd/drop-in-units/):
 
@@ -105,8 +44,6 @@ If you need the legacy behaviour, install this [systemd unit dropin](https://www
 RuntimeDirectory=anubis
 ```
 
-Just keep in mind that this will cause problems when Anubis restarts.
-
 ## v1.20.0: Thancred Waters
 
 The big ticket items are as follows:

docs/docs/admin/configuration/expressions.mdx

@@ -99,18 +99,15 @@ For this rule, if a request comes in matching [the signature of the `go get` com
 
 Anubis exposes the following variables to expressions:
 
-| Name            | Type                  | Explanation                                                                                                                                   | Example                                                      |
-| :-------------- | :-------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------- |
-| `headers`       | `map[string, string]` | The [headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers) of the request being processed.                            | `{"User-Agent": "Mozilla/5.0 Gecko/20100101 Firefox/137.0"}` |
-| `host`          | `string`              | The [HTTP hostname](https://web.dev/articles/url-parts#host) the request is targeted to.                                                      | `anubis.techaro.lol`                                         |
-| `load_1m`       | `double`              | The current system load average over the last one minute. This is useful for making [load-based checks](#using-the-system-load-average).      |
-| `load_5m`       | `double`              | The current system load average over the last five minutes. This is useful for making [load-based checks](#using-the-system-load-average).    |
-| `load_15m`      | `double`              | The current system load average over the last fifteen minutes. This is useful for making [load-based checks](#using-the-system-load-average). |
-| `method`        | `string`              | The [HTTP method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Methods) in the request being processed.                        | `GET`, `POST`, `DELETE`, etc.                                |
-| `path`          | `string`              | The [path](https://web.dev/articles/url-parts#pathname) of the request being processed.                                                       | `/`, `/api/memes/create`                                     |
-| `query`         | `map[string, string]` | The [query parameters](https://web.dev/articles/url-parts#query) of the request being processed.                                              | `?foo=bar` -> `{"foo": "bar"}`                               |
-| `remoteAddress` | `string`              | The IP address of the client.                                                                                                                 | `1.1.1.1`                                                    |
-| `userAgent`     | `string`              | The [`User-Agent`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent) string in the request being processed.     | `Mozilla/5.0 Gecko/20100101 Firefox/137.0`                   |
+| Name            | Type                  | Explanation                                                                                                                               | Example                                                      |
+| :-------------- | :-------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------- |
+| `headers`       | `map[string, string]` | The [headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers) of the request being processed.                        | `{"User-Agent": "Mozilla/5.0 Gecko/20100101 Firefox/137.0"}` |
+| `host`          | `string`              | The [HTTP hostname](https://web.dev/articles/url-parts#host) the request is targeted to.                                                  | `anubis.techaro.lol`                                         |
+| `method`        | `string`              | The [HTTP method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Methods) in the request being processed.                    | `GET`, `POST`, `DELETE`, etc.                                |
+| `path`          | `string`              | The [path](https://web.dev/articles/url-parts#pathname) of the request being processed.                                                   | `/`, `/api/memes/create`                                     |
+| `query`         | `map[string, string]` | The [query parameters](https://web.dev/articles/url-parts#query) of the request being processed.                                          | `?foo=bar` -> `{"foo": "bar"}`                               |
+| `remoteAddress` | `string`              | The IP address of the client.                                                                                                             | `1.1.1.1`                                                    |
+| `userAgent`     | `string`              | The [`User-Agent`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent) string in the request being processed. | `Mozilla/5.0 Gecko/20100101 Firefox/137.0`                   |
 
 Of note: in many languages when you look up a key in a map and there is nothing there, the language will return some "falsy" value like `undefined` in JavaScript, `None` in Python, or the zero value of the type in Go. In CEL, if you try to look up a value that does not exist, execution of the expression will fail and Anubis will return an error.
 
@@ -144,44 +141,6 @@ X-Real-Ip: 8.8.8.8
 
 Anubis would return a challenge because all of those conditions are true.
 
-### Using the system load average
-
-In Unix-like systems (such as Linux), every process on the system has to wait its turn to be able to run. This means that as more processes on the system are running, they need to wait longer to be able to execute. The [load average](<https://en.wikipedia.org/wiki/Load_(computing)>) represents the number of processes that want to be able to run but can't run yet. This metric isn't the most reliable to identify a cause, but is great at helping to identify symptoms.
-
-Anubis lets you use the system load average as an input to expressions so that you can make dynamic rules like "when the system is under a low amount of load, dial back the protection, but when it's under a lot of load, crank it up to the mix". This lets you get all of the blocking features of Anubis in the background but only really expose Anubis to users when the system is actively being attacked.
-
-This is best combined with the [weight](../policies.mdx#request-weight) and [threshold](./thresholds.mdx) systems so that you can have Anubis dynamically respond to attacks. Consider these rules in the default configuration file:
-
-```yaml
-## System load based checks.
-# If the system is under high load for the last minute, add weight.
-- name: high-load-average
-  action: WEIGH
-  expression: load_1m >= 10.0 # make sure to end the load comparison in a .0
-  weight:
-    adjust: 20
-
-# If it is not for the last 15 minutes, remove weight.
-- name: low-load-average
-  action: WEIGH
-  expression: load_15m <= 4.0 # make sure to end the load comparison in a .0
-  weight:
-    adjust: -10
-```
-
-This combination of rules makes Anubis dynamically react to the system load and only kick in when the system is under attack.
-
-Something to keep in mind about system load average is that it is not aware of the number of cores the system has. If you have a 16 core system that has 16 processes running but none of them is hogging the CPU, then you will get a load average below 16. If you are in doubt, make your "high load" metric at least two times the number of CPU cores and your "low load" metric at least half of the number of CPU cores. For example:
-
-|      Kind | Core count | Load threshold |
-| --------: | :--------- | :------------- |
-| high load | 4          | `8.0`          |
-|  low load | 4          | `2.0`          |
-| high load | 16         | `32.0`         |
-|  low load | 16         | `8`            |
-
-Also keep in mind that this does not account for other kinds of latency like I/O latency. A system can have its web applications unresponsive due to high latency from a MySQL server but still have that web application server report a load near or at zero.
-
 ## Functions exposed to Anubis expressions
 
 Anubis expressions can be augmented with the following functions:

docs/docs/index.mdx

@@ -58,13 +58,6 @@ Anubis is brought to you by sponsors and donors like:
 <a href="https://wildbase.xyz/">
   <img src="/img/sponsors/wildbase-logo.webp" alt="Wildbase" height="64" />
 </a>
-<a href="https://emma.pet">
-  <img
-    src="/img/sponsors/nepeat-logo.webp"
-    alt="Cat eyes over the word Emma in a serif font"
-    height="64"
-  />
-</a>
 
 ## Overview
 

docs/docs/user/known-instances.md

@@ -50,7 +50,6 @@ This page contains a non-exhaustive list with all websites using Anubis.
 - https://coinhoards.org/
 - https://pluralpedia.org/
 - https://git.aya.so/
-- https://marginalia-search.com/
 - <details>
   <summary>FreeCAD</summary>
   - https://forum.freecad.org/
@@ -95,7 +94,3 @@ This page contains a non-exhaustive list with all websites using Anubis.
   - https://find.library.duke.edu/
   - https://nicholas.duke.edu/
   </details>
-- <details>
-  <summary>Forschungszentrum Jülich</summary>
-  - https://juser.fz-juelich.de/
-  </details>

docs/docusaurus.config.ts

@@ -6,7 +6,7 @@ import type * as Preset from '@docusaurus/preset-classic';
 
 const config: Config = {
   title: 'Anubis',
-  tagline: 'Weigh the soul of incoming HTTP requests to protect your website!',
+  tagline: 'Weigh the soul of incoming HTTP requests using proof-of-work to stop AI crawlers',
   favicon: 'img/favicon.ico',
 
   // Set the production url of your site here
@@ -40,20 +40,27 @@ const config: Config = {
     [
       'classic',
       {
+        docs: {
+          sidebarPath: './sidebars.ts',
+          // Please change this to your repo.
+          // Remove this to remove the "edit this page" links.
+          editUrl:
+            'https://github.com/TecharoHQ/anubis/tree/main/docs/',
+        },
         blog: {
           showReadingTime: true,
           feedOptions: {
             type: ['rss', 'atom', "json"],
             xslt: true,
           },
-          editUrl: 'https://github.com/TecharoHQ/anubis/tree/main/docs/',
+          // Please change this to your repo.
+          // Remove this to remove the "edit this page" links.
+          editUrl:
+            'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
+          // Useful options to enforce blogging best practices
           onInlineTags: 'warn',
           onInlineAuthors: 'warn',
-          onUntruncatedBlogPosts: 'throw',
-        },
-        docs: {
-          sidebarPath: './sidebars.ts',
-          editUrl: 'https://github.com/TecharoHQ/anubis/tree/main/docs/',
+          onUntruncatedBlogPosts: 'warn',
         },
         theme: {
           customCss: './src/css/custom.css',
@@ -67,7 +74,7 @@ const config: Config = {
       respectPrefersColorScheme: true,
     },
     // Replace with your project's social card
-    image: 'img/social-card.jpg',
+    image: 'img/docusaurus-social-card.jpg',
     navbar: {
       title: 'Anubis',
       logo: {
@@ -75,28 +82,23 @@ const config: Config = {
         src: 'img/favicon.webp',
       },
       items: [
-        { to: '/blog', label: 'Blog', position: 'left' },
         {
           type: 'docSidebar',
           sidebarId: 'tutorialSidebar',
           position: 'left',
           label: 'Docs',
         },
+        { to: '/blog', label: 'Blog', position: 'left' },
         {
-          to: '/docs/admin/botstopper',
-          label: "Unbranded Version",
-          position: "left"
+          href: 'https://github.com/sponsors/Xe',
+          label: "Sponsorship",
+          position: 'left'
         },
         {
           href: 'https://github.com/TecharoHQ/anubis',
           label: 'GitHub',
           position: 'right',
         },
-        {
-          href: 'https://github.com/sponsors/Xe',
-          label: "Sponsor the Project",
-          position: 'right'
-        },
       ],
     },
     footer: {

docs/src/components/HomepageFeatures/index.tsx

@@ -5,50 +5,49 @@ import styles from "./styles.module.css";
 
 type FeatureItem = {
   title: string;
-  imageURL: string;
+  Svg: React.ComponentType<React.ComponentProps<"svg">>;
   description: ReactNode;
 };
 
 const FeatureList: FeatureItem[] = [
   {
     title: "Easy to Use",
-    imageURL: require("@site/static/img/anubis/happy.webp").default,
+    Svg: require("@site/static/img/undraw_docusaurus_mountain.svg").default,
     description: (
       <>
-        Anubis sits in the background and weighs the risk of incoming requests.
-        If it asks a client to complete a challenge, no user interaction is
-        required.
+        Anubis is easy to set up, lightweight, and helps get rid of the lowest
+        hanging fruit so you can sleep at night.
       </>
     ),
   },
   {
     title: "Lightweight",
-    imageURL: require("@site/static/img/anubis/pensive.webp").default,
+    Svg: require("@site/static/img/undraw_docusaurus_tree.svg").default,
     description: (
       <>
-        Anubis is so lightweight you'll forget it's there until you look at your
-        hosting bill. On average it uses less than 128 MB of ram.
+        Anubis is efficient and as lightweight as possible, blocking the worst
+        of the bots on the internet and makes it easy to protect what you host
+        online.
       </>
     ),
   },
   {
-    title: "Block the scrapers",
-    imageURL: require("@site/static/img/anubis/reject.webp").default,
+    title: "Multi-threaded",
+    Svg: require("@site/static/img/undraw_docusaurus_react.svg").default,
     description: (
       <>
-        Anubis uses a combination of heuristics to identify and block bots
-        before they take your website down. You can customize the rules with{" "}
-        <a href="/docs/admin/policies">your own policies</a>.
+        Anubis uses a multi-threaded proof of work check to ensure that users
+        browsers are up to date and support modern standards.
       </>
     ),
   },
 ];
 
-function Feature({ title, description, imageURL }: FeatureItem) {
+function Feature({ title, Svg, description }: FeatureItem) {
   return (
     <div className={clsx("col col--4")}>
       <div className="text--center">
-        <img src={imageURL} className={styles.featureSvg} role="img" />
+        <Svg className={styles.featureSvg} role="img" />
       </div>
       <div className="text--center padding-horiz--md">
         <Heading as="h3">{title}</Heading>

docs/src/pages/index.tsx

@@ -31,12 +31,19 @@ export default function Home(): ReactNode {
   const { siteConfig } = useDocusaurusContext();
   return (
     <Layout
-      title={`Anubis: Web AI Firewall Utility`}
-      description="Weigh the soul of incoming HTTP requests to protect your website!"
+      title={`Anubis: self hostable scraper defense software`}
+      description="Weigh the soul of incoming HTTP requests using proof-of-work to stop AI crawlers"
     >
       <HomepageHeader />
       <main>
         <HomepageFeatures />
+
+        <center>
+          <p>
+            This is all placeholder text. It will be fixed. Give me time. I am
+            one person and my project has unexpectedly gone viral.
+          </p>
+        </center>
       </main>
     </Layout>
   );

go.mod

@@ -19,8 +19,6 @@ require (
 	github.com/prometheus/client_golang v1.22.0
 	github.com/redis/go-redis/v9 v9.11.0
 	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a
-	github.com/shirou/gopsutil/v4 v4.25.1
-	github.com/testcontainers/testcontainers-go v0.37.0
 	go.etcd.io/bbolt v1.4.2
 	golang.org/x/net v0.41.0
 	golang.org/x/text v0.26.0
@@ -81,7 +79,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/go-git/go-git/v5 v5.14.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
-	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
@@ -129,6 +127,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.1 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.3.1 // indirect
@@ -139,6 +138,7 @@ require (
 	github.com/suzuki-shunsuke/logrus-error v0.1.4 // indirect
 	github.com/suzuki-shunsuke/pinact v1.6.0 // indirect
 	github.com/suzuki-shunsuke/urfave-cli-help-all v0.0.4 // indirect
+	github.com/testcontainers/testcontainers-go v0.37.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
@@ -149,12 +149,9 @@ require (
 	gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.37.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
-	go.opentelemetry.io/otel/metric v1.37.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
-	go.opentelemetry.io/otel/trace v1.37.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.3 // indirect
 	golang.org/x/crypto v0.39.0 // indirect
@@ -169,8 +166,8 @@ require (
 	golang.org/x/tools v0.34.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
 	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	honnef.co/go/tools v0.6.1 // indirect

go.sum

@@ -6,8 +6,6 @@ cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg=
 cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
 dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
 dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
 github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w=
 github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -148,8 +146,8 @@ github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj
 github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
 github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@@ -216,8 +214,6 @@ github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs
 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 h1:sGm2vDRFUrQJO/Veii4h4zG2vvqG6uWNkBHSTqXOZk0=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2/go.mod h1:wd1YpapPLivG6nQgbf7ZkG1hhSOXDhhn4MLTknx2aAc=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@@ -355,8 +351,6 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -401,22 +395,16 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
-go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
-go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
-go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
-go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
-go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
-go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
+go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
+go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
+go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
 go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
 go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
-go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
-go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
-go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
-go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE=
@@ -506,8 +494,6 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
 golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
-golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
-golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -525,10 +511,10 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk=
 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 h1:hE3bRWtU6uceqlh4fhrSnUyjKHMKB9KrTLLG+bc0ddM=
+google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463/go.mod h1:U90ffi8eUL9MwPcrJylN5+Mk2v3vuPDptd5yyNUiRR8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
 google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
@@ -547,8 +533,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
-gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
 honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=
 k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY=

internal/ogtags/cache.go

@@ -1,7 +1,6 @@
 package ogtags
 
 import (
-	"context"
 	"errors"
 	"log/slog"
 	"net/url"
@@ -9,7 +8,7 @@ import (
 )
 
 // GetOGTags is the main function that retrieves Open Graph tags for a URL
-func (c *OGTagCache) GetOGTags(ctx context.Context, url *url.URL, originalHost string) (map[string]string, error) {
+func (c *OGTagCache) GetOGTags(url *url.URL, originalHost string) (map[string]string, error) {
 	if url == nil {
 		return nil, errors.New("nil URL provided, cannot fetch OG tags")
 	}
@@ -22,12 +21,12 @@ func (c *OGTagCache) GetOGTags(ctx context.Context, url *url.URL, originalHost s
 	cacheKey := c.generateCacheKey(target, originalHost)
 
 	// Check cache first
-	if cachedTags := c.checkCache(ctx, cacheKey); cachedTags != nil {
+	if cachedTags := c.checkCache(cacheKey); cachedTags != nil {
 		return cachedTags, nil
 	}
 
 	// Fetch HTML content, passing the original host
-	doc, err := c.fetchHTMLDocumentWithCache(ctx, target, originalHost, cacheKey)
+	doc, err := c.fetchHTMLDocumentWithCache(target, originalHost, cacheKey)
 	if errors.Is(err, syscall.ECONNREFUSED) {
 		slog.Debug("Connection refused, returning empty tags")
 		return nil, nil
@@ -43,7 +42,7 @@ func (c *OGTagCache) GetOGTags(ctx context.Context, url *url.URL, originalHost s
 	ogTags := c.extractOGTags(doc)
 
 	// Store in cache
-	c.cache.Set(ctx, cacheKey, ogTags, c.ogTimeToLive)
+	c.cache.Set(cacheKey, ogTags, c.ogTimeToLive)
 
 	return ogTags, nil
 }
@@ -60,8 +59,8 @@ func (c *OGTagCache) generateCacheKey(target string, originalHost string) string
 }
 
 // checkCache checks if we have the tags cached and returns them if so
-func (c *OGTagCache) checkCache(ctx context.Context, cacheKey string) map[string]string {
-	if cachedTags, err := c.cache.Get(ctx, cacheKey); err == nil {
+func (c *OGTagCache) checkCache(cacheKey string) map[string]string {
+	if cachedTags, ok := c.cache.Get(cacheKey); ok {
 		slog.Debug("cache hit", "tags", cachedTags)
 		return cachedTags
 	}

internal/ogtags/cache_test.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 )
 
 func TestCacheReturnsDefault(t *testing.T) {
@@ -22,14 +21,14 @@ func TestCacheReturnsDefault(t *testing.T) {
 		TimeToLive:   time.Minute,
 		ConsiderHost: false,
 		Override:     want,
-	}, memory.New(t.Context()))
+	})
 
 	u, err := url.Parse("https://anubis.techaro.lol")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	result, err := cache.GetOGTags(t.Context(), u, "anubis.techaro.lol")
+	result, err := cache.GetOGTags(u, "anubis.techaro.lol")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -50,7 +49,7 @@ func TestCheckCache(t *testing.T) {
 		Enabled:      true,
 		TimeToLive:   time.Minute,
 		ConsiderHost: false,
-	}, memory.New(t.Context()))
+	})
 
 	// Set up test data
 	urlStr := "http://example.com/page"
@@ -61,16 +60,16 @@ func TestCheckCache(t *testing.T) {
 	cacheKey := cache.generateCacheKey(urlStr, "example.com")
 
 	// Test cache miss
-	tags := cache.checkCache(t.Context(), cacheKey)
+	tags := cache.checkCache(cacheKey)
 	if tags != nil {
 		t.Errorf("expected nil tags on cache miss, got %v", tags)
 	}
 
 	// Manually add to cache
-	cache.cache.Set(t.Context(), cacheKey, expectedTags, time.Minute)
+	cache.cache.Set(cacheKey, expectedTags, time.Minute)
 
 	// Test cache hit
-	tags = cache.checkCache(t.Context(), cacheKey)
+	tags = cache.checkCache(cacheKey)
 	if tags == nil {
 		t.Fatal("expected non-nil tags on cache hit, got nil")
 	}
@@ -113,7 +112,7 @@ func TestGetOGTags(t *testing.T) {
 		Enabled:      true,
 		TimeToLive:   time.Minute,
 		ConsiderHost: false,
-	}, memory.New(t.Context()))
+	})
 
 	// Parse the test server URL
 	parsedURL, err := url.Parse(ts.URL)
@@ -123,7 +122,7 @@ func TestGetOGTags(t *testing.T) {
 
 	// Test fetching OG tags from the test server
 	// Pass the host from the parsed test server URL
-	ogTags, err := cache.GetOGTags(t.Context(), parsedURL, parsedURL.Host)
+	ogTags, err := cache.GetOGTags(parsedURL, parsedURL.Host)
 	if err != nil {
 		t.Fatalf("failed to get OG tags: %v", err)
 	}
@@ -143,14 +142,14 @@ func TestGetOGTags(t *testing.T) {
 
 	// Test fetching OG tags from the cache
 	// Pass the host from the parsed test server URL
-	ogTags, err = cache.GetOGTags(t.Context(), parsedURL, parsedURL.Host)
+	ogTags, err = cache.GetOGTags(parsedURL, parsedURL.Host)
 	if err != nil {
 		t.Fatalf("failed to get OG tags from cache: %v", err)
 	}
 
 	// Test fetching OG tags from the cache (3rd time)
 	// Pass the host from the parsed test server URL
-	newOgTags, err := cache.GetOGTags(t.Context(), parsedURL, parsedURL.Host)
+	newOgTags, err := cache.GetOGTags(parsedURL, parsedURL.Host)
 	if err != nil {
 		t.Fatalf("failed to get OG tags from cache: %v", err)
 	}
@@ -264,10 +263,10 @@ func TestGetOGTagsWithHostConsideration(t *testing.T) {
 				Enabled:      true,
 				TimeToLive:   time.Minute,
 				ConsiderHost: tc.ogCacheConsiderHost,
-			}, memory.New(t.Context()))
+			})
 
 			for i, req := range tc.requests {
-				ogTags, err := cache.GetOGTags(t.Context(), parsedURL, req.host)
+				ogTags, err := cache.GetOGTags(parsedURL, req.host)
 				if err != nil {
 					t.Errorf("Request %d (host: %s): unexpected error: %v", i+1, req.host, err)
 					continue // Skip further checks for this request if error occurred

internal/ogtags/fetch.go

@@ -20,8 +20,8 @@ var (
 
 // fetchHTMLDocumentWithCache fetches the HTML document from the given URL string,
 // preserving the original host header.
-func (c *OGTagCache) fetchHTMLDocumentWithCache(ctx context.Context, urlStr string, originalHost string, cacheKey string) (*html.Node, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
+func (c *OGTagCache) fetchHTMLDocumentWithCache(urlStr string, originalHost string, cacheKey string) (*html.Node, error) {
+	req, err := http.NewRequestWithContext(context.Background(), "GET", urlStr, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create http request: %w", err)
 	}
@@ -41,7 +41,7 @@ func (c *OGTagCache) fetchHTMLDocumentWithCache(ctx context.Context, urlStr stri
 		var netErr net.Error
 		if errors.As(err, &netErr) && netErr.Timeout() {
 			slog.Debug("og: request timed out", "url", urlStr)
-			c.cache.Set(ctx, cacheKey, emptyMap, c.ogTimeToLive/2) // Cache empty result for half the TTL to not spam the server
+			c.cache.Set(cacheKey, emptyMap, c.ogTimeToLive/2) // Cache empty result for half the TTL to not spam the server
 		}
 		return nil, fmt.Errorf("http get failed: %w", err)
 	}
@@ -56,7 +56,7 @@ func (c *OGTagCache) fetchHTMLDocumentWithCache(ctx context.Context, urlStr stri
 
 	if resp.StatusCode != http.StatusOK {
 		slog.Debug("og: received non-OK status code", "url", urlStr, "status", resp.StatusCode)
-		c.cache.Set(ctx, cacheKey, emptyMap, c.ogTimeToLive) // Cache empty result for non-successful status codes
+		c.cache.Set(cacheKey, emptyMap, c.ogTimeToLive) // Cache empty result for non-successful status codes
 		return nil, fmt.Errorf("%w: page not found", ErrOgHandled)
 	}
 

internal/ogtags/fetch_test.go

@@ -1,7 +1,6 @@
 package ogtags
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -12,7 +11,6 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 	"golang.org/x/net/html"
 )
 
@@ -87,8 +85,8 @@ func TestFetchHTMLDocument(t *testing.T) {
 				Enabled:      true,
 				TimeToLive:   time.Minute,
 				ConsiderHost: false,
-			}, memory.New(t.Context()))
-			doc, err := cache.fetchHTMLDocument(t.Context(), ts.URL, "anything")
+			})
+			doc, err := cache.fetchHTMLDocument(ts.URL, "anything")
 
 			if tt.expectError {
 				if err == nil {
@@ -118,9 +116,9 @@ func TestFetchHTMLDocumentInvalidURL(t *testing.T) {
 		Enabled:      true,
 		TimeToLive:   time.Minute,
 		ConsiderHost: false,
-	}, memory.New(t.Context()))
+	})
 
-	doc, err := cache.fetchHTMLDocument(t.Context(), "http://invalid.url.that.doesnt.exist.example", "anything")
+	doc, err := cache.fetchHTMLDocument("http://invalid.url.that.doesnt.exist.example", "anything")
 
 	if err == nil {
 		t.Error("expected error for invalid URL, got nil")
@@ -132,7 +130,7 @@ func TestFetchHTMLDocumentInvalidURL(t *testing.T) {
 }
 
 // fetchHTMLDocument allows you to call fetchHTMLDocumentWithCache without a duplicate generateCacheKey call
-func (c *OGTagCache) fetchHTMLDocument(ctx context.Context, urlStr string, originalHost string) (*html.Node, error) {
+func (c *OGTagCache) fetchHTMLDocument(urlStr string, originalHost string) (*html.Node, error) {
 	cacheKey := c.generateCacheKey(urlStr, originalHost)
-	return c.fetchHTMLDocumentWithCache(ctx, urlStr, originalHost, cacheKey)
+	return c.fetchHTMLDocumentWithCache(urlStr, originalHost, cacheKey)
 }

internal/ogtags/integration_test.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 )
 
 func TestIntegrationGetOGTags(t *testing.T) {
@@ -111,7 +110,7 @@ func TestIntegrationGetOGTags(t *testing.T) {
 				Enabled:      true,
 				TimeToLive:   time.Minute,
 				ConsiderHost: false,
-			}, memory.New(t.Context()))
+			})
 
 			// Create URL for test
 			testURL, _ := url.Parse(ts.URL)
@@ -120,7 +119,7 @@ func TestIntegrationGetOGTags(t *testing.T) {
 
 			// Get OG tags
 			// Pass the host from the test URL
-			ogTags, err := cache.GetOGTags(t.Context(), testURL, testURL.Host)
+			ogTags, err := cache.GetOGTags(testURL, testURL.Host)
 
 			// Check error expectation
 			if tc.expectError {
@@ -148,7 +147,7 @@ func TestIntegrationGetOGTags(t *testing.T) {
 
 			// Test cache retrieval
 			// Pass the host from the test URL
-			cachedOGTags, err := cache.GetOGTags(t.Context(), testURL, testURL.Host)
+			cachedOGTags, err := cache.GetOGTags(testURL, testURL.Host)
 			if err != nil {
 				t.Fatalf("failed to get OG tags from cache: %v", err)
 			}

internal/ogtags/mem_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 	"golang.org/x/net/html"
 )
 
@@ -31,7 +30,7 @@ func BenchmarkGetTarget(b *testing.B) {
 
 	for _, tt := range tests {
 		b.Run(tt.name, func(b *testing.B) {
-			cache := NewOGTagCache(tt.target, config.OpenGraph{}, memory.New(b.Context()))
+			cache := NewOGTagCache(tt.target, config.OpenGraph{})
 			urls := make([]*url.URL, len(tt.paths))
 			for i, path := range tt.paths {
 				u, _ := url.Parse(path)
@@ -67,7 +66,7 @@ func BenchmarkExtractOGTags(b *testing.B) {
 		</head><body><div><p>Content</p></div></body></html>`,
 	}
 
-	cache := NewOGTagCache("http://example.com", config.OpenGraph{}, memory.New(b.Context()))
+	cache := NewOGTagCache("http://example.com", config.OpenGraph{})
 	docs := make([]*html.Node, len(htmlSamples))
 
 	for i, sample := range htmlSamples {
@@ -85,7 +84,7 @@ func BenchmarkExtractOGTags(b *testing.B) {
 
 // Memory usage test
 func TestMemoryUsage(t *testing.T) {
-	cache := NewOGTagCache("http://example.com", config.OpenGraph{}, memory.New(t.Context()))
+	cache := NewOGTagCache("http://example.com", config.OpenGraph{})
 
 	// Force GC and wait for it to complete
 	runtime.GC()

internal/ogtags/ogtags.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/TecharoHQ/anubis/decaymap"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store"
 )
 
 const (
@@ -22,7 +22,7 @@ const (
 )
 
 type OGTagCache struct {
-	cache     store.JSON[map[string]string]
+	cache     *decaymap.Impl[string, map[string]string]
 	targetURL *url.URL
 	client    *http.Client
 
@@ -36,7 +36,7 @@ type OGTagCache struct {
 	ogOverride          map[string]string
 }
 
-func NewOGTagCache(target string, conf config.OpenGraph, backend store.Interface) *OGTagCache {
+func NewOGTagCache(target string, conf config.OpenGraph) *OGTagCache {
 	// Predefined approved tags and prefixes
 	defaultApprovedTags := []string{"description", "keywords", "author"}
 	defaultApprovedPrefixes := []string{"og:", "twitter:", "fediverse:"}
@@ -77,10 +77,7 @@ func NewOGTagCache(target string, conf config.OpenGraph, backend store.Interface
 	}
 
 	return &OGTagCache{
-		cache: store.JSON[map[string]string]{
-			Underlying: backend,
-			Prefix:     "ogtags:",
-		},
+		cache:               decaymap.New[string, map[string]string](),
 		targetURL:           parsedTargetURL,
 		ogPassthrough:       conf.Enabled,
 		ogTimeToLive:        conf.TimeToLive,
@@ -127,3 +124,9 @@ func (c *OGTagCache) getTarget(u *url.URL) string {
 
 	return sb.String()
 }
+
+func (c *OGTagCache) Cleanup() {
+	if c.cache != nil {
+		c.cache.Cleanup()
+	}
+}

internal/ogtags/ogtags_fuzz_test.go

@@ -1,14 +1,12 @@
 package ogtags
 
 import (
-	"context"
 	"net/url"
 	"strings"
 	"testing"
 	"unicode/utf8"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 	"golang.org/x/net/html"
 )
 
@@ -48,7 +46,7 @@ func FuzzGetTarget(f *testing.F) {
 		}
 
 		// Create cache - should not panic
-		cache := NewOGTagCache(target, config.OpenGraph{}, memory.New(context.Background()))
+		cache := NewOGTagCache(target, config.OpenGraph{})
 
 		// Create URL
 		u := &url.URL{
@@ -132,7 +130,7 @@ func FuzzExtractOGTags(f *testing.F) {
 			return
 		}
 
-		cache := NewOGTagCache("http://example.com", config.OpenGraph{}, memory.New(context.Background()))
+		cache := NewOGTagCache("http://example.com", config.OpenGraph{})
 
 		// Should not panic
 		tags := cache.extractOGTags(doc)
@@ -188,7 +186,7 @@ func FuzzGetTargetRoundTrip(f *testing.F) {
 			t.Skip()
 		}
 
-		cache := NewOGTagCache(target, config.OpenGraph{}, memory.New(context.Background()))
+		cache := NewOGTagCache(target, config.OpenGraph{})
 		u := &url.URL{Path: path, RawQuery: query}
 
 		result := cache.getTarget(u)
@@ -245,7 +243,7 @@ func FuzzExtractMetaTagInfo(f *testing.F) {
 			},
 		}
 
-		cache := NewOGTagCache("http://example.com", config.OpenGraph{}, memory.New(context.Background()))
+		cache := NewOGTagCache("http://example.com", config.OpenGraph{})
 
 		// Should not panic
 		property, content := cache.extractMetaTagInfo(node)
@@ -298,7 +296,7 @@ func BenchmarkFuzzedGetTarget(b *testing.B) {
 
 	for _, input := range inputs {
 		b.Run(input.name, func(b *testing.B) {
-			cache := NewOGTagCache(input.target, config.OpenGraph{}, memory.New(context.Background()))
+			cache := NewOGTagCache(input.target, config.OpenGraph{})
 			u := &url.URL{Path: input.path, RawQuery: input.query}
 
 			b.ResetTimer()

internal/ogtags/ogtags_test.go

@@ -15,7 +15,6 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 )
 
 func TestNewOGTagCache(t *testing.T) {
@@ -45,7 +44,7 @@ func TestNewOGTagCache(t *testing.T) {
 				Enabled:      tt.ogPassthrough,
 				TimeToLive:   tt.ogTimeToLive,
 				ConsiderHost: false,
-			}, memory.New(t.Context()))
+			})
 
 			if cache == nil {
 				t.Fatal("expected non-nil cache, got nil")
@@ -85,7 +84,7 @@ func TestNewOGTagCache_UnixSocket(t *testing.T) {
 		Enabled:      true,
 		TimeToLive:   5 * time.Minute,
 		ConsiderHost: false,
-	}, memory.New(t.Context()))
+	})
 
 	if cache == nil {
 		t.Fatal("expected non-nil cache, got nil")
@@ -170,7 +169,7 @@ func TestGetTarget(t *testing.T) {
 				Enabled:      true,
 				TimeToLive:   time.Minute,
 				ConsiderHost: false,
-			}, memory.New(t.Context()))
+			})
 
 			u := &url.URL{
 				Path:     tt.path,
@@ -243,14 +242,14 @@ func TestIntegrationGetOGTags_UnixSocket(t *testing.T) {
 		Enabled:      true,
 		TimeToLive:   time.Minute,
 		ConsiderHost: false,
-	}, memory.New(t.Context()))
+	})
 
 	// Create a dummy URL for the request (path and query matter)
 	testReqURL, _ := url.Parse("/some/page?query=1")
 
 	// Get OG tags
 	// Pass an empty string for host, as it's irrelevant for unix sockets
-	ogTags, err := cache.GetOGTags(t.Context(), testReqURL, "")
+	ogTags, err := cache.GetOGTags(testReqURL, "")
 
 	if err != nil {
 		t.Fatalf("GetOGTags failed for unix socket: %v", err)
@@ -266,7 +265,7 @@ func TestIntegrationGetOGTags_UnixSocket(t *testing.T) {
 
 	// Test cache retrieval (should hit cache)
 	// Pass an empty string for host
-	cachedTags, err := cache.GetOGTags(t.Context(), testReqURL, "")
+	cachedTags, err := cache.GetOGTags(testReqURL, "")
 	if err != nil {
 		t.Fatalf("GetOGTags (cache hit) failed for unix socket: %v", err)
 	}

internal/ogtags/parse_test.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis/lib/policy/config"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
 	"golang.org/x/net/html"
 )
 
@@ -18,7 +17,7 @@ func TestExtractOGTags(t *testing.T) {
 		Enabled:      false,
 		ConsiderHost: false,
 		TimeToLive:   time.Minute,
-	}, memory.New(t.Context()))
+	})
 	// Manually set approved tags/prefixes based on the user request for clarity
 	testCache.approvedTags = []string{"description"}
 	testCache.approvedPrefixes = []string{"og:"}
@@ -199,7 +198,7 @@ func TestExtractMetaTagInfo(t *testing.T) {
 		Enabled:      false,
 		ConsiderHost: false,
 		TimeToLive:   time.Minute,
-	}, memory.New(t.Context()))
+	})
 	testCache.approvedTags = []string{"description"}
 	testCache.approvedPrefixes = []string{"og:"}
 

lib/anubis.go

@@ -70,6 +70,7 @@ type Server struct {
 	next        http.Handler
 	mux         *http.ServeMux
 	policy      *policy.ParsedConfig
+	DNSBLCache  *decaymap.Impl[string, dnsbl.DroneBLResponse]
 	OGTags      *ogtags.OGTagCache
 	ed25519Priv ed25519.PrivateKey
 	hs512Secret []byte
@@ -278,16 +279,15 @@ func (s *Server) checkRules(w http.ResponseWriter, r *http.Request, cr policy.Ch
 }
 
 func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, lg *slog.Logger) bool {
-	db := &store.JSON[dnsbl.DroneBLResponse]{Underlying: s.store, Prefix: "dronebl:"}
 	if s.policy.DNSBL && ip != "" {
-		resp, err := db.Get(r.Context(), ip)
-		if err != nil {
+		resp, ok := s.DNSBLCache.Get(ip)
+		if !ok {
 			lg.Debug("looking up ip in dnsbl")
 			resp, err := dnsbl.Lookup(ip)
 			if err != nil {
 				lg.Error("can't look up ip in dnsbl", "err", err)
 			}
-			db.Set(r.Context(), ip, resp, 24*time.Hour)
+			s.DNSBLCache.Set(ip, resp, 24*time.Hour)
 			droneBLHits.WithLabelValues(resp.String()).Inc()
 		}
 
@@ -551,3 +551,8 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
 		Rules: &checker.List{},
 	}, nil
 }
+
+func (s *Server) CleanupDecayMap() {
+	s.DNSBLCache.Cleanup()
+	s.OGTags.Cleanup()
+}

lib/config.go

@@ -15,7 +15,9 @@ import (
 
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/data"
+	"github.com/TecharoHQ/anubis/decaymap"
 	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal/ogtags"
 	"github.com/TecharoHQ/anubis/lib/challenge"
 	"github.com/TecharoHQ/anubis/lib/localization"
@@ -106,7 +108,8 @@ func New(opts Options) (*Server, error) {
 		hs512Secret: opts.HS512Secret,
 		policy:      opts.Policy,
 		opts:        opts,
-		OGTags:      ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph, opts.Policy.Store),
+		DNSBLCache:  decaymap.New[string, dnsbl.DroneBLResponse](),
+		OGTags:      ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph),
 		store:       opts.Policy.Store,
 	}
 

lib/http.go

@@ -138,7 +138,7 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
 	var ogTags map[string]string = nil
 	if s.opts.OpenGraph.Enabled {
 		var err error
-		ogTags, err = s.OGTags.GetOGTags(r.Context(), r.URL, r.Host)
+		ogTags, err = s.OGTags.GetOGTags(r.URL, r.Host)
 		if err != nil {
 			lg.Error("failed to get OG tags", "err", err)
 		}

lib/localization/locales/de.json

@@ -4,17 +4,17 @@
   "protected_by": "Geschützt durch",
   "made_with": "Mit ❤️ gemacht in 🇨🇦",
   "mascot_design": "Maskottchen erstellt von",
-  "ai_companies_explanation": "Diese Seite wird angezeigt, da der Betreiber der Webseite Anubis eingerichtet hat, um sie vor aggressiven KI-Website-Scrapern zu schützen. Diese können Ausfälle der Webseite verursachen, wodurch die Webseite für niemanden erreichbar ist.",
-  "anubis_compromise": "Anubis stellt einen Kompromiss dar. Es verwendet eine Proof-of-Work-Methode nach Hashcash, die ursprünglich zur Bekämpfung von E-Mail-Spam entwickelt wurde. Die Idee dahinter ist, dass ein legitimer Besucher die Webseite mit einer vernachlässigbaren Verzögerung erreichen kann. Massenhaftes Scraping wird dadurch jedoch aufwändig und teuer.",
-  "hack_purpose": "Man könnte dies als eine Lösung bezeichnen, die einem etwas Zeit für Fingerprinting und dem Identifizieren von Headless-Browsern verschafft. Besucher, die mit sehr hoher Wahrscheinlichkeit legitim sind, bekommen diese Seite nicht zu sehen.",
-  "jshelter_note": "Anubis benötigt moderne JavaScript-Features, welche von Plugins wie JShelter deaktiviert werden. Bitte deaktiviere JShelter oder ähnliche Plugins für diese Domain.",
-  "version_info": "Diese Webseite läuft mit der Anubis-Version",
-  "try_again": "Erneut versuchen",
+  "ai_companies_explanation": "Diese Seite wird angezeigt, da der Betreiber der Webseite Anubis eingerichtet hat, um sie vor aggressiven KI-Website-Scrapern zu schützen. Diese können Ausfälle der Webseite verursachen, wodurch die Webseite für jeden nicht erreichbar ist.",
+  "anubis_compromise": "Anubis ist eine Art Kompromiss. Es verwendet die sogenannte Proof-of-Work Methode nach Hashcash, ein Mechanismus, der ursprünglich zur E-Mail-Spam-Bekämpfung entwickelt wurde. Die Idee dahinter ist, dass ein einziger User nur eine kleine Verzögerung hat, auf die Webseite zu gelangen; bei Scrapern kann das allerdings große Auswirkungen haben.",
+  "hack_purpose": "Man könnte dies als eine Lösung bezeichnen, die gut genug ist, einem etwas Zeit zu verschaffen für Fingerprinting und dem Identifizieren von Headless Browsern, sodass im besten Fall normale User diese Seite garnicht erst zu sehen bekommen.",
+  "jshelter_note": "Anubis benötigt moderne JavaScript-Features, welche von Plugins wie zB JShelter deaktiviert werden. Bitte deaktiviere also JShelter oder ähnliche Plugins für diese Domain.",
+  "version_info": "Diese Webseite läuft mit Anubis version",
+  "try_again": "Nochmal probieren",
   "go_home": "Zur Hauptseite",
-  "contact_webmaster": "oder wenn Du glaubst, dass es sich hierbei um einen Fehler handelt, kontaktiere bitte den Administrator der Webseite unter",
-  "connection_security": "Bitte warte einen Moment, während wir sicherstellen, dass eine sichere Verbindung verwendet wird.",
-  "javascript_required": "Du musst JavaScript aktivieren, um diese Prüfung durchführen zu können. Dies ist notwendig, da KI-Unternehmen den sozialen Vertrag bezüglich des Hostings von Webseiten gebrochen haben. Eine Lösung ohne JavaScript ist in Entwicklung.",
-  "benchmark_requires_js": "Für die Nutzung des Benchmark-Tools muss JavaScript aktiviert werden.",
+  "contact_webmaster": "oder wenn es sich hier um einen Fehler handelt, kontaktiere bitte den Administrator der Webseite unter",
+  "connection_security": "Bitte warte einen Moment während wir sicherstellen, dass eine sichere Verbindung verwendet wird.",
+  "javascript_required": "Es muss leider JavaScript aktiviert werden, um den Check durchführen zu können. Dies ist leider notwendig weil Firmen im KI-Sektor die sozialen Verhältnisse geändert haben, wie Website-Hosting funktioniert. Eine Lösung ohne JavaScript ist in Entwicklung.",
+  "benchmark_requires_js": "Das Benchmark-Tool benötigt das Aktivieren von JavaScript.",
   "difficulty": "Schwierigkeit:",
   "algorithm": "Algorithmus:",
   "compare": "Vergleich:",
@@ -24,40 +24,40 @@
   "iters_a": "Iterationen A",
   "time_b": "Zeit B",
   "iters_b": "Iterationen B",
-  "static_check_endpoint": "Dies ist nur ein Endpunkt, der von einem Reverse-Proxy geprüft werden kann.",
+  "static_check_endpoint": "Dies ist nur ein Check-Endpunkt, der von beispielsweise einem Reverse-Proxy geprüft werden kann.",
   "authorization_required": "Zugriffserlaubnis benötigt",
-  "cookies_disabled": "Cookies sind in deinem Browser deaktiviert. Anubis benötigt Cookies, um sicherzustellen, dass es sich hierbei um einen legitimen Zugriff handelt. Bitte aktiviere Cookies für diese Domain.",
+  "cookies_disabled": "Cookies sind in Ihrem Browser deaktiviert. Anubis benötigt Cookies um sicherzustellen, dass es sich hierbei um einen validen Zugriff handelt. Bitte aktiviere Cookies für diese Domain.",
   "access_denied": "Zugriff verweigert: Fehlercode",
   "dronebl_entry": "Eintrag in DroneBL",
   "see_dronebl_lookup": "anzeigen",
-  "internal_server_error": "Interner Server-Fehler: Der Administrator hat Anubis fehlerhaft konfiguriert. Bitte kontaktiere den Administrator und bitte ihn, die Logs zu prüfen.",
+  "internal_server_error": "Interner Server Error: Misskonfiguration von Anubis. Bitte kontatkiere den Administrator damit dieser die Logs prüfen kann.",
   "invalid_redirect": "Ungültige Weiterleitung",
   "redirect_not_parseable": "URL der Weiterleitung kann nicht verarbeitet werden",
   "redirect_domain_not_allowed": "Domain der Weiterleitung nicht erlaubt",
-  "failed_to_sign_jwt": "JWT konnte nicht signiert werden",
-  "invalid_invocation": "Ungültiger Aufruf von MakeChallenge",
-  "client_error_browser": "Client-Fehler: Bitte stelle sicher, dass dein Browser aktuell ist und versuche es später erneut.",
-  "oh_noes": "Oh nein!",
+  "failed_to_sign_jwt": "Signierung des JWT fehlgeschlagen",
+  "invalid_invocation": "Aufrufen von MakeChallenge ungültig",
+  "client_error_browser": "Client Error: Bitte stelle sicher, dass der Browser aktuell ist und probiere es später erneut.",
+  "oh_noes": "Vermaledeit!",
   "benchmarking_anubis": "Benchmark wird durchgeführt!",
-  "you_are_not_a_bot": "Du bist kein Bot!",
-  "making_sure_not_bot": "Dein Browser wird geprüft!",
+  "you_are_not_a_bot": "Sie sind kein Bot!",
+  "making_sure_not_bot": "Ihr Browser wird geprüft!",
   "celphase": "CELPHASE",
-  "js_web_crypto_error": "Dein Browser hat kein funktionierendes web.crypto Element. Wird eine sichere Verbindung verwendet?",
-  "js_web_workers_error": "Dein Browser unterstützt keine Web-Worker (Anubis verwendet diese, damit der Browser nicht einfriert). Ist ein Plugin wie JShelter installiert?",
-  "js_cookies_error": "Dein Browser speichert keine Cookies. Anubis verwendet Cookies, um nach bestandener Prüfung ein signiertes Token abzulegen. Bitte aktiviere Cookies für diese Domain. Die Cookie-Namen von Anubis könnten sich jederzeit ändern. Cookie-Namen und die gespeicherten Werte sind kein Teil der öffentlichen API.",
+  "js_web_crypto_error": "Ihr Browser hat leider kein funktionierendes web.crypto Element. Wird eine sichere Verbindung verwendet?",
+  "js_web_workers_error": "Ihr Browser unterstützt keine Web-Worker (Anubis verwendet diese, damit der Browser nicht unresponsive wird). Ist eventuell ein Plugin wie zB JShelter installiert?",
+  "js_cookies_error": "Ihr Browser speichert keine Cookies. Anubis verwendet Cookies um ein gültiges Token zu speichern damit es wissen kann, welche Browser bereits geprüft wurden. Bitte aktiviere Cookies für diese Domain. Die Cookie-Namen von Anubis könnten sich jederzeit ändern. Cookie-Namen sind kein Teil der öffentlichen API.",
   "js_context_not_secure": "Diese Verbindung ist nicht sicher!",
-  "js_context_not_secure_msg": "Bitte versuche, dich via HTTPS zu verbinden oder weise den Administrator darauf hin, HTTPS einzurichten. Mehr Informationen unter: <a href=\"https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts#when_is_a_context_considered_secure\">MDN</a>.",
+  "js_context_not_secure_msg": "Bitte probiere, dich via HTTPS zu verbinden und lass den Webseiten-Administrator wissen, sauber HTTPS einzurichten. Mehr Informationen unter: <a href=\"https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts#when_is_a_context_considered_secure\">MDN</a>.",
   "js_calculating": "Berechnung wird durchgeführt...",
   "js_missing_feature": "Fehlendes Feature",
-  "js_challenge_error": "Prüfung fehlgeschlagen!",
-  "js_challenge_error_msg": "Der Prüf-Algorithmus konnte nicht geladen werden. Bitte lade diese Seite erneut.",
+  "js_challenge_error": "Fehler während des Checks!",
+  "js_challenge_error_msg": "Der Check-Algorithmus konnte nicht geladen werden. Bitte lade diese Seite erneut.",
   "js_calculating_difficulty": "Berechnung wird durchgeführt...<br/>Schwierigkeit:",
   "js_speed": "Geschwindigkeit:",
-  "js_verification_longer": "Die Prüfung benötigt länger als erwartet. Bitte bleibe auf der Seite und lade diese nicht neu.",
+  "js_verification_longer": "Der Check benötigt länger als erwartet. Bitte bleibe auf der Seite.",
   "js_success": "Erfolgreich!",
   "js_done_took": "Fertig! Dauer:",
   "js_iterations": "Iterationen",
   "js_finished_reading": "Fertig gelesen, weiter zur Seite →",
-  "js_calculation_error": "Berechnung fehlgeschlagen!",
-  "js_calculation_error_msg": "Fehler bei der Berechnung der Prüfung:"
+  "js_calculation_error": "Fehler bei der Berechnung!",
+  "js_calculation_error_msg": "Fehler bei der Berechnung des Checks:"
 }

lib/localization/locales/manifest.json

@@ -1,3 +1,3 @@
 {
-  "supportedLanguages": ["en", "fr", "es", "pt-BR", "de", "tr", "zh-TW"]
+  "supportedLanguages": ["en", "fr", "es", "pt-BR", "de", "tr"]
 }

lib/localization/locales/zh-TW.json

@@ -1,63 +0,0 @@
-{
-  "loading": "載入中...",
-  "why_am_i_seeing": "為什麼我看到這個？",
-  "protected_by": "保護由",
-  "made_with": "在 🇨🇦 用 ❤️ 製作",
-  "mascot_design": "吉祥物由",
-  "ai_companies_explanation": "您會看到這個畫面，是因為網站管理員啟用了 Anubis 來保護伺服器，避免 AI 公司大量爬取網站內容。這類行為會導致網站當機，讓所有使用者都無法正常存取資源。",
-  "anubis_compromise": "Anubis 是一種折衷做法。它採用了類似 Hashcash 的工作量證明機制（Proof-of-Work），該機制最初是為了減少垃圾郵件而提出。其核心概念是：對個別使用者而言，額外的運算負擔可以忽略，但對大規模爬蟲來說，累積起來的成本將大幅增加，從而讓爬取行為變得更困難。",
-  "hack_purpose": "本質上，這是一種權宜的解法，目的是提供一個「夠用」的暫時性防護措施，好讓開發者有更多時間針對無頭瀏覽器進行指紋特徵辨識（例如：分析其字型渲染方式），以便未來不再需要對可能為合法使用者的訪客展示工作量證明頁面。",
-  "jshelter_note": "請注意，Anubis 需要使用現代 JavaScript 功能，而像 JShelter 這類外掛可能會阻擋這些功能。請為此網域停用 JShelter 或類似的插件。",
-  "version_info": "這個網站正在運行 Anubis 版本",
-  "try_again": "再試一次",
-  "go_home": "回首頁",
-  "contact_webmaster": "或者您覺得您不應該被封鎖，請聯絡站點管理員於",
-  "connection_security": "請稍等，我們需要在繼續之前檢閱您的連線安全性。",
-  "javascript_required": "很遺憾，您必須啟用 JavaScript 才能通過這項驗證。這是因為 AI 公司已經改變了網站託管的社會契約，因此我們必須採取這樣的保護機制。無需 JavaScript 的解法仍在開發中。",
-  "benchmark_requires_js": "執行基準測試工具需要啟用 JavaScript。",
-  "difficulty": "難度：",
-  "algorithm": "演算法：",
-  "compare": "比較：",
-  "time": "時間",
-  "iters": "迭代",
-  "time_a": "時間 A",
-  "iters_a": "迭代 A",
-  "time_b": "時間 B",
-  "iters_b": "迭代 B",
-  "static_check_endpoint": "這是提供給您的反向代理伺服器使用的檢查端點。",
-  "authorization_required": "需要認證",
-  "cookies_disabled": "您的瀏覽器目前已停用 Cookie，為了確認您是合法使用者，Anubis 需要啟用 Cookie。 請為此網域啟用 Cookie",
-  "access_denied": "拒絕存取：錯誤代碼",
-  "dronebl_entry": "DroneBL 回報了一筆紀錄",
-  "see_dronebl_lookup": "見",
-  "internal_server_error": "內部伺服器錯誤：管理員錯誤地配置了 Anubis。 請聯絡管理員要求他們檢閱日誌",
-  "invalid_redirect": "無效的重新導向",
-  "redirect_not_parseable": "重新導向 URL 無法解析",
-  "redirect_domain_not_allowed": "重新導向的網域並不允許",
-  "failed_to_sign_jwt": "簽署 JWT 失敗",
-  "invalid_invocation": "無效的 MakeChallenge 呼叫",
-  "client_error_browser": "客戶端錯誤：請確保您的瀏覽器是最新版本並稍候再試。",
-  "oh_noes": "哎呀糟糕了！",
-  "benchmarking_anubis": "正在進行 Anubis 效能測試！",
-  "you_are_not_a_bot": "你不是機器人！",
-  "making_sure_not_bot": "正在確認你是不是機器人！",
-  "celphase": "CELPHASE 設計",
-  "js_web_crypto_error": "您的瀏覽器無法正常使用 web.crypto 元件。您是否透過安全連線（HTTPS）檢視此網站？",
-  "js_web_workers_error": "您的瀏覽器並不支援 Web workers （Anubis 使用這個來避免凍結您的瀏覽器 ）您有安裝像是 JShelter 之類的插件嗎？",
-  "js_cookies_error": "您的瀏覽器無法儲存 Cookie。 Anubis 會使用 Cookie 儲存簽署的憑證，以判斷使用者是否已通過驗證。請為此網域啟用 Cookie 儲存功能。 請注意，Anubis 儲存的 Cookie 名稱可能會變動，且其名稱與內容不屬於公開 API 的一部分。",
-  "js_context_not_secure": "您的內容並不安全",
-  "js_context_not_secure_msg": "請嘗試使用 HTTPS 連線，或聯繫網站管理員設定 HTTPS。更多資訊請參見 <a href=\"https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts#when_is_a_context_considered_secure\">MDN</a>。",
-  "js_calculating": "計算中...",
-  "js_missing_feature": "缺少功能",
-  "js_challenge_error": "挑戰錯誤！",
-  "js_challenge_error_msg": "解決檢查演算法失敗。 您可能會想要重整頁面。",
-  "js_calculating_difficulty": "計算中...<br/>難度：",
-  "js_speed": "速度：",
-  "js_verification_longer": "驗證所花的時間高於預期。 請不要重整頁面。",
-  "js_success": "成功！",
-  "js_done_took": "完成！ 花費",
-  "js_iterations": "迭代",
-  "js_finished_reading": "我讀完了，繼續 →",
-  "js_calculation_error": "計算錯誤！",
-  "js_calculation_error_msg": "計算挑戰失敗："
-}

lib/localization/localization_test.go

@@ -43,14 +43,6 @@ func TestLocalizationService(t *testing.T) {
 		}
 	})
 
-	t.Run("Traditional Chinese localization", func(t *testing.T) {
-		localizer := service.GetLocalizer("zh-TW")
-		result := localizer.MustLocalize(&i18n.LocalizeConfig{MessageID: "loading"})
-		if result != "載入中..." {
-			t.Errorf("Expected '載入中...', got '%s'", result)
-		}
-	})
-
 	t.Run("All required keys exist in English", func(t *testing.T) {
 		localizer := service.GetLocalizer("en")
 		requiredKeys := []string{
@@ -95,21 +87,6 @@ func TestLocalizationService(t *testing.T) {
 			}
 		}
 	})
-
-	t.Run("All required keys exist in Traditional Chinese", func(t *testing.T) {
-		localizer := service.GetLocalizer("zh-TW")
-		requiredKeys := []string{
-			"loading", "why_am_i_seeing", "protected_by", "made_with",
-			"mascot_design", "try_again", "go_home", "javascript_required",
-		}
-
-		for _, key := range requiredKeys {
-			result := localizer.MustLocalize(&i18n.LocalizeConfig{MessageID: key})
-			if result == "" {
-				t.Errorf("Key '%s' returned empty string", key)
-			}
-		}
-	})
 }
 
 type manifest struct {

lib/policy/celchecker.go

@@ -73,12 +73,6 @@ func (cr *CELRequest) ResolveName(name string) (any, bool) {
 		return expressions.URLValues{Values: cr.URL.Query()}, true
 	case "headers":
 		return expressions.HTTPHeaders{Header: cr.Header}, true
-	case "load_1m":
-		return expressions.Load1(), true
-	case "load_5m":
-		return expressions.Load5(), true
-	case "load_15m":
-		return expressions.Load15(), true
 	default:
 		return nil, false
 	}

lib/policy/expressions/environment.go

@@ -23,9 +23,6 @@ func BotEnvironment() (*cel.Env, error) {
 		cel.Variable("path", cel.StringType),
 		cel.Variable("query", cel.MapType(cel.StringType, cel.StringType)),
 		cel.Variable("headers", cel.MapType(cel.StringType, cel.StringType)),
-		cel.Variable("load_1m", cel.DoubleType),
-		cel.Variable("load_5m", cel.DoubleType),
-		cel.Variable("load_15m", cel.DoubleType),
 	)
 }
 

lib/policy/expressions/loadavg.go

@@ -1,69 +0,0 @@
-package expressions
-
-import (
-	"context"
-	"log/slog"
-	"sync"
-	"time"
-
-	"github.com/shirou/gopsutil/v4/load"
-)
-
-type loadAvg struct {
-	lock sync.RWMutex
-	data *load.AvgStat
-}
-
-func (l *loadAvg) updateThread(ctx context.Context) {
-	ticker := time.NewTicker(15 * time.Second)
-	defer ticker.Stop()
-
-	l.update()
-
-	for {
-		select {
-		case <-ticker.C:
-			l.update()
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (l *loadAvg) update() {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	var err error
-	l.data, err = load.Avg()
-	if err != nil {
-		slog.Debug("can't get load average", "err", err)
-	}
-}
-
-var (
-	globalLoadAvg *loadAvg
-)
-
-func init() {
-	globalLoadAvg = &loadAvg{}
-	go globalLoadAvg.updateThread(context.Background())
-}
-
-func Load1() float64 {
-	globalLoadAvg.lock.RLock()
-	defer globalLoadAvg.lock.RUnlock()
-	return globalLoadAvg.data.Load1
-}
-
-func Load5() float64 {
-	globalLoadAvg.lock.RLock()
-	defer globalLoadAvg.lock.RUnlock()
-	return globalLoadAvg.data.Load5
-}
-
-func Load15() float64 {
-	globalLoadAvg.lock.RLock()
-	defer globalLoadAvg.lock.RUnlock()
-	return globalLoadAvg.data.Load15
-}

lib/store/bbolt/bbolt.go

@@ -151,7 +151,7 @@ func (s *Store) cleanup(ctx context.Context) error {
 }
 
 func (s *Store) cleanupThread(ctx context.Context) {
-	t := time.NewTicker(time.Hour)
+	t := time.NewTicker(5 * time.Minute)
 	defer t.Stop()
 
 	for {

lib/store/interface.go

@@ -43,22 +43,13 @@ func z[T any]() T { return *new(T) }
 
 type JSON[T any] struct {
 	Underlying Interface
-	Prefix     string
 }
 
 func (j *JSON[T]) Delete(ctx context.Context, key string) error {
-	if j.Prefix != "" {
-		key = j.Prefix + key
-	}
-
 	return j.Underlying.Delete(ctx, key)
 }
 
 func (j *JSON[T]) Get(ctx context.Context, key string) (T, error) {
-	if j.Prefix != "" {
-		key = j.Prefix + key
-	}
-
 	data, err := j.Underlying.Get(ctx, key)
 	if err != nil {
 		return z[T](), err
@@ -73,10 +64,6 @@ func (j *JSON[T]) Get(ctx context.Context, key string) (T, error) {
 }
 
 func (j *JSON[T]) Set(ctx context.Context, key string, value T, expiry time.Duration) error {
-	if j.Prefix != "" {
-		key = j.Prefix + key
-	}
-
 	data, err := json.Marshal(value)
 	if err != nil {
 		return fmt.Errorf("%w: %w", ErrCantEncode, err)

lib/store/json_test.go

@@ -1,50 +0,0 @@
-package store_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/TecharoHQ/anubis/lib/store"
-	"github.com/TecharoHQ/anubis/lib/store/memory"
-)
-
-func TestJSON(t *testing.T) {
-	type data struct {
-		ID string `json:"id"`
-	}
-
-	st := memory.New(t.Context())
-	db := store.JSON[data]{
-		Underlying: st,
-		Prefix:     "foo:",
-	}
-
-	if err := db.Set(t.Context(), "test", data{ID: t.Name()}, time.Minute); err != nil {
-		t.Fatal(err)
-	}
-
-	got, err := db.Get(t.Context(), "test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if got.ID != t.Name() {
-		t.Fatalf("got wrong data for key \"test\", wanted %q but got: %q", t.Name(), got.ID)
-	}
-
-	if err := db.Delete(t.Context(), "test"); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, err := db.Get(t.Context(), "test"); err == nil {
-		t.Fatal("wanted invalid get to fail, it did not")
-	}
-
-	if err := st.Set(t.Context(), "foo:test", []byte("}"), time.Minute); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, err := db.Get(t.Context(), "test"); err == nil {
-		t.Fatal("wanted invalid get to fail, it did not")
-	}
-}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@techaro/anubis",
-  "version": "1.21.0-pre1",
+  "version": "1.20.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@techaro/anubis",
-      "version": "1.21.0-pre1",
+      "version": "1.20.0",
       "license": "ISC",
       "devDependencies": {
         "cssnano": "^7.0.7",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@techaro/anubis",
-  "version": "1.21.0-pre1",
+  "version": "1.20.0",
   "description": "",
   "main": "index.js",
   "scripts": {