feat(anubis): enable pprof endpoints on the metrics server

Closes: #1366
Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/allow.txt

@@ -31,3 +31,6 @@ Stargate
 FFXIV
 uvensys
 de
+resourced
+envoyproxy
+unipromos

cmd/anubis/main.go

@@ -17,6 +17,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/httputil"
+	"net/http/pprof"
 	"net/url"
 	"os"
 	"os/signal"
@@ -522,6 +523,11 @@ func metricsServer(ctx context.Context, lg slog.Logger, done func()) {
 	defer done()
 
 	mux := http.NewServeMux()
+	mux.HandleFunc("GET /debug/pprof/", pprof.Index)
+	mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
 	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		st, ok := internal.GetHealth("anubis")

data/crawlers/_allow-good.yaml

@@ -8,4 +8,5 @@
 - import: (data)/crawlers/marginalia.yaml
 - import: (data)/crawlers/mojeekbot.yaml
 - import: (data)/crawlers/commoncrawl.yaml
+- import: (data)/crawlers/wikimedia-citoid.yaml
 - import: (data)/crawlers/yandexbot.yaml

data/crawlers/wikimedia-citoid.yaml

@@ -0,0 +1,18 @@
+# Wikimedia Foundation citation services
+# https://www.mediawiki.org/wiki/Citoid
+
+- name: wikimedia-citoid
+  user_agent_regex: "Citoid/WMF"
+  action: ALLOW
+  remote_addresses: [
+    "208.80.152.0/22",
+    "2620:0:860::/46",
+  ]
+
+- name: wikimedia-zotero-translation-server
+  user_agent_regex: "ZoteroTranslationServer/WMF"
+  action: ALLOW
+  remote_addresses: [
+    "208.80.152.0/22",
+    "2620:0:860::/46",
+  ]

docs/docs/CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Expose [pprof endpoints](https://pkg.go.dev/net/http/pprof) on the metrics listener to enable profiling Anubis in production.
 - fix: prevent nil pointer panic in challenge validation when threshold rules match during PassChallenge (#1463)
 - Instruct reverse proxies to not cache error pages.
 - Fixed mixed tab/space indentation in Caddy documentation code block

docs/docs/user/frequently-asked-questions.mdx

@@ -22,3 +22,13 @@ If you use a browser extension such as [JShelter](https://jshelter.org/), you wi
 ## Does Anubis mine Bitcoin?
 
 No. Anubis does not mine Bitcoin or any other cryptocurrency.
+
+## I disabled Just-in-time compilation in my browser. Why is Anubis slow?
+
+Anubis proof-of-work checks run an open source JavaScript program in your browser. These checks do a lot of complicated math and aim to be done quickly, so the execution speed depends on [Just-in-time (JIT) compilation](https://en.wikipedia.org/wiki/Just-in-time_compilation). JIT compiles JavaScript from the Internet into native machine code at runtime. The code produced by the JIT engine is almost as good as if it was written in a native programming language and compiled for your computer in particular. Without JIT, all JavaScript programs on every website you visit run through a slow interpreter.
+
+This interpreter is much slower than native code because it has to translate each low level JavaScript operation into many dozens of calls to execute. This means that using the interpreter incurs a massive performance hit by its very nature; it takes longer to add numbers than if the CPU just added the numbers directly.
+
+Some users choose to disable JIT as a hardening measure against theoretical browser exploits. This is a reasonable choice if you face targeted attacks from well-resourced adversaries (such as nation-state actors), but it comes with real performance costs.
+
+If you've disabled JIT and find Anubis checks slow, re-enabling JIT is the fix. There is no way for Anubis to work around this on our end.

lib/localization/locales/ja.json

@@ -9,7 +9,7 @@
   "anubis_compromise": "Anubisは妥協策です。AnubisはHashcashのようなProof-of-Work方式を採用しており、これは元々メールスパムを減らすために提案された仕組みです。個人レベルでは追加の負荷は無視できる程度ですが、大規模なスクレイピングでは負荷が積み重なり、スクレイピングのコストが大幅に増加します。",
   "hack_purpose": "最終的に、これはヘッドレスブラウザのフィンガープリントと識別に時間を費やすためのプレースホルダーソリューションです（例：フォントレンダリングの方法による）。これにより、正当なユーザーにはチャレンジのプルーフオブワークページを提示する必要がなくなります。",
   "jshelter_note": "Anubisは、JShelterのようなプラグインが無効化する最新のJavaScript機能を必要とします。このドメインではJShelterや同様のプラグインを無効にしてください。",
-  "version_info": "このウェブサイトはAnubisバージョンで動作しています",
+  "version_info": "このウェブサイトはAnubisで動作しています バージョン",
   "try_again": "再試行",
   "go_home": "ホームに戻る",
   "contact_webmaster": "もしブロックされるべきでないと思われる場合は、ウェブマスターにご連絡ください：",