fix(data): add services folder to embedded filesystem

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -214,7 +214,6 @@ nicksnyder
 nobots
 NONINFRINGEMENT
 nosleep
-nullglob
 OCOB
 ogtag
 oklch
@@ -279,7 +278,6 @@ Seo
 setsebool
 shellcheck
 shirou
-shopt
 Sidetrade
 simprint
 sitemap

data/embed.go

@@ -3,6 +3,6 @@ package data
 import "embed"
 
 var (
-	//go:embed botPolicies.yaml all:apps all:bots all:clients all:common all:crawlers all:meta
+	//go:embed botPolicies.yaml all:apps all:bots all:clients all:common all:crawlers all:meta all:services
 	BotPolicies embed.FS
 )

docs/docs/CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Document missing environment variables in installation guide: `SLOG_LEVEL`, `COOKIE_PREFIX`, `FORCED_LANGUAGE`, and `TARGET_DISABLE_KEEPALIVE` ([#1086](https://github.com/TecharoHQ/anubis/pull/1086))
 - Fixed `robots2policy` to properly group consecutive user agents into `any:` instead of only processing the last one ([#925](https://github.com/TecharoHQ/anubis/pull/925))
 - Add the [`s3api` storage backend](./admin/policies.mdx#s3api) to allow Anubis to use S3 API compatible object storage as its storage backend.
+- Add the `services` folder to the embedded data filesystem.
 
 ### Bug Fixes
 

lib/challenge/metarefresh/metarefresh_templ.go

@@ -93,9 +93,9 @@ func page(redir string, difficulty int, loc *localization.SimpleLocalizer) templ
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var6 string
-		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d; url=%s", difficulty+1, redir))
+		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d; url=%s", difficulty, redir))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 16, Col: 85}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 16, Col: 83}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {

lib/challenge/preact/build.sh

@@ -40,9 +40,9 @@ for the JavaScript code in this page.
 
 mkdir -p static/js
 
-for file in js/*.tsx; do
+for file in js/*.jsx; do
   filename="${file##*/}"       # Extracts "app.jsx" from "./js/app.jsx"
-  output="${filename%.tsx}.js"  # Changes "app.jsx" to "app.js"
+  output="${filename%.jsx}.js"  # Changes "app.jsx" to "app.js"
   echo $output
 
   esbuild "${file}" --minify --bundle --outfile=static/"${output}" --banner:js="${LICENSE}"

lib/challenge/preact/js/app.jsx

@@ -0,0 +1,62 @@
+import { render, h, Fragment } from 'preact';
+import { useState, useEffect } from 'preact/hooks';
+import { g, j, u, x } from "./xeact.js";
+import { Sha256 } from '@aws-crypto/sha256-js';
+
+/** @jsx h */
+/** @jsxFrag Fragment */
+
+function toHexString(arr) {
+  return Array.from(arr)
+    .map((c) => c.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+const App = () => {
+  const [state, setState] = useState(null);
+  const [imageURL, setImageURL] = useState(null);
+  const [passed, setPassed] = useState(false);
+  const [challenge, setChallenge] = useState(null);
+
+  useEffect(() => {
+    setState(j("preact_info"));
+  });
+
+  useEffect(() => {
+    setImageURL(state.pensive_url);
+    const hash = new Sha256('');
+    hash.update(state.challenge);
+    setChallenge(toHexString(hash.digestSync()));
+  }, [state]);
+
+  useEffect(() => {
+    const timer = setTimeout(() => {
+      setPassed(true);
+    }, state.difficulty * 125);
+
+    return () => clearTimeout(timer);
+  }, [challenge]);
+
+  useEffect(() => {
+    window.location.href = u(state.redir, {
+      result: challenge,
+    });
+  }, [passed]);
+
+  return (
+    <>
+      {imageURL !== null && (
+        <img src={imageURL} style="width:100%;max-width:256px;" />
+      )}
+      {state !== null && (
+        <>
+          <p id="status">{state.loading_message}</p>
+          <p>{state.connection_security_message}</p>
+        </>
+      )}
+    </>
+  );
+};
+
+x(g("app"));
+render(<App />, g("app"));

lib/challenge/preact/js/app.tsx

@@ -1,87 +0,0 @@
-import { render, h, Fragment } from "preact";
-import { useState, useEffect } from "preact/hooks";
-import { g, j, r, u, x } from "./xeact.js";
-import { Sha256 } from "@aws-crypto/sha256-js";
-
-/** @jsx h */
-/** @jsxFrag Fragment */
-
-function toHexString(arr: Uint8Array) {
-  return Array.from(arr)
-    .map((c) => c.toString(16).padStart(2, "0"))
-    .join("");
-}
-
-interface PreactInfo {
-  redir: string;
-  challenge: string;
-  difficulty: number;
-  connection_security_message: string;
-  loading_message: string;
-  pensive_url: string;
-}
-
-const App = () => {
-  const [state, setState] = useState<PreactInfo>();
-  const [imageURL, setImageURL] = useState<string | null>(null);
-  const [passed, setPassed] = useState<boolean>(false);
-  const [challenge, setChallenge] = useState<string | null>(null);
-
-  useEffect(() => {
-    setState(j("preact_info"));
-  });
-
-  useEffect(() => {
-    if (state === undefined) {
-      return;
-    }
-
-    setImageURL(state?.pensive_url);
-    const hash = new Sha256("");
-    hash.update(state.challenge);
-    setChallenge(toHexString(hash.digestSync()));
-  }, [state]);
-
-  useEffect(() => {
-    if (state === undefined) {
-      return;
-    }
-
-    const timer = setTimeout(() => {
-      setPassed(true);
-    }, state?.difficulty * 125);
-
-    return () => clearTimeout(timer);
-  }, [challenge]);
-
-  useEffect(() => {
-    if (state === undefined) {
-      return;
-    }
-
-    if (challenge === null) {
-      return;
-    }
-
-    window.location.href = u(state.redir, {
-      result: challenge,
-    });
-  }, [passed]);
-
-  return (
-    <>
-      {imageURL !== null && (
-        <img src={imageURL} style={{ width: "100%", maxWidth: "256px" }} />
-      )}
-      {state !== undefined && (
-        <>
-          <p id="status">{state.loading_message}</p>
-          <p>{state.connection_security_message}</p>
-        </>
-      )}
-    </>
-  );
-};
-
-x(g("app"));
-render(<App />, g("app"));

web/build.sh

@@ -39,18 +39,9 @@ for the JavaScript code in this page.
 mkdir -p static/locales
 cp ../lib/localization/locales/*.json static/locales/
 
-shopt -s nullglob globstar
-
-for file in js/**/*.ts js/**/*.mjs; do
-  out="static/${file}"
-  if [[ "$file" == *.ts ]]; then
-    out="static/${file%.ts}.mjs"
-  fi
-
-  mkdir -p "$(dirname "$out")"
-
-  esbuild "$file" --sourcemap --bundle --minify --outfile="$out" --banner:js="$LICENSE"
-  gzip -f -k -n "$out"
-  zstd -f -k --ultra -22 "$out"
-  brotli -fZk "$out"
+for file in js/*.mjs js/worker/*.mjs; do
+  esbuild "${file}" --sourcemap --bundle --minify --outfile=static/"${file}" --banner:js="${LICENSE}"
+  gzip -f -k -n static/${file}
+  zstd -f -k --ultra -22 static/${file}
+  brotli -fZk static/${file}
 done

web/js/algorithms/fast.mjs

@@ -1,21 +1,11 @@
-type ProgressCallback = (nonce: number) => void;
-
-interface ProcessOptions {
-  basePrefix: string;
-  version: string;
-}
-
-const getHardwareConcurrency = () =>
-  navigator.hardwareConcurrency !== undefined ? navigator.hardwareConcurrency : 1;
-
 export default function process(
-  options: ProcessOptions,
-  data: string,
-  difficulty: number = 5,
-  signal: AbortSignal | null = null,
-  progressCallback?: ProgressCallback,
-  threads: number = Math.trunc(Math.max(getHardwareConcurrency() / 2, 1)),
-): Promise<string> {
+  { basePrefix, version },
+  data,
+  difficulty = 5,
+  signal = null,
+  progressCallback = null,
+  threads = Math.trunc(Math.max(navigator.hardwareConcurrency / 2, 1)),
+) {
   console.debug("fast algo");
 
   let workerMethod = window.crypto !== undefined ? "webcrypto" : "purejs";
@@ -26,17 +16,13 @@ export default function process(
   }
 
   return new Promise((resolve, reject) => {
-    let webWorkerURL = `${options.basePrefix}/.within.website/x/cmd/anubis/static/js/worker/sha256-${workerMethod}.mjs?cacheBuster=${options.version}`;
+    let webWorkerURL = `${basePrefix}/.within.website/x/cmd/anubis/static/js/worker/sha256-${workerMethod}.mjs?cacheBuster=${version}`;
 
-    const workers: Worker[] = [];
+    console.log(webWorkerURL);
+
+    const workers = [];
     let settled = false;
 
-    const onAbort = () => {
-      console.log("PoW aborted");
-      cleanup();
-      reject(new DOMException("Aborted", "AbortError"));
-    };
-
     const cleanup = () => {
       if (settled) {
         return;
@@ -48,6 +34,12 @@ export default function process(
       }
     };
 
+    const onAbort = () => {
+      console.log("PoW aborted");
+      cleanup();
+      reject(new DOMException("Aborted", "AbortError"));
+    };
+
     if (signal != null) {
       if (signal.aborted) {
         return onAbort();

web/js/algorithms/index.mjs

@@ -1,4 +1,4 @@
-import fast from "./fast";
+import fast from "./fast.mjs";
 
 export default {
   fast: fast,

web/js/bench.mjs

@@ -1,24 +1,20 @@
-import algorithms from "./algorithms";
+import algorithms from "./algorithms/index.mjs";
 
 const defaultDifficulty = 4;
 
-const status: HTMLParagraphElement = document.getElementById("status") as HTMLParagraphElement;
-const difficultyInput: HTMLInputElement = document.getElementById("difficulty-input") as HTMLInputElement;
-const algorithmSelect: HTMLSelectElement = document.getElementById("algorithm-select") as HTMLSelectElement;
-const compareSelect: HTMLSelectElement = document.getElementById("compare-select") as HTMLSelectElement;
-const header: HTMLTableRowElement = document.getElementById("table-header") as HTMLTableRowElement;
-const headerCompare: HTMLTableSectionElement = document.getElementById("table-header-compare") as HTMLTableSectionElement;
-const results: HTMLTableRowElement = document.getElementById("results") as HTMLTableRowElement;
+const status = document.getElementById("status");
+const difficultyInput = document.getElementById("difficulty-input");
+const algorithmSelect = document.getElementById("algorithm-select");
+const compareSelect = document.getElementById("compare-select");
+const header = document.getElementById("table-header");
+const headerCompare = document.getElementById("table-header-compare");
+const results = document.getElementById("results");
 
 const setupControls = () => {
-  if (defaultDifficulty == null) {
-    return;
-  }
-
-  difficultyInput.value = defaultDifficulty.toString();
+  difficultyInput.value = defaultDifficulty;
   for (const alg of Object.keys(algorithms)) {
     const option1 = document.createElement("option");
-    algorithmSelect?.append(option1);
+    algorithmSelect.append(option1);
     const option2 = document.createElement("option");
     compareSelect.append(option2);
     option1.value = option1.innerText = option2.value = option2.innerText = alg;
@@ -120,13 +116,13 @@ const benchmarkLoop = async (controller) => {
   await benchmarkLoop(controller);
 };
 
-let controller: AbortController | null = null;
+let controller = null;
 const reset = () => {
   stats.time = stats.iters = 0;
   comparison.time = comparison.iters = 0;
   results.innerHTML = status.innerText = "";
 
-  const table = results.parentElement as HTMLElement;
+  const table = results.parentElement;
   if (compareSelect.value !== "NONE") {
     table.style.gridTemplateColumns = "repeat(4,auto)";
     header.style.display = "none";

web/js/main.mjs

@@ -1,21 +1,12 @@
-import algorithms from "./algorithms";
+import algorithms from "./algorithms/index.mjs";
 
 // from Xeact
-const u = (url: string = "", params: Record<string, any> = {}) => {
+const u = (url = "", params = {}) => {
   let result = new URL(url, window.location.href);
   Object.entries(params).forEach(([k, v]) => result.searchParams.set(k, v));
   return result.toString();
 };
 
-const j = (id: string): any | null => {
-  const elem = document.getElementById(id);
-  if (elem === null) {
-    return null;
-  }
-
-  return JSON.parse(elem.textContent);
-};
-
 const imageURL = (mood, cacheBuster, basePrefix) =>
   u(`${basePrefix}/.within.website/x/cmd/anubis/static/img/${mood}.webp`, {
     cacheBuster,
@@ -23,10 +14,9 @@ const imageURL = (mood, cacheBuster, basePrefix) =>
 
 // Detect available languages by loading the manifest
 const getAvailableLanguages = async () => {
-  const basePrefix = j("anubis_base_prefix");
-  if (basePrefix === null) {
-    return;
-  }
+  const basePrefix = JSON.parse(
+    document.getElementById("anubis_base_prefix").textContent,
+  );
 
   try {
     const response = await fetch(`${basePrefix}/.within.website/x/cmd/anubis/static/locales/manifest.json`);
@@ -48,11 +38,9 @@ const getBrowserLanguage = async () =>
 
 // Load translations from JSON files
 const loadTranslations = async (lang) => {
-  const basePrefix = j("anubis_base_prefix");
-  if (basePrefix === null) {
-    return;
-  }
-
+  const basePrefix = JSON.parse(
+    document.getElementById("anubis_base_prefix").textContent,
+  );
   try {
     const response = await fetch(`${basePrefix}/.within.website/x/cmd/anubis/static/locales/${lang}.json`);
     return await response.json();
@@ -66,10 +54,9 @@ const loadTranslations = async (lang) => {
 };
 
 const getRedirectUrl = () => {
-  const publicUrl = j("anubis_public_url");
-  if (publicUrl === null) {
-    return;
-  }
+  const publicUrl = JSON.parse(
+    document.getElementById("anubis_public_url").textContent,
+  );
   if (publicUrl && window.location.href.startsWith(publicUrl)) {
     const urlParams = new URLSearchParams(window.location.search);
     return urlParams.get('redir');
@@ -104,14 +91,16 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
       value: navigator.cookieEnabled,
     },
   ];
-
-  const status: HTMLParagraphElement = document.getElementById("status") as HTMLParagraphElement;
-  const image: HTMLImageElement = document.getElementById("image") as HTMLImageElement;
-  const title: HTMLHeadingElement = document.getElementById("title") as HTMLHeadingElement;
-  const progress: HTMLDivElement = document.getElementById("progress") as HTMLDivElement;
-
-  const anubisVersion = j("anubis_version");
-  const basePrefix = j("anubis_base_prefix");
+  const status = document.getElementById("status");
+  const image = document.getElementById("image");
+  const title = document.getElementById("title");
+  const progress = document.getElementById("progress");
+  const anubisVersion = JSON.parse(
+    document.getElementById("anubis_version").textContent,
+  );
+  const basePrefix = JSON.parse(
+    document.getElementById("anubis_base_prefix").textContent,
+  );
   const details = document.querySelector("details");
   let userReadDetails = false;
 
@@ -143,7 +132,9 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
     }
   }
 
-  const { challenge, rules } = j("anubis_challenge");
+  const { challenge, rules } = JSON.parse(
+    document.getElementById("anubis_challenge").textContent,
+  );
 
   const process = algorithms[rules.algorithm];
   if (!process) {
@@ -191,9 +182,7 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
         const probability = Math.pow(1 - likelihood, iters);
         const distance = (1 - Math.pow(probability, 2)) * 100;
         progress["aria-valuenow"] = distance;
-        if (progress.firstElementChild !== null) {
-          (progress.firstElementChild as HTMLElement).style.width = `${distance}%`;
-        }
+        progress.firstElementChild.style.width = `${distance}%`;
 
         if (probability < 0.1 && !showingApology) {
           status.append(
@@ -208,7 +197,7 @@ const t = (key) => translations[`js_${key}`] || translations[key] || key;
     console.log({ hash, nonce });
 
     if (userReadDetails) {
-      const container: HTMLDivElement = document.getElementById("progress") as HTMLDivElement;
+      const container = document.getElementById("progress");
 
       // Style progress bar as a continue button
       container.style.display = "flex";

web/js/worker/sha256-purejs.mjs

@@ -6,7 +6,7 @@ const calculateSHA256 = (text) => {
   return hash.digest();
 };
 
-function toHexString(arr: Uint8Array): string {
+function toHexString(arr) {
   return Array.from(arr)
     .map((c) => c.toString(16).padStart(2, "0"))
     .join("");

web/js/worker/sha256-webcrypto.mjs

@@ -1,11 +1,10 @@
 const encoder = new TextEncoder();
-
-const calculateSHA256 = async (input: string) => {
+const calculateSHA256 = async (input) => {
   const data = encoder.encode(input);
   return await crypto.subtle.digest("SHA-256", data);
 };
 
-const toHexString = (byteArray: Uint8Array) => {
+const toHexString = (byteArray) => {
   return byteArray.reduce((str, byte) => str + byte.toString(16).padStart(2, "0"), "");
 };