mirror of
https://github.com/TecharoHQ/anubis.git
synced 2026-04-11 02:58:49 +00:00
Compare commits
1 Commits
Xe/unblock
...
Xe/analyti
| Author | SHA1 | Date | |
|---|---|---|---|
|
e9ef5f651b |
1
.github/workflows/smoke-tests.yml
vendored
1
.github/workflows/smoke-tests.yml
vendored
@@ -23,7 +23,6 @@ jobs:
|
||||
- i18n
|
||||
- palemoon/amd64
|
||||
#- palemoon/i386
|
||||
- robots_txt
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
- import: (data)/bots/custom-async-http-client.yaml
|
||||
- import: (data)/crawlers/alibaba-cloud.yaml
|
||||
- import: (data)/crawlers/huawei-cloud.yaml
|
||||
- import: (data)/crawlers/tencent-cloud.yaml
|
||||
|
||||
@@ -13,9 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
<!-- This changes the project to: -->
|
||||
|
||||
- Fix `SERVE_ROBOTS_TXT` setting file after the double slash fix broke it.
|
||||
- Remove the default configuration rule to block Tencent cloud. If users see abuse from Tencent cloud IP ranges, please contact abuse@tencent.com and mention that you are using Anubis to protect your services. Please include source IP address, source port, timestamp, target IP address, target port, request headers (including the User-Agent header), and target endpoints/patterns.
|
||||
|
||||
## v1.23.0: Lyse Hext
|
||||
|
||||
- Add default tencent cloud DENY rule.
|
||||
|
||||
2
go.mod
2
go.mod
@@ -3,7 +3,7 @@ module github.com/TecharoHQ/anubis
|
||||
go 1.24.2
|
||||
|
||||
require (
|
||||
github.com/TecharoHQ/thoth-proto v0.4.0
|
||||
github.com/TecharoHQ/thoth-proto v0.5.0
|
||||
github.com/a-h/templ v0.3.924
|
||||
github.com/aws/aws-sdk-go-v2 v1.38.3
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.6
|
||||
|
||||
2
go.sum
2
go.sum
@@ -37,6 +37,8 @@ github.com/Songmu/gitconfig v0.2.1 h1:cZsqELfMtxWVI8ovq17gbvsR4qLfoYLAiXy5GwtJWb
|
||||
github.com/Songmu/gitconfig v0.2.1/go.mod h1:XM4O3SoXFnli9Ql2G7qXK2Fg7LJwf7Hs8GLFEOJlzmM=
|
||||
github.com/TecharoHQ/thoth-proto v0.4.0 h1:UbkvfgCku0Dm1R6O4ug3HOsJNnE6F3wB8x+Dpw2lzFI=
|
||||
github.com/TecharoHQ/thoth-proto v0.4.0/go.mod h1:IcGnZt3iYUZQVEa0Lwk5l4ix0hCeXlWUV1TJMZvbWx0=
|
||||
github.com/TecharoHQ/thoth-proto v0.5.0 h1:Fa663s4soYiURSU8MfW9tZ2wF+LsCRSaYmjUSyagfBM=
|
||||
github.com/TecharoHQ/thoth-proto v0.5.0/go.mod h1:C/U7FqTxpVn4V/qebC/GcW32I0h9xzsmWehF27KFOJs=
|
||||
github.com/TecharoHQ/yeet v0.6.3 h1:Iev6TYt/tpFYU73kbkNIYjCObYTvlihtby+htGF4Us8=
|
||||
github.com/TecharoHQ/yeet v0.6.3/go.mod h1:ltt+PWPjnvmQJxEHsdJ5K9u3GoWK83vSLWCCp8XbxqI=
|
||||
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo=
|
||||
|
||||
@@ -345,15 +345,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Forward robots.txt requests to mux when ServeRobotsTXT is enabled
|
||||
if s.opts.ServeRobotsTXT {
|
||||
path := strings.TrimPrefix(r.URL.Path, anubis.BasePrefix)
|
||||
if path == "/robots.txt" || path == "/.well-known/robots.txt" {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
s.maybeReverseProxyOrPage(w, r)
|
||||
}
|
||||
|
||||
|
||||
4
package-lock.json
generated
4
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@techaro/anubis",
|
||||
"version": "1.23.0",
|
||||
"version": "1.23.0-pre2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@techaro/anubis",
|
||||
"version": "1.23.0",
|
||||
"version": "1.23.0-pre2",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-crypto/sha256-js": "^5.2.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@techaro/anubis",
|
||||
"version": "1.23.0",
|
||||
"version": "1.23.0-pre2",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
bots:
|
||||
- name: challenge
|
||||
user_agent_regex: CHALLENGE
|
||||
action: CHALLENGE
|
||||
|
||||
status_codes:
|
||||
CHALLENGE: 200
|
||||
DENY: 403
|
||||
@@ -1,27 +0,0 @@
|
||||
async function getRobotsTxt() {
|
||||
return fetch("http://localhost:8923/robots.txt", {
|
||||
headers: {
|
||||
"Accept-Language": "en",
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
}
|
||||
})
|
||||
.then(resp => {
|
||||
if (resp.status !== 200) {
|
||||
throw new Error(`wanted status 200, got status: ${resp.status}`);
|
||||
}
|
||||
return resp;
|
||||
})
|
||||
.then(resp => resp.text());
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const page = await getRobotsTxt();
|
||||
|
||||
if (page.includes(`<html>`)) {
|
||||
console.log(page)
|
||||
throw new Error("serve robots.txt smoke test failed");
|
||||
}
|
||||
|
||||
console.log("serve-robots-txt serves robots.txt");
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
function cleanup() {
|
||||
pkill -P $$
|
||||
}
|
||||
|
||||
trap cleanup EXIT SIGINT
|
||||
|
||||
# Build static assets
|
||||
(cd ../.. && npm ci && npm run assets)
|
||||
|
||||
go tool anubis --help 2>/dev/null || :
|
||||
|
||||
go run ../cmd/unixhttpd &
|
||||
|
||||
go tool anubis \
|
||||
--policy-fname ./anubis.yaml \
|
||||
--use-remote-address \
|
||||
--serve-robots-txt \
|
||||
--target=unix://$(pwd)/unixhttpd.sock &
|
||||
|
||||
backoff-retry node ./test.mjs
|
||||
2
test/robots_txt/var/.gitignore
vendored
2
test/robots_txt/var/.gitignore
vendored
@@ -1,2 +0,0 @@
|
||||
*
|
||||
!.gitignore
|
||||
Reference in New Issue
Block a user