mirror of
https://github.com/TecharoHQ/anubis.git
synced 2026-04-29 19:42:41 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Xe Iaso
|
e9ef5f651b |
@@ -23,7 +23,6 @@ jobs:
|
|||||||
- i18n
|
- i18n
|
||||||
- palemoon/amd64
|
- palemoon/amd64
|
||||||
#- palemoon/i386
|
#- palemoon/i386
|
||||||
- robots_txt
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
|
|||||||
@@ -4,3 +4,4 @@
|
|||||||
- import: (data)/bots/custom-async-http-client.yaml
|
- import: (data)/bots/custom-async-http-client.yaml
|
||||||
- import: (data)/crawlers/alibaba-cloud.yaml
|
- import: (data)/crawlers/alibaba-cloud.yaml
|
||||||
- import: (data)/crawlers/huawei-cloud.yaml
|
- import: (data)/crawlers/huawei-cloud.yaml
|
||||||
|
- import: (data)/crawlers/tencent-cloud.yaml
|
||||||
|
|||||||
@@ -13,9 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
<!-- This changes the project to: -->
|
<!-- This changes the project to: -->
|
||||||
|
|
||||||
- Fix `SERVE_ROBOTS_TXT` setting file after the double slash fix broke it.
|
|
||||||
- Remove the default configuration rule to block Tencent cloud. If users see abuse from Tencent cloud IP ranges, please contact abuse@tencent.com and mention that you are using Anubis to protect your services. Please include source IP address, source port, timestamp, target IP address, target port, request headers (including the User-Agent header), and target endpoints/patterns.
|
|
||||||
|
|
||||||
## v1.23.0: Lyse Hext
|
## v1.23.0: Lyse Hext
|
||||||
|
|
||||||
- Add default tencent cloud DENY rule.
|
- Add default tencent cloud DENY rule.
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module github.com/TecharoHQ/anubis
|
|||||||
go 1.24.2
|
go 1.24.2
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/TecharoHQ/thoth-proto v0.4.0
|
github.com/TecharoHQ/thoth-proto v0.5.0
|
||||||
github.com/a-h/templ v0.3.924
|
github.com/a-h/templ v0.3.924
|
||||||
github.com/aws/aws-sdk-go-v2 v1.38.3
|
github.com/aws/aws-sdk-go-v2 v1.38.3
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.31.6
|
github.com/aws/aws-sdk-go-v2/config v1.31.6
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ github.com/Songmu/gitconfig v0.2.1 h1:cZsqELfMtxWVI8ovq17gbvsR4qLfoYLAiXy5GwtJWb
|
|||||||
github.com/Songmu/gitconfig v0.2.1/go.mod h1:XM4O3SoXFnli9Ql2G7qXK2Fg7LJwf7Hs8GLFEOJlzmM=
|
github.com/Songmu/gitconfig v0.2.1/go.mod h1:XM4O3SoXFnli9Ql2G7qXK2Fg7LJwf7Hs8GLFEOJlzmM=
|
||||||
github.com/TecharoHQ/thoth-proto v0.4.0 h1:UbkvfgCku0Dm1R6O4ug3HOsJNnE6F3wB8x+Dpw2lzFI=
|
github.com/TecharoHQ/thoth-proto v0.4.0 h1:UbkvfgCku0Dm1R6O4ug3HOsJNnE6F3wB8x+Dpw2lzFI=
|
||||||
github.com/TecharoHQ/thoth-proto v0.4.0/go.mod h1:IcGnZt3iYUZQVEa0Lwk5l4ix0hCeXlWUV1TJMZvbWx0=
|
github.com/TecharoHQ/thoth-proto v0.4.0/go.mod h1:IcGnZt3iYUZQVEa0Lwk5l4ix0hCeXlWUV1TJMZvbWx0=
|
||||||
|
github.com/TecharoHQ/thoth-proto v0.5.0 h1:Fa663s4soYiURSU8MfW9tZ2wF+LsCRSaYmjUSyagfBM=
|
||||||
|
github.com/TecharoHQ/thoth-proto v0.5.0/go.mod h1:C/U7FqTxpVn4V/qebC/GcW32I0h9xzsmWehF27KFOJs=
|
||||||
github.com/TecharoHQ/yeet v0.6.3 h1:Iev6TYt/tpFYU73kbkNIYjCObYTvlihtby+htGF4Us8=
|
github.com/TecharoHQ/yeet v0.6.3 h1:Iev6TYt/tpFYU73kbkNIYjCObYTvlihtby+htGF4Us8=
|
||||||
github.com/TecharoHQ/yeet v0.6.3/go.mod h1:ltt+PWPjnvmQJxEHsdJ5K9u3GoWK83vSLWCCp8XbxqI=
|
github.com/TecharoHQ/yeet v0.6.3/go.mod h1:ltt+PWPjnvmQJxEHsdJ5K9u3GoWK83vSLWCCp8XbxqI=
|
||||||
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo=
|
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo=
|
||||||
|
|||||||
@@ -345,15 +345,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Forward robots.txt requests to mux when ServeRobotsTXT is enabled
|
|
||||||
if s.opts.ServeRobotsTXT {
|
|
||||||
path := strings.TrimPrefix(r.URL.Path, anubis.BasePrefix)
|
|
||||||
if path == "/robots.txt" || path == "/.well-known/robots.txt" {
|
|
||||||
s.mux.ServeHTTP(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s.maybeReverseProxyOrPage(w, r)
|
s.maybeReverseProxyOrPage(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@techaro/anubis",
|
"name": "@techaro/anubis",
|
||||||
"version": "1.23.0",
|
"version": "1.23.0-pre2",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@techaro/anubis",
|
"name": "@techaro/anubis",
|
||||||
"version": "1.23.0",
|
"version": "1.23.0-pre2",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-crypto/sha256-js": "^5.2.0",
|
"@aws-crypto/sha256-js": "^5.2.0",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@techaro/anubis",
|
"name": "@techaro/anubis",
|
||||||
"version": "1.23.0",
|
"version": "1.23.0-pre2",
|
||||||
"description": "",
|
"description": "",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,8 +0,0 @@
|
|||||||
bots:
|
|
||||||
- name: challenge
|
|
||||||
user_agent_regex: CHALLENGE
|
|
||||||
action: CHALLENGE
|
|
||||||
|
|
||||||
status_codes:
|
|
||||||
CHALLENGE: 200
|
|
||||||
DENY: 403
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
async function getRobotsTxt() {
|
|
||||||
return fetch("http://localhost:8923/robots.txt", {
|
|
||||||
headers: {
|
|
||||||
"Accept-Language": "en",
|
|
||||||
"User-Agent": "Mozilla/5.0",
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.then(resp => {
|
|
||||||
if (resp.status !== 200) {
|
|
||||||
throw new Error(`wanted status 200, got status: ${resp.status}`);
|
|
||||||
}
|
|
||||||
return resp;
|
|
||||||
})
|
|
||||||
.then(resp => resp.text());
|
|
||||||
}
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
const page = await getRobotsTxt();
|
|
||||||
|
|
||||||
if (page.includes(`<html>`)) {
|
|
||||||
console.log(page)
|
|
||||||
throw new Error("serve robots.txt smoke test failed");
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("serve-robots-txt serves robots.txt");
|
|
||||||
process.exit(0);
|
|
||||||
})();
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
function cleanup() {
|
|
||||||
pkill -P $$
|
|
||||||
}
|
|
||||||
|
|
||||||
trap cleanup EXIT SIGINT
|
|
||||||
|
|
||||||
# Build static assets
|
|
||||||
(cd ../.. && npm ci && npm run assets)
|
|
||||||
|
|
||||||
go tool anubis --help 2>/dev/null || :
|
|
||||||
|
|
||||||
go run ../cmd/unixhttpd &
|
|
||||||
|
|
||||||
go tool anubis \
|
|
||||||
--policy-fname ./anubis.yaml \
|
|
||||||
--use-remote-address \
|
|
||||||
--serve-robots-txt \
|
|
||||||
--target=unix://$(pwd)/unixhttpd.sock &
|
|
||||||
|
|
||||||
backoff-retry node ./test.mjs
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
*
|
|
||||||
!.gitignore
|
|
||||||
Reference in New Issue
Block a user