arian/anubis-mirror
1
0
Fork 0
mirror of https://github.com/TecharoHQ/anubis.git synced 2026-04-26 10:02:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

docs: add crawl delay weight adjustment and deny user agents option to robots2policy CLI

Browse Source
This commit is contained in:
Jason Cameron
2025-06-13 14:14:50 -04:00
parent 995abcbe68
commit 7a74dddb3e
2 changed files with 12 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/robots2policy/main.go
+3 -3
View File
@@ -39,17 +39,17 @@ type Weight struct {
} }
type Challenge struct { type Challenge struct {
Difficulty int `yaml:"difficulty,omitempty" json:"difficulty,omitempty"`
Algorithm string `yaml:"algorithm,omitempty" json:"algorithm,omitempty"` Algorithm string `yaml:"algorithm,omitempty" json:"algorithm,omitempty"`
Difficulty int `yaml:"difficulty,omitempty" json:"difficulty,omitempty"`
ReportAs int `yaml:"report_as,omitempty" json:"report_as,omitempty"` ReportAs int `yaml:"report_as,omitempty" json:"report_as,omitempty"`
} }
type AnubisRule struct { type AnubisRule struct {
Name string `yaml:"name" json:"name"`
Action string `yaml:"action" json:"action"`
Expression map[string]interface{} `yaml:"expression,omitempty" json:"expression,omitempty"` Expression map[string]interface{} `yaml:"expression,omitempty" json:"expression,omitempty"`
Challenge *Challenge `yaml:"challenge,omitempty" json:"challenge,omitempty"` Challenge *Challenge `yaml:"challenge,omitempty" json:"challenge,omitempty"`
Weight *Weight `yaml:"weight,omitempty" json:"weight,omitempty"` Weight *Weight `yaml:"weight,omitempty" json:"weight,omitempty"`
Name string `yaml:"name" json:"name"`
Action string `yaml:"action" json:"action"`
} }
func main() { func main() {
docs/docs/admin/robots2policy.mdx
+9 -7
View File
@@ -34,13 +34,15 @@ robots2policy -input robots.txt -action DENY -format json
## Options ## Options
| Flag | Description | Default | | Flag | Description | Default |
|-----------|--------------------------------------------------------------------|---------------------| |-----------------------|--------------------------------------------------------------------|---------------------|
| `-input` | robots.txt file path or URL (use `-` for stdin) | *required* | | `-input` | robots.txt file path or URL (use `-` for stdin) | *required* |
| `-output` | Output file (use `-` for stdout) | stdout | | `-output` | Output file (use `-` for stdout) | stdout |
| `-format` | Output format: `yaml` or `json` | `yaml` | | `-format` | Output format: `yaml` or `json` | `yaml` |
| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` | | `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` |
| `-name` | Policy name prefix | `robots-txt-policy` | | `-name` | Policy name prefix | `robots-txt-policy` |
| `-crawl-delay-weight` | Weight adjustment for crawl-delay rules | `3` |
| `-deny-user-agents` | Action for blacklisted user agents | `DENY` |
## Example ## Example