diff --git a/cmd/robots2policy/main.go b/cmd/robots2policy/main.go index 3e76d398..f42f6b0f 100644 --- a/cmd/robots2policy/main.go +++ b/cmd/robots2policy/main.go @@ -39,17 +39,17 @@ type Weight struct { } type Challenge struct { - Difficulty int `yaml:"difficulty,omitempty" json:"difficulty,omitempty"` Algorithm string `yaml:"algorithm,omitempty" json:"algorithm,omitempty"` + Difficulty int `yaml:"difficulty,omitempty" json:"difficulty,omitempty"` ReportAs int `yaml:"report_as,omitempty" json:"report_as,omitempty"` } type AnubisRule struct { - Name string `yaml:"name" json:"name"` - Action string `yaml:"action" json:"action"` Expression map[string]interface{} `yaml:"expression,omitempty" json:"expression,omitempty"` Challenge *Challenge `yaml:"challenge,omitempty" json:"challenge,omitempty"` Weight *Weight `yaml:"weight,omitempty" json:"weight,omitempty"` + Name string `yaml:"name" json:"name"` + Action string `yaml:"action" json:"action"` } func main() { diff --git a/docs/docs/admin/robots2policy.mdx b/docs/docs/admin/robots2policy.mdx index 6154c193..30f0eab0 100644 --- a/docs/docs/admin/robots2policy.mdx +++ b/docs/docs/admin/robots2policy.mdx @@ -34,13 +34,15 @@ robots2policy -input robots.txt -action DENY -format json ## Options -| Flag | Description | Default | -|-----------|--------------------------------------------------------------------|---------------------| -| `-input` | robots.txt file path or URL (use `-` for stdin) | *required* | -| `-output` | Output file (use `-` for stdout) | stdout | -| `-format` | Output format: `yaml` or `json` | `yaml` | -| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` | -| `-name` | Policy name prefix | `robots-txt-policy` | +| Flag | Description | Default | +|-----------------------|--------------------------------------------------------------------|---------------------| +| `-input` | robots.txt file path or URL (use `-` for stdin) | *required* | +| `-output` | Output file (use `-` for stdout) | stdout | +| `-format` | Output format: `yaml` or `json` | `yaml` | +| `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` | +| `-name` | Policy name prefix | `robots-txt-policy` | +| `-crawl-delay-weight` | Weight adjustment for crawl-delay rules | `3` | +| `-deny-user-agents` | Action for blacklisted user agents | `DENY` | ## Example