mirror of
https://github.com/TecharoHQ/anubis.git
synced 2026-04-11 02:58:49 +00:00
* feat: add robots2policy CLI utility to convert robots.txt to Anubis challenge policies
* feat: add documentation for robots2policy CLI tool
* feat: implement crawl delay handling as weight adjustment in Anubis rules
* feat: add various robots.txt and YAML configurations for user agent handling and crawl delays
* test: add comprehensive tests for robots2policy conversion and parsing
* fix: update example URL in usage instructions for robots2policy CLI
* Update metadata
check-spelling run (pull_request) for json/robots2policycli
Signed-off-by: check-spelling-bot <check-spelling-bot@users.noreply.github.com>
on-behalf-of: @check-spelling <check-spelling-bot@check-spelling.dev>
* docs: add crawl delay weight adjustment and deny user agents option to robots2policy CLI
* Update cmd/robots2policy/main.go
Co-authored-by: Xe Iaso <me@xeiaso.net>
Signed-off-by: Jason Cameron <jasoncameron.all@gmail.com>
* Update cmd/robots2policy/main.go
Co-authored-by: Xe Iaso <me@xeiaso.net>
Signed-off-by: Jason Cameron <jasoncameron.all@gmail.com>
* fix(robots2policy): use sigs.k8s.io/yaml
Signed-off-by: Xe Iaso <me@xeiaso.net>
* feat(config): properly marshal bot policy rules
Signed-off-by: Xe Iaso <me@xeiaso.net>
* chore(yeetfile): expose robots2policy in libexec
Signed-off-by: Xe Iaso <me@xeiaso.net>
* fix(yeetfile): put robots2policy in $PATH
Signed-off-by: Xe Iaso <me@xeiaso.net>
* Update metadata
check-spelling run (pull_request) for json/robots2policycli
Signed-off-by: check-spelling-bot <check-spelling-bot@users.noreply.github.com>
on-behalf-of: @check-spelling <check-spelling-bot@check-spelling.dev>
* style: reorder imports
* refactor: use preexisting structs in config
* fix: correct flag check in main function
* fix: reorder fields in AnubisRule struct for better alignment
* style: improve alignment of struct fields in AnubisRule and OGTagCache
* Update metadata
check-spelling run (pull_request) for json/robots2policycli
Signed-off-by: check-spelling-bot <check-spelling-bot@users.noreply.github.com>
on-behalf-of: @check-spelling <check-spelling-bot@check-spelling.dev>
* fix: add validation for generated Anubis rules from robots.txt
* feat: add batch processing for robots.txt files to generate Anubis CEL policies
* fix: improve usage message and error handling for input file requirement
* refactor: update AnubisRule structure to use ExpressionOrList for improved expression handling
* refactor: reorganize policy definitions in YAML files for consistency and clarity
* fix: correct indentation in blacklist and complex YAML files for consistency
* test: enhance output comparison in robots2policy tests for YAML and JSON formats
* Revert "fix: improve usage message and error handling for input file requirement"
This reverts commit ddcde1f2a3.
* fix: improve usage message and error handling in robots2policy
Signed-off-by: Jason Cameron <git@jasoncameron.dev>
---------
Signed-off-by: check-spelling-bot <check-spelling-bot@users.noreply.github.com>
Signed-off-by: Jason Cameron <jasoncameron.all@gmail.com>
Signed-off-by: Xe Iaso <me@xeiaso.net>
Signed-off-by: Jason Cameron <git@jasoncameron.dev>
Co-authored-by: Xe Iaso <me@xeiaso.net>
103 lines
2.3 KiB
Go
103 lines
2.3 KiB
Go
package config
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
)
|
|
|
|
var (
|
|
ErrExpressionOrListMustBeStringOrObject = errors.New("config: this must be a string or an object")
|
|
ErrExpressionEmpty = errors.New("config: this expression is empty")
|
|
ErrExpressionCantHaveBoth = errors.New("config: expression block can't contain multiple expression types")
|
|
)
|
|
|
|
type ExpressionOrList struct {
|
|
Expression string `json:"-" yaml:"-"`
|
|
All []string `json:"all,omitempty" yaml:"all,omitempty"`
|
|
Any []string `json:"any,omitempty" yaml:"any,omitempty"`
|
|
}
|
|
|
|
func (eol ExpressionOrList) Equal(rhs *ExpressionOrList) bool {
|
|
if eol.Expression != rhs.Expression {
|
|
return false
|
|
}
|
|
|
|
if !slices.Equal(eol.All, rhs.All) {
|
|
return false
|
|
}
|
|
|
|
if !slices.Equal(eol.Any, rhs.Any) {
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func (eol *ExpressionOrList) MarshalYAML() (any, error) {
|
|
switch {
|
|
case len(eol.All) == 1 && len(eol.Any) == 0:
|
|
eol.Expression = eol.All[0]
|
|
eol.All = nil
|
|
case len(eol.Any) == 1 && len(eol.All) == 0:
|
|
eol.Expression = eol.Any[0]
|
|
eol.Any = nil
|
|
}
|
|
|
|
if eol.Expression != "" {
|
|
return eol.Expression, nil
|
|
}
|
|
|
|
type RawExpressionOrList ExpressionOrList
|
|
return RawExpressionOrList(*eol), nil
|
|
}
|
|
|
|
func (eol *ExpressionOrList) MarshalJSON() ([]byte, error) {
|
|
switch {
|
|
case len(eol.All) == 1 && len(eol.Any) == 0:
|
|
eol.Expression = eol.All[0]
|
|
eol.All = nil
|
|
case len(eol.Any) == 1 && len(eol.All) == 0:
|
|
eol.Expression = eol.Any[0]
|
|
eol.Any = nil
|
|
}
|
|
|
|
if eol.Expression != "" {
|
|
return json.Marshal(string(eol.Expression))
|
|
}
|
|
|
|
type RawExpressionOrList ExpressionOrList
|
|
val := RawExpressionOrList(*eol)
|
|
return json.Marshal(val)
|
|
}
|
|
|
|
func (eol *ExpressionOrList) UnmarshalJSON(data []byte) error {
|
|
switch string(data[0]) {
|
|
case `"`: // string
|
|
return json.Unmarshal(data, &eol.Expression)
|
|
case "{": // object
|
|
type RawExpressionOrList ExpressionOrList
|
|
var val RawExpressionOrList
|
|
if err := json.Unmarshal(data, &val); err != nil {
|
|
return err
|
|
}
|
|
eol.All = val.All
|
|
eol.Any = val.Any
|
|
|
|
return nil
|
|
}
|
|
|
|
return ErrExpressionOrListMustBeStringOrObject
|
|
}
|
|
|
|
func (eol *ExpressionOrList) Valid() error {
|
|
if eol.Expression == "" && len(eol.All) == 0 && len(eol.Any) == 0 {
|
|
return ErrExpressionEmpty
|
|
}
|
|
if len(eol.All) != 0 && len(eol.Any) != 0 {
|
|
return ErrExpressionCantHaveBoth
|
|
}
|
|
|
|
return nil
|
|
}
|