diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index 782daa34..c0c810b2 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Enable [metrics serving via TLS](./admin/policies.mdx#tls), including [mutual TLS (mTLS)](./admin/policies.mdx#mtls). - Enable [HTTP basic auth](./admin/policies.mdx#http-basic-authentication) for the metrics server. - Fix a bug in the dataset poisoning maze that could allow denial of service [#1580](https://github.com/TecharoHQ/anubis/issues/1580). +- Add config option to add ASN to logs/metrics. ## v1.25.0: Necron diff --git a/docs/docs/admin/policies.mdx b/docs/docs/admin/policies.mdx index a7d4e8b7..be53550d 100644 --- a/docs/docs/admin/policies.mdx +++ b/docs/docs/admin/policies.mdx @@ -411,6 +411,7 @@ Anubis exposes the following logging settings in the policy file: | `level` | [log level](#log-levels) | `info` | The logging level threshold. Any logs that are at or above this threshold will be drained to the sink. Any other logs will be discarded. | | `sink` | string | `stdio`, `file` | The sink where the logs drain to as they are being recorded in Anubis. | | `parameters` | object | | Parameters for the given logging sink. This will vary based on the logging sink of choice. See below for more information. | +| `asn` | bool | `true`, `false` | Add ASN information to logs/metrics. (Requires a Thoth client configured) | Anubis supports the following logging sinks: diff --git a/lib/anubis.go b/lib/anubis.go index 2ff5ec92..ddcb577e 100644 --- a/lib/anubis.go +++ b/lib/anubis.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "net/url" + "strconv" "strings" "time" @@ -32,6 +33,7 @@ import ( "github.com/TecharoHQ/anubis/lib/policy" "github.com/TecharoHQ/anubis/lib/policy/checker" "github.com/TecharoHQ/anubis/lib/store" + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" // challenge implementations _ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh" @@ -39,31 +41,52 @@ import ( _ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork" ) +type contextKey int + +const asnContextKey contextKey = iota + +type asnInfo struct { + ASN string + Description string +} + +func asnFromContext(ctx context.Context) (string, string) { + if v, ok := ctx.Value(asnContextKey).(asnInfo); ok { + return v.ASN, v.Description + } + return "", "" +} + var ( challengesIssued = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "anubis_challenges_issued", Help: "The total number of challenges issued", - }, []string{"method"}) + }, []string{"method", "asn", "asn_description"}) challengesValidated = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "anubis_challenges_validated", Help: "The total number of challenges validated", - }, []string{"method"}) + }, []string{"method", "asn", "asn_description"}) droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "anubis_dronebl_hits", Help: "The total number of hits from DroneBL", - }, []string{"status"}) + }, []string{"status", "asn", "asn_description"}) failedValidations = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "anubis_failed_validations", Help: "The total number of failed validations", - }, []string{"method"}) + }, []string{"method", "asn", "asn_description"}) requestsProxied = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "anubis_proxied_requests_total", Help: "Number of requests proxied through Anubis to upstream targets", - }, []string{"host"}) + }, []string{"host", "asn", "asn_description"}) + + requestsByASN = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "anubis_requests_by_asn_total", + Help: "Number of requests by ASN", + }, []string{"asn", "asn_description"}) ) type Server struct { @@ -78,6 +101,28 @@ type Server struct { hs512Secret []byte } +func (s *Server) getRequestLogger(r *http.Request) (*slog.Logger, *http.Request) { + lg := internal.GetRequestLogger(s.logger, r) + + if s.policy.LogASN && s.policy.ThothClient != nil { + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) + defer cancel() + + ip := r.Header.Get("X-Real-Ip") + if info, err := s.policy.ThothClient.IPToASN.Lookup(ctx, &iptoasnv1.LookupRequest{IpAddress: ip}); err == nil && info.GetAnnounced() { + asn := strconv.FormatUint(uint64(info.GetAsNumber()), 10) + lg = lg.With("asn", info.GetAsNumber(), "asn_description", info.GetDescription()) + requestsByASN.WithLabelValues(asn, info.GetDescription()).Inc() + r = r.WithContext(context.WithValue(r.Context(), asnContextKey, asnInfo{ + ASN: asn, + Description: info.GetDescription(), + })) + } + } + + return lg, r +} + func (s *Server) getTokenKeyfunc() jwt.Keyfunc { // return ED25519 key if HS512 is not set if len(s.hs512Secret) == 0 { @@ -193,7 +238,7 @@ func (s *Server) maybeReverseProxyOrPage(w http.ResponseWriter, r *http.Request) } func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpStatusOnly bool) { - lg := internal.GetRequestLogger(s.logger, r) + lg, r := s.getRequestLogger(r) if val, _ := s.store.Get(r.Context(), fmt.Sprintf("ogtags:allow:%s%s", r.Host, r.URL.String())); val != nil { lg.Debug("serving opengraph tag asset") @@ -218,7 +263,10 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request, httpS r.Header.Add("X-Anubis-Rule", cr.Name) r.Header.Add("X-Anubis-Action", string(cr.Rule)) lg = lg.With("check_result", cr) - policy.Applications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1) + { + asn, asnDesc := asnFromContext(r.Context()) + policy.Applications.WithLabelValues(cr.Name, string(cr.Rule), asn, asnDesc).Add(1) + } ip := r.Header.Get("X-Real-Ip") @@ -348,7 +396,8 @@ func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, lg.Error("can't look up ip in dnsbl", "err", err) } db.Set(r.Context(), ip, resp, 24*time.Hour) - droneBLHits.WithLabelValues(resp.String()).Inc() + asn, asnDesc := asnFromContext(r.Context()) + droneBLHits.WithLabelValues(resp.String(), asn, asnDesc).Inc() } if resp != dnsbl.AllGood { @@ -366,7 +415,7 @@ func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, } func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) { - lg := internal.GetRequestLogger(s.logger, r) + lg, r := s.getRequestLogger(r) localizer := localization.GetLocalizer(r) redir := r.FormValue("redir") @@ -435,11 +484,14 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) { return } lg.Debug("made challenge", "challenge", chall, "rules", rule.Challenge, "cr", cr) - challengesIssued.WithLabelValues("api").Inc() + { + asn, asnDesc := asnFromContext(r.Context()) + challengesIssued.WithLabelValues("api", asn, asnDesc).Inc() + } } func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) { - lg := internal.GetRequestLogger(s.logger, r) + lg, r := s.getRequestLogger(r) localizer := localization.GetLocalizer(r) redir := r.FormValue("redir") @@ -530,7 +582,8 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) { } if err := impl.Validate(r, lg, in); err != nil { - failedValidations.WithLabelValues(rule.Challenge.Algorithm).Inc() + asn, asnDesc := asnFromContext(r.Context()) + failedValidations.WithLabelValues(rule.Challenge.Algorithm, asn, asnDesc).Inc() var cerr *challenge.Error s.ClearCookie(w, CookieOpts{Path: cookiePath, Host: r.Host}) lg.Debug("challenge validate call failed", "err", err) @@ -590,7 +643,10 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) { lg.Debug("can't update information about challenge", "err", err) } - challengesValidated.WithLabelValues(rule.Challenge.Algorithm).Inc() + { + asn, asnDesc := asnFromContext(r.Context()) + challengesValidated.WithLabelValues(rule.Challenge.Algorithm, asn, asnDesc).Inc() + } lg.Debug("challenge passed, redirecting to app") http.Redirect(w, r, redir, http.StatusFound) } @@ -629,7 +685,8 @@ func (s *Server) check(r *http.Request, lg *slog.Logger) (policy.CheckResult, *p return cr("bot/"+b.Name, b.Action, weight), &b, nil case config.RuleWeigh: lg.Debug("adjusting weight", "name", b.Name, "delta", b.Weight.Adjust) - policy.Applications.WithLabelValues("bot/"+b.Name, "WEIGH").Add(1) + asn, asnDesc := asnFromContext(r.Context()) + policy.Applications.WithLabelValues("bot/"+b.Name, "WEIGH", asn, asnDesc).Add(1) weight += b.Weight.Adjust } } diff --git a/lib/config/logging.go b/lib/config/logging.go index f82cac4f..a6499672 100644 --- a/lib/config/logging.go +++ b/lib/config/logging.go @@ -17,6 +17,7 @@ type Logging struct { Sink string `json:"sink"` // Logging sink, either "stdio" or "file" Level *slog.Level `json:"level"` // Log level, if set supersedes the level in flags Parameters *LoggingFileConfig `json:"parameters"` // Logging parameters, to be dynamic in the future + LogASN bool `json:"asn" yaml:"asn"` } const ( diff --git a/lib/http.go b/lib/http.go index a46cd83d..0276282c 100644 --- a/lib/http.go +++ b/lib/http.go @@ -207,7 +207,7 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, cr policy.C return } - lg := internal.GetRequestLogger(s.logger, r) + lg, r := s.getRequestLogger(r) if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && randomChance(64) { lg.Error("client was given a challenge but does not in fact support gzip compression") @@ -215,7 +215,10 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, cr policy.C return } - challengesIssued.WithLabelValues("embedded").Add(1) + { + asn, asnDesc := asnFromContext(r.Context()) + challengesIssued.WithLabelValues("embedded", asn, asnDesc).Add(1) + } chall, err := s.issueChallenge(r.Context(), r, lg, cr, rule) if err != nil { lg.Error("can't get challenge", "err", err) @@ -306,14 +309,14 @@ func (s *Server) constructRedirectURL(r *http.Request) (string, error) { case "http", "https": // allowed default: - lg := internal.GetRequestLogger(s.logger, r) + lg, _ := s.getRequestLogger(r) lg.Warn("invalid protocol in X-Forwarded-Proto", "proto", proto) return "", errors.New(localizer.T("invalid_redirect")) } // Check if host is allowed in RedirectDomains (supports '*' via glob) if len(s.opts.RedirectDomains) > 0 && !matchRedirectDomain(s.opts.RedirectDomains, host) { - lg := internal.GetRequestLogger(s.logger, r) + lg, _ := s.getRequestLogger(r) lg.Debug("domain not allowed", "domain", host) return "", errors.New(localizer.T("redirect_domain_not_allowed")) } @@ -415,7 +418,7 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) { case "", "http", "https": // allowed: empty scheme means relative URL default: - lg := internal.GetRequestLogger(s.logger, r) + lg, _ := s.getRequestLogger(r) lg.Warn("XSS attempt blocked, invalid redirect scheme", "scheme", urlParsed.Scheme, "redir", redir) s.respondWithStatus(w, r, localizer.T("invalid_redirect"), "", http.StatusBadRequest) return @@ -427,7 +430,7 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) { hostMismatch := r.URL.Host != "" && urlParsed.Host != "" && urlParsed.Host != r.URL.Host if hostNotAllowed || hostMismatch { - lg := internal.GetRequestLogger(s.logger, r) + lg, _ := s.getRequestLogger(r) lg.Debug("domain not allowed", "domain", urlParsed.Host) s.respondWithStatus(w, r, localizer.T("redirect_domain_not_allowed"), makeCode(err), http.StatusBadRequest) return @@ -442,7 +445,8 @@ func (s *Server) ServeHTTPNext(w http.ResponseWriter, r *http.Request) { web.Base(localizer.T("you_are_not_a_bot"), web.StaticHappy(localizer), s.policy.Impressum, localizer), ).ServeHTTP(w, r) } else { - requestsProxied.WithLabelValues(r.Host).Inc() + asn, asnDesc := asnFromContext(r.Context()) + requestsProxied.WithLabelValues(r.Host, asn, asnDesc).Inc() r = s.stripBasePrefixFromRequest(r) s.next.ServeHTTP(w, r) } diff --git a/lib/policy/policy.go b/lib/policy/policy.go index 25e2148a..fd38079f 100644 --- a/lib/policy/policy.go +++ b/lib/policy/policy.go @@ -27,7 +27,7 @@ var ( Applications = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "anubis_policy_results", Help: "The results of each policy rule", - }, []string{"rule", "action"}) + }, []string{"rule", "action", "asn", "asn_description"}) ErrChallengeRuleHasWrongAlgorithm = errors.New("config.Bot.ChallengeRules: algorithm is invalid") warnedAboutThresholds = &atomic.Bool{} @@ -47,6 +47,8 @@ type ParsedConfig struct { Dns *dns.Dns Logger *slog.Logger Metrics *config.Metrics + ThothClient *thoth.Client + LogASN bool } func newParsedConfig(orig *config.Config) *ParsedConfig { @@ -70,6 +72,10 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic result := newParsedConfig(c) result.DefaultDifficulty = defaultDifficulty + result.LogASN = c.Logging.LogASN + if hasThothClient { + result.ThothClient = tc + } if c.Logging.Level != nil { logLevel = c.Logging.Level.String() @@ -94,6 +100,10 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic lg := result.Logger.With("at", "config-validate") + if result.LogASN && !hasThothClient { + lg.Warn("logging.asn is enabled but no Thoth client is configured; ASN logging and metrics will be skipped. Please read https://anubis.techaro.lol/docs/admin/thoth for more information") + } + stFac, ok := store.Get(c.Store.Backend) switch ok { case true: