- action: WEIGH expression: userAgent.contains("Bingbot") name: robots-txt-policy-crawl-delay-1 weight: adjust: 5 - action: WEIGH expression: userAgent.contains("Googlebot") name: robots-txt-policy-crawl-delay-2 weight: adjust: 5 - action: WEIGH expression: userAgent.contains("SeoBot") name: robots-txt-policy-crawl-delay-3 weight: adjust: 5 - action: WEIGH expression: "true" name: robots-txt-policy-crawl-delay-4 weight: adjust: 5 - action: DENY expression: any: - userAgent.contains("BadBot") - userAgent.contains("SeoBot") name: robots-txt-policy-blacklist-5 - action: CHALLENGE expression: all: - userAgent.contains("TestBot") - path.matches("^/.*/admin") name: robots-txt-policy-disallow-6 - action: CHALLENGE expression: all: - userAgent.contains("Bingbot") - path.startsWith("/admin/") name: robots-txt-policy-disallow-8 - action: CHALLENGE expression: path.startsWith("/api/internal/") name: robots-txt-policy-disallow-9 - action: CHALLENGE expression: all: - userAgent.contains("TestBot") - path.matches("^/file.\\.log") name: robots-txt-policy-disallow-10 - action: CHALLENGE expression: path.startsWith("/private/") name: robots-txt-policy-disallow-11 - action: CHALLENGE expression: all: - userAgent.contains("Googlebot") - path.startsWith("/search/") name: robots-txt-policy-disallow-13 - action: CHALLENGE expression: all: - userAgent.contains("Bingbot") - path.startsWith("/search/") name: robots-txt-policy-disallow-14 - action: CHALLENGE expression: all: - userAgent.contains("TestBot") - path.matches("^/temp.*\\.html") name: robots-txt-policy-disallow-15