"""Classify request traffic from User-Agent and related hints.""" from __future__ import annotations import re from enum import StrEnum class TrafficType(StrEnum): HUMAN = "human" AI_BOT = "ai_bot" SEARCH_INDEXER = "search_indexer" SOCIAL_BOT = "social_bot" MONITORING = "monitoring" OTHER_BOT = "other_bot" UNKNOWN = "unknown" TRAFFIC_TYPE_LABELS = { TrafficType.HUMAN: "Human traffic", TrafficType.AI_BOT: "AI bot / AI search", TrafficType.SEARCH_INDEXER: "Search indexing", TrafficType.SOCIAL_BOT: "Social / preview bot", TrafficType.MONITORING: "Monitoring / uptime", TrafficType.OTHER_BOT: "Other bot", TrafficType.UNKNOWN: "Unknown", } # Order matters: first match wins. _AI_BOT_PATTERNS = ( r"GPTBot", r"ChatGPT-User", r"OAI-SearchBot", r"ClaudeBot", r"anthropic-ai", r"Claude-Web", r"Google-Extended", r"GoogleOther", r"Google-CloudVertexBot", r"Bytespider", r"CCBot", r"Diffbot", r"FacebookBot", # Meta AI crawler (distinct from facebookexternalhit) r"meta-externalagent", r"Meta-ExternalAgent", r"PerplexityBot", r"Perplexity-User", r"YouBot", r"Amazonbot", r"Applebot-Extended", r"cohere-ai", r"Cohere-ai", r"AI2Bot", r"omgili", r"ImagesiftBot", r"Timpibot", r"Webzio-Extended", r"DuckAssistBot", r"iAskBot", r"MistralAI-User", r"xAI-Bot", r"GrokBot", ) _SEARCH_INDEXER_PATTERNS = ( r"Googlebot", r"Googlebot-Image", r"Googlebot-News", r"Googlebot-Video", r"Storebot-Google", r"AdsBot-Google", r"Mediapartners-Google", r"Bingbot", r"bingbot", r"BingPreview", r"adidxbot", r"DuckDuckBot", r"Slurp", # Yahoo r"YandexBot", r"YandexImages", r"Baiduspider", r"Sogou", r"Applebot", r"SeznamBot", r"Qwantify", r"ecosia", r"BraveBot", r"PetalBot", ) _SOCIAL_BOT_PATTERNS = ( r"facebookexternalhit", r"Facebot", r"Twitterbot", r"LinkedInBot", r"Slackbot", r"Discordbot", r"WhatsApp", r"TelegramBot", r"Pinterest", r"vkShare", r"SkypeUriPreview", r"redditbot", r"Embedly", r"Iframely", ) _MONITORING_PATTERNS = ( r"UptimeRobot", r"Pingdom", r"StatusCake", r"Site24x7", r"Better Uptime", r"BetterStack", r"Healthchecks", r"NewRelic", r"Datadog", r"Synthetics", r"GhostInspector", r"HeadlessChrome", # often synthetic monitors ) _GENERIC_BOT_PATTERNS = ( r"bot", r"crawler", r"spider", r"scraper", r"curl/", r"wget/", r"python-requests", r"Go-http-client", r"httpx", r"aiohttp", r"Java/", r"libwww", r"scrapy", ) # Browser-like tokens used to avoid over-classifying humans as bots when UA contains "bot" in odd places. _BROWSER_HINTS = ( r"Mozilla/", r"Chrome/", r"Safari/", r"Firefox/", r"Edg/", ) def _compile(patterns: tuple[str, ...]) -> re.Pattern[str]: return re.compile("|".join(f"(?:{p})" for p in patterns), re.IGNORECASE) _AI_RE = _compile(_AI_BOT_PATTERNS) _SEARCH_RE = _compile(_SEARCH_INDEXER_PATTERNS) _SOCIAL_RE = _compile(_SOCIAL_BOT_PATTERNS) _MONITOR_RE = _compile(_MONITORING_PATTERNS) _GENERIC_BOT_RE = _compile(_GENERIC_BOT_PATTERNS) _BROWSER_RE = _compile(_BROWSER_HINTS) def classify_user_agent(user_agent: str | None) -> TrafficType: ua = (user_agent or "").strip() if not ua: return TrafficType.UNKNOWN if _AI_RE.search(ua): return TrafficType.AI_BOT if _SEARCH_RE.search(ua): return TrafficType.SEARCH_INDEXER if _SOCIAL_RE.search(ua): return TrafficType.SOCIAL_BOT if _MONITOR_RE.search(ua): return TrafficType.MONITORING if _GENERIC_BOT_RE.search(ua): # Some real browsers mention "bot" rarely; prefer human if clearly a browser UA # without other bot signals already matched above. if _BROWSER_RE.search(ua) and not re.search( r"(?:bot|crawler|spider|scraper)", ua, re.IGNORECASE ): return TrafficType.HUMAN return TrafficType.OTHER_BOT return TrafficType.HUMAN def traffic_type_label(value: str) -> str: try: return TRAFFIC_TYPE_LABELS[TrafficType(value)] except ValueError: return value