190 lines
4.3 KiB
Python
190 lines
4.3 KiB
Python
"""Classify request traffic from User-Agent and related hints."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from enum import StrEnum
|
|
|
|
|
|
class TrafficType(StrEnum):
|
|
HUMAN = "human"
|
|
AI_BOT = "ai_bot"
|
|
SEARCH_INDEXER = "search_indexer"
|
|
SOCIAL_BOT = "social_bot"
|
|
MONITORING = "monitoring"
|
|
OTHER_BOT = "other_bot"
|
|
UNKNOWN = "unknown"
|
|
|
|
|
|
TRAFFIC_TYPE_LABELS = {
|
|
TrafficType.HUMAN: "Human traffic",
|
|
TrafficType.AI_BOT: "AI bot / AI search",
|
|
TrafficType.SEARCH_INDEXER: "Search indexing",
|
|
TrafficType.SOCIAL_BOT: "Social / preview bot",
|
|
TrafficType.MONITORING: "Monitoring / uptime",
|
|
TrafficType.OTHER_BOT: "Other bot",
|
|
TrafficType.UNKNOWN: "Unknown",
|
|
}
|
|
|
|
# Order matters: first match wins.
|
|
_AI_BOT_PATTERNS = (
|
|
r"GPTBot",
|
|
r"ChatGPT-User",
|
|
r"OAI-SearchBot",
|
|
r"ClaudeBot",
|
|
r"anthropic-ai",
|
|
r"Claude-Web",
|
|
r"Google-Extended",
|
|
r"GoogleOther",
|
|
r"Google-CloudVertexBot",
|
|
r"Bytespider",
|
|
r"CCBot",
|
|
r"Diffbot",
|
|
r"FacebookBot", # Meta AI crawler (distinct from facebookexternalhit)
|
|
r"meta-externalagent",
|
|
r"Meta-ExternalAgent",
|
|
r"PerplexityBot",
|
|
r"Perplexity-User",
|
|
r"YouBot",
|
|
r"Amazonbot",
|
|
r"Applebot-Extended",
|
|
r"cohere-ai",
|
|
r"Cohere-ai",
|
|
r"AI2Bot",
|
|
r"omgili",
|
|
r"ImagesiftBot",
|
|
r"Timpibot",
|
|
r"Webzio-Extended",
|
|
r"DuckAssistBot",
|
|
r"iAskBot",
|
|
r"MistralAI-User",
|
|
r"xAI-Bot",
|
|
r"GrokBot",
|
|
)
|
|
|
|
_SEARCH_INDEXER_PATTERNS = (
|
|
r"Googlebot",
|
|
r"Googlebot-Image",
|
|
r"Googlebot-News",
|
|
r"Googlebot-Video",
|
|
r"Storebot-Google",
|
|
r"AdsBot-Google",
|
|
r"Mediapartners-Google",
|
|
r"Bingbot",
|
|
r"bingbot",
|
|
r"BingPreview",
|
|
r"adidxbot",
|
|
r"DuckDuckBot",
|
|
r"Slurp", # Yahoo
|
|
r"YandexBot",
|
|
r"YandexImages",
|
|
r"Baiduspider",
|
|
r"Sogou",
|
|
r"Applebot",
|
|
r"SeznamBot",
|
|
r"Qwantify",
|
|
r"ecosia",
|
|
r"BraveBot",
|
|
r"PetalBot",
|
|
)
|
|
|
|
_SOCIAL_BOT_PATTERNS = (
|
|
r"facebookexternalhit",
|
|
r"Facebot",
|
|
r"Twitterbot",
|
|
r"LinkedInBot",
|
|
r"Slackbot",
|
|
r"Discordbot",
|
|
r"WhatsApp",
|
|
r"TelegramBot",
|
|
r"Pinterest",
|
|
r"vkShare",
|
|
r"SkypeUriPreview",
|
|
r"redditbot",
|
|
r"Embedly",
|
|
r"Iframely",
|
|
)
|
|
|
|
_MONITORING_PATTERNS = (
|
|
r"UptimeRobot",
|
|
r"Pingdom",
|
|
r"StatusCake",
|
|
r"Site24x7",
|
|
r"Better Uptime",
|
|
r"BetterStack",
|
|
r"Healthchecks",
|
|
r"NewRelic",
|
|
r"Datadog",
|
|
r"Synthetics",
|
|
r"GhostInspector",
|
|
r"HeadlessChrome", # often synthetic monitors
|
|
)
|
|
|
|
_GENERIC_BOT_PATTERNS = (
|
|
r"bot",
|
|
r"crawler",
|
|
r"spider",
|
|
r"scraper",
|
|
r"curl/",
|
|
r"wget/",
|
|
r"python-requests",
|
|
r"Go-http-client",
|
|
r"httpx",
|
|
r"aiohttp",
|
|
r"Java/",
|
|
r"libwww",
|
|
r"scrapy",
|
|
)
|
|
|
|
# Browser-like tokens used to avoid over-classifying humans as bots when UA contains "bot" in odd places.
|
|
_BROWSER_HINTS = (
|
|
r"Mozilla/",
|
|
r"Chrome/",
|
|
r"Safari/",
|
|
r"Firefox/",
|
|
r"Edg/",
|
|
)
|
|
|
|
|
|
def _compile(patterns: tuple[str, ...]) -> re.Pattern[str]:
|
|
return re.compile("|".join(f"(?:{p})" for p in patterns), re.IGNORECASE)
|
|
|
|
|
|
_AI_RE = _compile(_AI_BOT_PATTERNS)
|
|
_SEARCH_RE = _compile(_SEARCH_INDEXER_PATTERNS)
|
|
_SOCIAL_RE = _compile(_SOCIAL_BOT_PATTERNS)
|
|
_MONITOR_RE = _compile(_MONITORING_PATTERNS)
|
|
_GENERIC_BOT_RE = _compile(_GENERIC_BOT_PATTERNS)
|
|
_BROWSER_RE = _compile(_BROWSER_HINTS)
|
|
|
|
|
|
def classify_user_agent(user_agent: str | None) -> TrafficType:
|
|
ua = (user_agent or "").strip()
|
|
if not ua:
|
|
return TrafficType.UNKNOWN
|
|
|
|
if _AI_RE.search(ua):
|
|
return TrafficType.AI_BOT
|
|
if _SEARCH_RE.search(ua):
|
|
return TrafficType.SEARCH_INDEXER
|
|
if _SOCIAL_RE.search(ua):
|
|
return TrafficType.SOCIAL_BOT
|
|
if _MONITOR_RE.search(ua):
|
|
return TrafficType.MONITORING
|
|
if _GENERIC_BOT_RE.search(ua):
|
|
# Some real browsers mention "bot" rarely; prefer human if clearly a browser UA
|
|
# without other bot signals already matched above.
|
|
if _BROWSER_RE.search(ua) and not re.search(
|
|
r"(?:bot|crawler|spider|scraper)", ua, re.IGNORECASE
|
|
):
|
|
return TrafficType.HUMAN
|
|
return TrafficType.OTHER_BOT
|
|
return TrafficType.HUMAN
|
|
|
|
|
|
def traffic_type_label(value: str) -> str:
|
|
try:
|
|
return TRAFFIC_TYPE_LABELS[TrafficType(value)]
|
|
except ValueError:
|
|
return value
|