monica_site/site/messaging/webhooks.py

"""SMTP2GO email/SMS event webhooks → Message + ProviderEvent updates."""

from __future__ import annotations

import json
import logging
import uuid
from typing import Any

from django.core.exceptions import ValidationError
from django.http import HttpRequest

from contacts.models import Channel, Contact
from messaging.models import Message, ProviderEvent
from messaging.services import set_channel_consent

logger = logging.getLogger(__name__)

PROVIDER_EMAIL = "smtp2go_email"
PROVIDER_SMS = "smtp2go_sms"
PROVIDER_PCM = "pcm"
PROVIDER = PROVIDER_EMAIL  # backward-compatible alias

# Do not move a message backward to a weaker delivery / engagement state.
_STATUS_RANK = {
    Message.Status.DRAFT: 0,
    Message.Status.SCHEDULED: 1,
    Message.Status.QUEUED: 2,
    Message.Status.SENT: 3,
    Message.Status.FAILED: 3,
    Message.Status.DELIVERED: 4,
    Message.Status.OPENED: 5,
    Message.Status.CLICKED: 6,
    Message.Status.BOUNCED: 7,
    Message.Status.SUPPRESSED: 7,
}

_ENGAGED_OR_DELIVERED = frozenset(
    {
        Message.Status.DELIVERED,
        Message.Status.OPENED,
        Message.Status.CLICKED,
    }
)
_SENT_OR_BETTER = frozenset(
    {
        Message.Status.SENT,
        Message.Status.DELIVERED,
        Message.Status.OPENED,
        Message.Status.CLICKED,
    }
)

_MONICA_HEADER_KEYS = (
    "X-Monica-Message-Id",
    "x-monica-message-id",
    "X_Monica_Message_Id",
    "monica-message-id",
)

# SMTP2GO UI labels → canonical event strings from their docs.
_EMAIL_EVENT_ALIASES = {
    "bounced": "bounce",
    "rejected": "reject",
    "opened": "open",
    "clicked": "click",
    "unsubscribed": "unsubscribe",
    "resubscribed": "resubscribe",
}


def _as_str(value: Any) -> str:
    """Coerce webhook field values to a stripped string (lists / None safe)."""
    if value is None:
        return ""
    if isinstance(value, (list, tuple)):
        if not value:
            return ""
        value = value[0]
    if isinstance(value, bytes):
        value = value.decode("utf-8", errors="replace")
    return str(value).strip()


def _json_safe(value: Any) -> Any:
    """Ensure ProviderEvent.payload can be stored as JSON."""
    if value is None or isinstance(value, (str, int, float, bool)):
        return value
    if isinstance(value, dict):
        return {str(k): _json_safe(v) for k, v in value.items()}
    if isinstance(value, (list, tuple)):
        return [_json_safe(v) for v in value]
    return str(value)


def _normalize_email_event(event: str) -> str:
    event = (event or "").strip().lower()
    return _EMAIL_EVENT_ALIASES.get(event, event)


def parse_webhook_payload(request: HttpRequest) -> dict[str, Any]:
    """Accept JSON or form-encoded SMTP2GO webhook bodies."""
    content_type = (request.content_type or "").lower()
    if "application/json" in content_type:
        try:
            data = json.loads(request.body.decode() or "{}")
        except (json.JSONDecodeError, UnicodeDecodeError):
            return {}
        return data if isinstance(data, dict) else {}
    # Form-encoded (SMTP2GO default)
    return {key: request.POST.get(key) for key in request.POST.keys()}


def extract_monica_message_id(payload: dict[str, Any]) -> str:
    """Pull our correlation id from flat keys or a nested headers object."""
    for key in _MONICA_HEADER_KEYS:
        value = _as_str(payload.get(key))
        if value:
            return value

    headers = payload.get("headers") or payload.get("email_headers") or {}
    if isinstance(headers, dict):
        for key in _MONICA_HEADER_KEYS:
            value = _as_str(headers.get(key))
            if value:
                return value
        # Case-insensitive scan
        lower_map = {str(k).lower(): v for k, v in headers.items()}
        for key in _MONICA_HEADER_KEYS:
            value = _as_str(lower_map.get(key.lower()))
            if value:
                return value
    elif isinstance(headers, list):
        # Some ESP shapes send [["X-Monica-Message-Id", "..."], ...]
        for item in headers:
            if isinstance(item, (list, tuple)) and len(item) >= 2:
                if _as_str(item[0]).lower() in {
                    k.lower() for k in _MONICA_HEADER_KEYS
                }:
                    value = _as_str(item[1])
                    if value:
                        return value
            elif isinstance(item, str) and ":" in item:
                name, _, rest = item.partition(":")
                if name.strip().lower() in {k.lower() for k in _MONICA_HEADER_KEYS}:
                    value = rest.strip()
                    if value:
                        return value
    return ""


def _message_by_pk(pk: str) -> Message | None:
    """Lookup Message by UUID pk without raising on malformed ids."""
    try:
        uuid.UUID(str(pk))
    except (ValueError, AttributeError, TypeError):
        return None
    try:
        return (
            Message.objects.select_related("contact", "campaign")
            .filter(pk=pk)
            .first()
        )
    except (ValidationError, ValueError):
        return None


def find_message_for_email_event(payload: dict[str, Any]) -> Message | None:
    monica_id = extract_monica_message_id(payload)
    if monica_id:
        message = _message_by_pk(monica_id)
        if message:
            return message

    email_id = _as_str(payload.get("email_id") or payload.get("email-id"))
    if email_id:
        message = (
            Message.objects.select_related("contact", "campaign")
            .filter(provider_message_id=email_id)
            .first()
        )
        if message:
            return message

    rcpt = _as_str(payload.get("rcpt")).lower()
    if not rcpt:
        recipients = payload.get("recipients")
        if isinstance(recipients, str) and recipients.strip():
            rcpt = recipients.split(",")[0].strip().lower()
        elif isinstance(recipients, list) and recipients:
            rcpt = _as_str(recipients[0]).lower()

    if not rcpt:
        return None

    contact = Contact.objects.filter(email__iexact=rcpt).first()
    if not contact:
        return None

    return (
        Message.objects.select_related("contact", "campaign")
        .filter(
            contact=contact,
            channel=Channel.EMAIL,
            status__in=[
                Message.Status.QUEUED,
                Message.Status.SENT,
                Message.Status.DELIVERED,
                Message.Status.OPENED,
                Message.Status.CLICKED,
                Message.Status.FAILED,
                Message.Status.BOUNCED,
            ],
        )
        .order_by("-sent_at", "-updated_at")
        .first()
    )


def _maybe_upgrade_status(message: Message, new_status: str, *, error: str = "") -> None:
    current_rank = _STATUS_RANK.get(message.status, 0)
    new_rank = _STATUS_RANK.get(new_status, 0)
    # Always allow bounce/suppress to overwrite delivered; allow delivered over sent.
    if new_rank < current_rank and new_status not in {
        Message.Status.BOUNCED,
        Message.Status.SUPPRESSED,
        Message.Status.FAILED,
    }:
        return
    if message.status in {Message.Status.BOUNCED, Message.Status.SUPPRESSED} and new_status in {
        Message.Status.SENT,
        Message.Status.DELIVERED,
        Message.Status.OPENED,
        Message.Status.CLICKED,
    }:
        return

    fields = ["status", "updated_at"]
    message.status = new_status
    if error:
        message.error = error[:2000]
        fields.append("error")
    elif new_status in _ENGAGED_OR_DELIVERED:
        message.error = ""
        fields.append("error")
    message.save(update_fields=fields)


def _apply_email_event(message: Message, event: str, payload: dict[str, Any]) -> None:
    event = _normalize_email_event(event)
    bounce_kind = _as_str(payload.get("bounce")).lower()
    err = _as_str(payload.get("message") or payload.get("context"))

    email_id = _as_str(payload.get("email_id") or payload.get("email-id"))
    if email_id and message.provider_message_id != email_id:
        message.provider_message_id = email_id
        message.provider = PROVIDER_EMAIL
        message.save(
            update_fields=["provider_message_id", "provider", "updated_at"]
        )

    if event == "processed":
        if message.status in {Message.Status.QUEUED, Message.Status.DRAFT}:
            _maybe_upgrade_status(message, Message.Status.SENT)
        return

    if event == "delivered":
        _maybe_upgrade_status(message, Message.Status.DELIVERED)
        return

    if event == "bounce":
        status = Message.Status.BOUNCED
        _maybe_upgrade_status(
            message,
            status,
            error=err or f"{bounce_kind or 'unknown'} bounce",
        )
        if bounce_kind == "hard":
            set_channel_consent(
                message.contact,
                Channel.EMAIL,
                opted_in=False,
                reason="smtp2go_hard_bounce",
            )
        return

    if event == "reject":
        _maybe_upgrade_status(
            message, Message.Status.FAILED, error=err or "rejected by provider"
        )
        return

    if event == "spam":
        _maybe_upgrade_status(
            message, Message.Status.SUPPRESSED, error=err or "spam complaint"
        )
        set_channel_consent(
            message.contact,
            Channel.EMAIL,
            opted_in=False,
            reason="smtp2go_spam",
        )
        return

    if event == "unsubscribe":
        _maybe_upgrade_status(
            message, Message.Status.SUPPRESSED, error="provider unsubscribe"
        )
        set_channel_consent(
            message.contact,
            Channel.EMAIL,
            opted_in=False,
            reason="smtp2go_unsubscribe",
        )
        return

    if event == "open":
        # Open implies delivery; do not overwrite a stronger click status.
        if message.status != Message.Status.CLICKED:
            _maybe_upgrade_status(message, Message.Status.OPENED)
        return

    if event == "click":
        _maybe_upgrade_status(message, Message.Status.CLICKED)
        return

    # resubscribe — event row only (status unchanged)


def process_smtp2go_email_webhook(payload: dict[str, Any]) -> ProviderEvent | None:
    """
    Persist ProviderEvent and update Message delivery status when possible.

    Returns the stored event (even if message could not be matched).
    """
    event = _normalize_email_event(_as_str(payload.get("event")))
    if not event:
        logger.warning("SMTP2GO webhook missing event: %s", payload)
        return None

    message = find_message_for_email_event(payload)
    if message:
        _apply_email_event(message, event, payload)
        message.refresh_from_db()
    else:
        logger.info(
            "SMTP2GO webhook unmatched event=%s rcpt=%s email_id=%s",
            event,
            payload.get("rcpt"),
            payload.get("email_id"),
        )

    return ProviderEvent.objects.create(
        message=message,
        provider=PROVIDER_EMAIL,
        event_type=event[:64],
        payload=_json_safe(payload) if isinstance(payload, dict) else {},
    )


def normalize_phone(value: str) -> str:
    return "".join(ch for ch in (value or "") if ch.isdigit())


def find_message_for_sms_event(payload: dict[str, Any]) -> Message | None:
    provider_id = (
        payload.get("message_id")
        or payload.get("sms_id")
        or payload.get("id")
        or ""
    )
    provider_id = str(provider_id).strip()
    if provider_id:
        message = (
            Message.objects.select_related("contact", "campaign")
            .filter(channel=Channel.SMS, provider_message_id=provider_id)
            .first()
        )
        if message:
            return message

    raw_phone = (
        payload.get("destination_number")
        or payload.get("to")
        or payload.get("phone")
        or payload.get("from")
        or ""
    )
    digits = normalize_phone(str(raw_phone))
    if len(digits) < 7:
        return None

    # Match last 10 digits so +1 / formatting differences still hit.
    tail = digits[-10:]
    contacts = Contact.objects.exclude(phone="").only("id", "phone")
    contact = None
    for row in contacts.iterator():
        if normalize_phone(row.phone).endswith(tail):
            contact = row
            break
    if not contact:
        return None

    return (
        Message.objects.select_related("contact", "campaign")
        .filter(
            contact=contact,
            channel=Channel.SMS,
            status__in=[
                Message.Status.QUEUED,
                Message.Status.SENT,
                Message.Status.DELIVERED,
                Message.Status.FAILED,
            ],
        )
        .order_by("-sent_at", "-updated_at")
        .first()
    )


def _apply_sms_event(message: Message, event: str, payload: dict[str, Any]) -> None:
    event = (event or "").strip().lower().replace("-", "_")
    err = (
        payload.get("message")
        or payload.get("status_code")
        or payload.get("context")
        or ""
    )
    err = str(err).strip()

    provider_id = (
        payload.get("message_id") or payload.get("sms_id") or ""
    )
    provider_id = str(provider_id).strip()
    if provider_id and message.provider_message_id != provider_id:
        message.provider_message_id = provider_id
        message.provider = PROVIDER_SMS
        message.save(
            update_fields=["provider_message_id", "provider", "updated_at"]
        )

    if event in {"sms_sending", "sending", "sms_submitted", "submitted"}:
        if message.status in {Message.Status.QUEUED, Message.Status.DRAFT}:
            _maybe_upgrade_status(message, Message.Status.SENT)
        return

    if event in {"sms_delivered", "delivered"}:
        _maybe_upgrade_status(message, Message.Status.DELIVERED)
        return

    if event in {"sms_failed", "failed", "sms_rejected", "rejected"}:
        _maybe_upgrade_status(
            message,
            Message.Status.FAILED,
            error=err or event,
        )
        return

    if event in {"sms_opt_out", "opt_out", "optout"}:
        _maybe_upgrade_status(
            message, Message.Status.SUPPRESSED, error="sms opt-out"
        )
        set_channel_consent(
            message.contact,
            Channel.SMS,
            opted_in=False,
            reason="smtp2go_sms_opt_out",
        )
        return


def process_smtp2go_sms_webhook(payload: dict[str, Any]) -> ProviderEvent | None:
    """Persist SMS delivery/opt-out ProviderEvent and update Message when matched."""
    event = (payload.get("event") or "").strip().lower()
    if not event:
        logger.warning("SMTP2GO SMS webhook missing event: %s", payload)
        return None

    message = find_message_for_sms_event(payload)
    if message:
        _apply_sms_event(message, event, payload)
        message.refresh_from_db()
    else:
        # Opt-out with no matched campaign message still suppresses by phone.
        if event.replace("-", "_") in {"sms_opt_out", "opt_out", "optout"}:
            phone = (
                payload.get("destination_number")
                or payload.get("from")
                or payload.get("source_number")
                or ""
            )
            if phone:
                from messaging.services import record_sms_stop

                record_sms_stop(str(phone))
        logger.info(
            "SMTP2GO SMS webhook unmatched event=%s phone=%s message_id=%s",
            event,
            payload.get("destination_number"),
            payload.get("message_id"),
        )

    return ProviderEvent.objects.create(
        message=message,
        provider=PROVIDER_SMS,
        event_type=event,
        payload=payload,
    )


def is_inbound_sms_stop(payload: dict[str, Any]) -> bool:
    """True for gateway-style inbound reply payloads (STOP / UNSUBSCRIBE)."""
    if payload.get("event"):
        return False
    text = (
        payload.get("text")
        or payload.get("message")
        or payload.get("message_content")
        or ""
    )
    text = str(text).strip().upper()
    return text in {"STOP", "UNSUBSCRIBE", "CANCEL", "END", "QUIT"}


def _pcm_event_type(payload: dict[str, Any]) -> str:
    for key in ("event", "eventType", "event_type", "type", "status"):
        value = payload.get(key)
        if value:
            return str(value).strip()
    return "unknown"


def find_message_for_pcm_event(payload: dict[str, Any]) -> Message | None:
    """Correlate PCM webhook to Message via extRefNbr or orderID."""
    ext = (
        payload.get("extRefNbr")
        or payload.get("ext_ref_nbr")
        or payload.get("externalReference")
        or ""
    )
    if not ext and isinstance(payload.get("recipient"), dict):
        ext = payload["recipient"].get("extRefNbr") or ""
    ext = str(ext).strip()
    if ext:
        message = _message_by_pk(ext)
        if message:
            return message

    order_id = (
        payload.get("orderID")
        or payload.get("orderId")
        or payload.get("order_id")
        or ""
    )
    order_id = str(order_id).strip()
    if order_id:
        message = (
            Message.objects.select_related("contact", "campaign")
            .filter(provider_message_id=order_id, channel=Channel.POSTCARD)
            .first()
        )
        if message:
            return message
    return None


def _apply_pcm_status(message: Message, status: str, payload: dict[str, Any]) -> None:
    status_norm = (status or "").strip().lower()
    err = (
        payload.get("message")
        or payload.get("error")
        or payload.get("reason")
        or ""
    )
    err = str(err).strip()

    order_id = (
        payload.get("orderID")
        or payload.get("orderId")
        or payload.get("order_id")
        or ""
    )
    if order_id and message.provider_message_id != str(order_id):
        message.provider_message_id = str(order_id)
        message.provider = PROVIDER_PCM
        message.save(
            update_fields=["provider_message_id", "provider", "updated_at"]
        )

    if status_norm in {"delivered"}:
        _maybe_upgrade_status(message, Message.Status.DELIVERED)
        return
    if status_norm in {"undeliverable", "returned"}:
        _maybe_upgrade_status(
            message,
            Message.Status.BOUNCED,
            error=err or "undeliverable",
        )
        return
    if status_norm in {"canceled", "cancelled"}:
        _maybe_upgrade_status(
            message, Message.Status.FAILED, error=err or "canceled"
        )
        return
    if status_norm in {"pending", "processing", "processed", "mailed", "intransit", "in_transit"}:
        if message.status in {
            Message.Status.QUEUED,
            Message.Status.DRAFT,
            Message.Status.SCHEDULED,
        }:
            _maybe_upgrade_status(message, Message.Status.SENT)
        return


def process_pcm_postcard_webhook(payload: dict[str, Any]) -> ProviderEvent | None:
    """Record a PCM Integrations postcard event and advance Message status."""
    if not payload:
        return None

    # Nested data wrappers some webhook UIs use.
    if "data" in payload and isinstance(payload["data"], dict):
        inner = dict(payload["data"])
        for key in ("event", "eventType", "type"):
            if key in payload and key not in inner:
                inner[key] = payload[key]
        payload = inner

    event_type = _pcm_event_type(payload)
    message = find_message_for_pcm_event(payload)
    if message:
        status_for_apply = (
            payload.get("status")
            or payload.get("orderStatus")
            or event_type
        )
        _apply_pcm_status(message, str(status_for_apply), payload)

    return ProviderEvent.objects.create(
        message=message,
        provider=PROVIDER_PCM,
        event_type=event_type[:64],
        payload=payload,
    )


def _engagement_chart_bars(metrics: list[tuple[str, int]]) -> list[dict]:
    """Build bar heights (percent) for the campaign engagement chart."""
    peak = max((value for _, value in metrics), default=0)
    bars: list[dict] = []
    for label, value in metrics:
        if peak <= 0:
            pct = 12 if value == 0 else 100
        else:
            pct = max(12, int(round((value / peak) * 100))) if value else 8
        bars.append({"label": label, "value": value, "pct": pct})
    return bars


def campaign_engagement_stats(campaign) -> dict:
    """Aggregate delivery + open/click counts for the campaign report."""
    messages_qs = campaign.messages.all()
    statuses = list(messages_qs.values_list("status", flat=True))
    message_ids = list(messages_qs.values_list("pk", flat=True))

    events = ProviderEvent.objects.filter(message_id__in=message_ids)
    open_message_ids = set(
        events.filter(event_type__iexact="open").values_list("message_id", flat=True)
    )
    click_message_ids = set(
        events.filter(event_type__iexact="click").values_list("message_id", flat=True)
    )

    # Status-based engagement also counts (webhook may set opened/clicked).
    status_opened = sum(1 for s in statuses if s in {"opened", "clicked"})
    status_clicked = sum(1 for s in statuses if s == "clicked")
    opens = max(len(open_message_ids), status_opened)
    clicks = max(len(click_message_ids), status_clicked)

    sent = sum(1 for s in statuses if s in _SENT_OR_BETTER)
    delivered = sum(1 for s in statuses if s in _ENGAGED_OR_DELIVERED)
    failed = sum(1 for s in statuses if s in {"failed", "bounced"})
    suppressed = sum(1 for s in statuses if s == "suppressed")

    if campaign.channel == Channel.EMAIL:
        chart_metrics = [
            ("Sent", sent),
            ("Delivered", delivered),
            ("Opens", opens),
            ("Clicks", clicks),
            ("Failed", failed),
        ]
    else:
        chart_metrics = [
            ("Sent", sent),
            ("Delivered", delivered),
            ("Failed", failed),
            ("Suppressed", suppressed),
        ]

    return {
        "total": len(statuses),
        "sent": sent,
        "delivered": delivered,
        "failed": failed,
        "bounced": sum(1 for s in statuses if s == "bounced"),
        "suppressed": suppressed,
        "opens": opens,
        "clicks": clicks,
        "open_events": events.filter(event_type__iexact="open").count(),
        "click_events": events.filter(event_type__iexact="click").count(),
        "chart_bars": _engagement_chart_bars(chart_metrics),
    }