From b3a6ee0cd04563ee452ca841dec4e9f2d4f1b0a8 Mon Sep 17 00:00:00 2001 From: Ryan Westfall Date: Sun, 9 Aug 2026 10:42:22 -0500 Subject: [PATCH] Improve outreach compose, contact merge, and email assets. Add a Quill email editor with DB-backed image storage, selectable PCM designs, postcard defaults for addressed contacts, and merge-by-phone/address on the contact form. --- site/contacts/services.py | 202 +++++++++++++ site/contacts/templates/contacts/detail.html | 2 +- site/contacts/tests_merge.py | 154 ++++++++++ site/messaging/admin.py | 9 +- site/messaging/migrations/0004_stored_file.py | 34 +++ site/messaging/models.py | 28 ++ site/messaging/providers/email/smtp2go.py | 10 +- site/messaging/services.py | 45 ++- .../templates/messaging/campaign_detail.html | 14 +- .../templates/messaging/campaign_list.html | 246 +++++++++++++--- .../messaging/postcard_designer.html | 3 +- site/messaging/tests.py | 142 +++++++++ site/messaging/urls.py | 10 + site/messaging/views.py | 272 ++++++++++++++---- site/monica_site/settings/base.py | 5 + site/monica_site/settings/dev.py | 3 + site/public/email_branding.py | 63 ++++ site/public/templates/emails/base_email.html | 8 +- site/public/views.py | 37 ++- 19 files changed, 1158 insertions(+), 129 deletions(-) create mode 100644 site/contacts/services.py create mode 100644 site/contacts/tests_merge.py create mode 100644 site/messaging/migrations/0004_stored_file.py diff --git a/site/contacts/services.py b/site/contacts/services.py new file mode 100644 index 0000000..0bf604c --- /dev/null +++ b/site/contacts/services.py @@ -0,0 +1,202 @@ +"""Contact identity matching and merge helpers.""" + +from __future__ import annotations + +import re + +from contacts.models import Contact + +_PHONE_MIN_DIGITS = 10 + + +def normalize_phone_digits(value: str) -> str: + return "".join(ch for ch in (value or "") if ch.isdigit()) + + +def phones_match(a: str, b: str) -> bool: + """True when both phones have ≥10 digits and last-10 match.""" + da = normalize_phone_digits(a) + db = normalize_phone_digits(b) + if len(da) < _PHONE_MIN_DIGITS or len(db) < _PHONE_MIN_DIGITS: + return False + return da[-10:] == db[-10:] + + +def _norm_addr_part(value: str) -> str: + text = (value or "").strip().lower() + text = re.sub(r"[.,#]", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def addresses_match(a: dict | None, b: dict | None) -> bool: + """ + Strong postal match: same street line1 + ZIP, or line1 + city + state. + + Empty / partial addresses never match. + """ + if not a or not b: + return False + line1_a = _norm_addr_part(a.get("line1") or "") + line1_b = _norm_addr_part(b.get("line1") or "") + if not line1_a or not line1_b or line1_a != line1_b: + return False + + zip_a = _norm_addr_part(a.get("zip") or "") + zip_b = _norm_addr_part(b.get("zip") or "") + if zip_a and zip_b: + return zip_a[:5] == zip_b[:5] + + city_a = _norm_addr_part(a.get("city") or "") + city_b = _norm_addr_part(b.get("city") or "") + state_a = _norm_addr_part(a.get("state") or "") + state_b = _norm_addr_part(b.get("state") or "") + if city_a and city_b and state_a and state_b: + return city_a == city_b and state_a == state_b + return False + + +def find_matching_contact( + *, + email: str = "", + phone: str = "", + postal_address: dict | None = None, +) -> tuple[Contact | None, str]: + """ + Resolve an existing contact. + + Priority: email → phone (last 10) → mailing address. + Returns (contact, reason) where reason is email|phone|address|"". + """ + email_norm = (email or "").strip().lower() + if email_norm: + hit = Contact.objects.filter(email__iexact=email_norm).first() + if hit: + return hit, "email" + + phone_digits = normalize_phone_digits(phone) + if len(phone_digits) >= _PHONE_MIN_DIGITS: + tail = phone_digits[-10:] + for row in Contact.objects.exclude(phone="").only("id", "phone").iterator(): + other = normalize_phone_digits(row.phone) + if len(other) >= _PHONE_MIN_DIGITS and other[-10:] == tail: + return Contact.objects.get(pk=row.pk), "phone" + + if Contact.postal_address_has_content(postal_address): + line1 = (postal_address.get("line1") or "").strip() + zip_code = (postal_address.get("zip") or "").strip() + candidates = Contact.objects.exclude(postal_address={}) + if zip_code: + candidates = candidates.filter( + postal_address__zip__istartswith=zip_code[:5] + ) + elif line1: + candidates = candidates.filter(postal_address__line1__iexact=line1) + for row in candidates.iterator(): + if addresses_match(postal_address, row.postal_address): + return row, "address" + + return None, "" + + +def _merge_notes(existing: str, addition: str) -> str: + existing = (existing or "").strip() + addition = (addition or "").strip() + if not addition: + return existing + if not existing: + return addition + if addition in existing: + return existing + return f"{existing}\n{addition}".strip() + + +def upsert_contact( + *, + email: str, + first_name: str = "", + last_name: str = "", + phone: str = "", + postal_address: dict | None = None, + source: str = Contact.Source.CONTACT_FORM, + notes_append: str = "", +) -> tuple[Contact, bool, str]: + """ + Find or create a contact, merging on email / phone / address. + + Returns (contact, created, match_reason). + When merged onto a different email, the submitted email is noted in ``notes``. + """ + email_norm = (email or "").strip().lower() + phone = (phone or "").strip() + postal = postal_address if isinstance(postal_address, dict) else {} + has_postal = Contact.postal_address_has_content(postal) + + existing, reason = find_matching_contact( + email=email_norm, + phone=phone, + postal_address=postal if has_postal else None, + ) + + if existing is None: + contact = Contact.objects.create( + email=email_norm or None, + first_name=(first_name or "").strip(), + last_name=(last_name or "").strip(), + phone=phone, + postal_address=postal if has_postal else {}, + source=source, + notes=(notes_append or "").strip(), + ) + return contact, True, "" + + changed: list[str] = [] + if first_name and first_name.strip(): + existing.first_name = first_name.strip() + changed.append("first_name") + if last_name is not None and str(last_name).strip() != "": + existing.last_name = last_name.strip() + changed.append("last_name") + if phone: + existing.phone = phone + changed.append("phone") + if has_postal: + existing.postal_address = postal + changed.append("postal_address") + + existing_email = (existing.email or "").strip().lower() + if email_norm and email_norm != existing_email: + if not existing_email: + taken = ( + Contact.objects.filter(email__iexact=email_norm) + .exclude(pk=existing.pk) + .exists() + ) + if not taken: + existing.email = email_norm + changed.append("email") + else: + existing.notes = _merge_notes( + existing.notes, + f"Alternate email from form (owned elsewhere): {email_norm}", + ) + changed.append("notes") + else: + existing.notes = _merge_notes( + existing.notes, + f"Alternate email from form: {email_norm}", + ) + changed.append("notes") + + if notes_append: + existing.notes = _merge_notes(existing.notes, notes_append) + changed.append("notes") + + if source and existing.source == Contact.Source.OTHER: + existing.source = source + changed.append("source") + + if changed: + fields = sorted(set(changed) | {"updated_at"}) + existing.save(update_fields=fields) + + return existing, False, reason diff --git a/site/contacts/templates/contacts/detail.html b/site/contacts/templates/contacts/detail.html index 4ebb0e8..91f4790 100644 --- a/site/contacts/templates/contacts/detail.html +++ b/site/contacts/templates/contacts/detail.html @@ -74,7 +74,7 @@
-

Postcard campaigns need a street address and postcard consent. Opt-outs also write a suppression so campaigns skip this contact.

+

Postcard mailings default on when a street address is on file. Uncheck to opt out. Opt-outs also write a suppression so campaigns skip this contact.

diff --git a/site/contacts/tests_merge.py b/site/contacts/tests_merge.py new file mode 100644 index 0000000..b112487 --- /dev/null +++ b/site/contacts/tests_merge.py @@ -0,0 +1,154 @@ +from django.test import Client, TestCase +from django.urls import reverse + +from contacts.models import Contact +from contacts.services import ( + addresses_match, + find_matching_contact, + phones_match, + upsert_contact, +) +from leads.models import Lead + + +class ContactMatchHelpersTests(TestCase): + def test_phones_match_ignores_formatting(self): + self.assertTrue(phones_match("(630) 452-4443", "+1-630-452-4443")) + self.assertFalse(phones_match("6304524443", "6304524444")) + self.assertFalse(phones_match("4524443", "6304524443")) # too short + + def test_addresses_match_by_line1_and_zip(self): + a = Contact.make_postal_address( + line1="123 Main St.", + city="Naperville", + state="IL", + zip_code="60540-1234", + ) + b = Contact.make_postal_address( + line1="123 Main St", + city="Elsewhere", + state="IL", + zip_code="60540", + ) + self.assertTrue(addresses_match(a, b)) + + def test_find_by_phone_then_address(self): + existing = Contact.objects.create( + email="one@example.com", + phone="6305551212", + first_name="Pat", + postal_address=Contact.make_postal_address( + line1="9 Oak Ave", + city="Wheaton", + state="IL", + zip_code="60187", + ), + ) + by_phone, reason = find_matching_contact( + email="other@example.com", + phone="(630) 555-1212", + ) + self.assertEqual(by_phone, existing) + self.assertEqual(reason, "phone") + + by_addr, reason = find_matching_contact( + email="third@example.com", + phone="9995550000", + postal_address=Contact.make_postal_address( + line1="9 Oak Ave", + city="Wheaton", + state="IL", + zip_code="60187", + ), + ) + self.assertEqual(by_addr, existing) + self.assertEqual(reason, "address") + + +class UpsertContactMergeTests(TestCase): + def test_merge_by_phone_keeps_one_contact(self): + original = Contact.objects.create( + email="ryan@example.com", + phone="6305559999", + first_name="Ryan", + ) + contact, created, reason = upsert_contact( + email="ryan.alt@example.com", + first_name="Ryan", + last_name="Westfall", + phone="630-555-9999", + ) + self.assertFalse(created) + self.assertEqual(reason, "phone") + self.assertEqual(contact.pk, original.pk) + self.assertEqual(Contact.objects.count(), 1) + contact.refresh_from_db() + self.assertEqual(contact.email, "ryan@example.com") + self.assertIn("ryan.alt@example.com", contact.notes) + self.assertEqual(contact.last_name, "Westfall") + + def test_merge_by_address(self): + original = Contact.objects.create( + email="home@example.com", + first_name="Sam", + postal_address=Contact.make_postal_address( + line1="100 Lake St", + city="Naperville", + state="IL", + zip_code="60540", + ), + ) + contact, created, reason = upsert_contact( + email="new@example.com", + first_name="Sam", + postal_address=Contact.make_postal_address( + line1="100 Lake St", + city="Naperville", + state="IL", + zip_code="60540", + ), + ) + self.assertFalse(created) + self.assertEqual(reason, "address") + self.assertEqual(contact.pk, original.pk) + self.assertEqual(Contact.objects.count(), 1) + + +class ContactFormMergeTests(TestCase): + def setUp(self): + self.client = Client() + self.existing = Contact.objects.create( + email="primary@example.com", + phone="6301112222", + first_name="Alex", + postal_address=Contact.make_postal_address( + line1="55 River Rd", + city="Aurora", + state="IL", + zip_code="60505", + ), + ) + + def test_contact_form_merges_on_phone(self): + url = reverse("public:contact") + response = self.client.post( + url, + { + "first_name": "Alex", + "last_name": "Lee", + "email": "alt@example.com", + "phone": "(630) 111-2222", + "address_line1": "55 River Rd", + "address_city": "Aurora", + "address_state": "IL", + "address_zip": "60505", + "interest": "buying", + "message": "Looking to buy", + }, + ) + self.assertEqual(response.status_code, 302) + self.assertEqual(Contact.objects.count(), 1) + lead = Lead.objects.get() + self.assertEqual(lead.contact_id, self.existing.pk) + self.assertIn("alt@example.com", lead.message) + self.assertIn("merged by phone", lead.message) diff --git a/site/messaging/admin.py b/site/messaging/admin.py index 4bb0bd8..8488278 100644 --- a/site/messaging/admin.py +++ b/site/messaging/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from messaging.models import Campaign, Message, MessageTemplate, ProviderEvent +from messaging.models import Campaign, Message, MessageTemplate, ProviderEvent, StoredFile @admin.register(MessageTemplate) @@ -38,3 +38,10 @@ class MessageAdmin(admin.ModelAdmin): @admin.register(ProviderEvent) class ProviderEventAdmin(admin.ModelAdmin): list_display = ("provider", "event_type", "created_at") + + +@admin.register(StoredFile) +class StoredFileAdmin(admin.ModelAdmin): + list_display = ("filename", "kind", "content_type", "size", "created_at") + list_filter = ("kind", "content_type") + readonly_fields = ("size", "content_type", "created_at", "updated_at") \ No newline at end of file diff --git a/site/messaging/migrations/0004_stored_file.py b/site/messaging/migrations/0004_stored_file.py new file mode 100644 index 0000000..833155b --- /dev/null +++ b/site/messaging/migrations/0004_stored_file.py @@ -0,0 +1,34 @@ +# Generated by Django 6.1 on 2026-08-09 13:02 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('messaging', '0003_campaign_notify_sent_at'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='StoredFile', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('kind', models.CharField(choices=[('campaign_image', 'Campaign image')], default='campaign_image', max_length=32)), + ('filename', models.CharField(blank=True, max_length=255)), + ('content_type', models.CharField(max_length=128)), + ('size', models.PositiveIntegerField(default=0)), + ('data', models.BinaryField()), + ('uploaded_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='uploaded_files', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + ] diff --git a/site/messaging/models.py b/site/messaging/models.py index fb9cd08..d389f3e 100644 --- a/site/messaging/models.py +++ b/site/messaging/models.py @@ -113,3 +113,31 @@ class ProviderEvent(TimeStampedModel): provider = models.CharField(max_length=64) event_type = models.CharField(max_length=64) payload = models.JSONField(default=dict, blank=True) + + +class StoredFile(UUIDPrimaryKeyModel, TimeStampedModel): + """Binary file blob in the database (no filesystem media storage).""" + + class Kind(models.TextChoices): + CAMPAIGN_IMAGE = "campaign_image", "Campaign image" + + kind = models.CharField( + max_length=32, choices=Kind.choices, default=Kind.CAMPAIGN_IMAGE + ) + filename = models.CharField(max_length=255, blank=True) + content_type = models.CharField(max_length=128) + size = models.PositiveIntegerField(default=0) + data = models.BinaryField() + uploaded_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="uploaded_files", + ) + + class Meta: + ordering = ["-created_at"] + + def __str__(self) -> str: + return self.filename or str(self.pk) diff --git a/site/messaging/providers/email/smtp2go.py b/site/messaging/providers/email/smtp2go.py index 08c2fd9..4f4f2ca 100644 --- a/site/messaging/providers/email/smtp2go.py +++ b/site/messaging/providers/email/smtp2go.py @@ -6,7 +6,11 @@ from django.template.loader import get_template from contacts.models import Channel from messaging.services import one_click_unsubscribe_url, preferences_url -from public.email_branding import email_brand_context, plain_text_to_email_html +from public.email_branding import ( + campaign_body_to_email_html, + campaign_body_to_plain_text, + email_brand_context, +) # Reported back on SMTP2GO webhooks when this header is selected in webhook settings. MONICA_MESSAGE_HEADER = "X-Monica-Message-Id" @@ -33,8 +37,8 @@ def send_email(message) -> str: ctx = email_brand_context( title=subject, - content=body, - content_html=plain_text_to_email_html(body), + content=campaign_body_to_plain_text(body), + content_html=campaign_body_to_email_html(body), prefs_url=prefs_url, one_click_url=one_click_url, ) diff --git a/site/messaging/services.py b/site/messaging/services.py index 2d05357..e12db32 100644 --- a/site/messaging/services.py +++ b/site/messaging/services.py @@ -32,14 +32,29 @@ def contact_may_receive(contact: Contact, channel: str) -> bool: ).exists(): return False consent = ConsentRecord.objects.filter(contact=contact, channel=channel).first() + if channel == Channel.POSTCARD: + # Address on file defaults to postcard-eligible until explicit opt-out. + if consent is None: + return Contact.postal_address_has_content(contact.postal_address) + return bool(consent.opted_in) return bool(consent and consent.opted_in) def channel_preferences(contact: Contact) -> dict[str, bool]: - """Current opt-in flags for every channel (missing record = False).""" + """Current opt-in flags for every channel (missing record = False). + + Postcard: missing consent + postal address → shown as opted in (default). + """ flags = {c.value: False for c in Channel} + seen: set[str] = set() for record in contact.consents.all(): flags[record.channel] = record.opted_in + seen.add(record.channel) + if ( + Channel.POSTCARD not in seen + and Contact.postal_address_has_content(contact.postal_address) + ): + flags[Channel.POSTCARD] = True return flags @@ -165,6 +180,20 @@ def opted_in_contacts(channel: str) -> QuerySet[Contact]: suppressed = Suppression.objects.filter( channel=channel, active=True ).values_list("contact_id", flat=True) + + if channel == Channel.POSTCARD: + # Explicit opt-in, or address on file with no postcard consent row yet. + with_address = Contact.objects.filter( + postal_address__has_key="line1", + ).exclude(postal_address__line1="") + explicit = with_address.filter( + consents__channel=Channel.POSTCARD, + consents__opted_in=True, + ) + implicit = with_address.exclude(consents__channel=Channel.POSTCARD) + qs = (explicit | implicit).exclude(pk__in=suppressed).distinct() + return qs.order_by("first_name", "last_name", "email") + qs = ( Contact.objects.filter( consents__channel=channel, @@ -174,10 +203,6 @@ def opted_in_contacts(channel: str) -> QuerySet[Contact]: .distinct() .order_by("first_name", "last_name", "email") ) - if channel == Channel.POSTCARD: - qs = qs.filter(postal_address__has_key="line1").exclude( - postal_address__line1="" - ) return qs @@ -399,7 +424,11 @@ def send_campaign_test_email(campaign: Campaign, to_email: str) -> None: from django.core.mail import EmailMultiAlternatives from django.template.loader import get_template - from public.email_branding import email_brand_context, plain_text_to_email_html + from public.email_branding import ( + campaign_body_to_email_html, + campaign_body_to_plain_text, + email_brand_context, + ) if campaign.channel != Channel.EMAIL: raise ValueError("Test send is only available for email campaigns.") @@ -420,9 +449,9 @@ def send_campaign_test_email(campaign: Campaign, to_email: str) -> None: ) ctx = email_brand_context( title=f"[TEST] {subject}", - content=f"{body}\n\n{notice}", + content=f"{campaign_body_to_plain_text(body)}\n\n{notice}", content_html=( - f"{plain_text_to_email_html(body)}" + f"{campaign_body_to_email_html(body)}" f'

{notice}

' ), ) diff --git a/site/messaging/templates/messaging/campaign_detail.html b/site/messaging/templates/messaging/campaign_detail.html index 9575e8c..b7defb8 100644 --- a/site/messaging/templates/messaging/campaign_detail.html +++ b/site/messaging/templates/messaging/campaign_detail.html @@ -108,7 +108,7 @@ {% if event.message %}{{ event.message.contact }}{% else %}—{% endif %} {% empty %} - No provider events yet. Configure the SMTP2GO webhook after first send. + No webhook events yet. SMTP2GO must POST opens/clicks to /portal/messaging/webhooks/email/ (see messaging README). SMTP2GO’s own “Clicked” feed does not fill this table by itself. {% endfor %} @@ -126,7 +126,12 @@ {% for message in messages %} - {{ message.contact }} + +
{{ message.contact }}
+ {% if message.destination %} +
{{ message.destination }}
+ {% endif %} + {{ message.get_status_display }} {{ message.provider_message_id|default:"—" }} {{ message.error|truncatechars:60|default:"—" }} @@ -165,8 +170,11 @@ body.innerHTML = 'No messages on this campaign.'; } else { body.innerHTML = data.messages.map(function (m) { + var dest = m.destination + ? '
' + esc(m.destination) + '
' + : ''; return "" + - "" + esc(m.contact) + "" + + "
" + esc(m.contact) + "
" + dest + "" + "" + esc(m.status_display) + "" + "" + esc(m.provider_message_id || "—") + "" + "" + esc(m.error || "—") + ""; diff --git a/site/messaging/templates/messaging/campaign_list.html b/site/messaging/templates/messaging/campaign_list.html index 2a575ff..3a327d9 100644 --- a/site/messaging/templates/messaging/campaign_list.html +++ b/site/messaging/templates/messaging/campaign_list.html @@ -1,11 +1,22 @@ {% extends "portal_base.html" %} {% block title %}Campaigns · Portal{% endblock %} {% block topbar_title %}Campaign composer{% endblock %} +{% block extra_head %} + + +{% endblock %} {% block portal_content %} -
- Email - SMS - Postcard +
@@ -26,7 +37,7 @@
-
+
Email only — ignored for SMS / postcard
-
- - +
Bold, fonts, sizes, links, images · merge tags: {first_name}, {last_name}
+
+
- + -
Open postcard designer
+
Open postcard designer to create or edit designs
@@ -83,7 +100,7 @@
Preview updates as you type.

After send → open the campaign report for delivery & engagement.

@@ -122,37 +139,172 @@
{% endblock %} {% block extra_js %} + {% endblock %} diff --git a/site/messaging/templates/messaging/postcard_designer.html b/site/messaging/templates/messaging/postcard_designer.html index 59eda7f..6f4462c 100644 --- a/site/messaging/templates/messaging/postcard_designer.html +++ b/site/messaging/templates/messaging/postcard_designer.html @@ -81,7 +81,8 @@

- Saved templates appear when composing a postcard campaign. + Designs listed above are selectable in Campaigns when Recipients is postcard. + Saving as a template keeps a named local copy.

diff --git a/site/messaging/tests.py b/site/messaging/tests.py index a012970..2de7874 100644 --- a/site/messaging/tests.py +++ b/site/messaging/tests.py @@ -5,9 +5,11 @@ from django.urls import reverse from contacts.models import Channel, ConsentRecord, Contact, Suppression from messaging.models import Campaign, Message, MessageTemplate, ProviderEvent from messaging.services import ( + channel_preferences, contact_may_receive, create_campaign_draft, make_unsubscribe_token, + opted_in_contacts, set_channel_consent, ) @@ -757,3 +759,143 @@ class PcmAuthTests(TestCase): with self.assertRaises(pcm_mod.PcmApiError) as ctx: pcm_mod.login(force=True) self.assertIn("PCM_API_SECRET", str(ctx.exception)) + + +class PostcardAddressDefaultConsentTests(TestCase): + def test_address_without_consent_is_postcard_eligible(self): + contact = Contact.objects.create( + email="addr@example.com", + first_name="Ann", + postal_address=Contact.make_postal_address( + line1="1 Oak St", + city="Naperville", + state="IL", + zip_code="60540", + ), + ) + self.assertTrue(contact_may_receive(contact, Channel.POSTCARD)) + prefs = channel_preferences(contact) + self.assertTrue(prefs[Channel.POSTCARD]) + self.assertEqual(opted_in_contacts(Channel.POSTCARD).count(), 1) + + def test_explicit_postcard_opt_out_respected(self): + contact = Contact.objects.create( + email="out@example.com", + postal_address=Contact.make_postal_address(line1="2 Oak St"), + ) + set_channel_consent( + contact, Channel.POSTCARD, opted_in=False, reason="opt_out" + ) + self.assertFalse(contact_may_receive(contact, Channel.POSTCARD)) + self.assertEqual(opted_in_contacts(Channel.POSTCARD).count(), 0) + + +class CampaignHtmlBodyTests(TestCase): + def test_html_body_preserved_in_email(self): + from django.core import mail + + User = get_user_model() + user = User.objects.create_user( + username="htmlsender", password="test-pass-123", email="s@example.com" + ) + contact = Contact.objects.create(email="pat@example.com", first_name="Pat") + set_channel_consent(contact, Channel.EMAIL, opted_in=True, reason="test") + campaign = create_campaign_draft( + name="HTML tip", + audience=Campaign.Audience.EMAIL_OPT_IN, + subject="Styled", + body='

Hello

', + created_by=user, + ) + from messaging.providers.email.smtp2go import send_email + + send_email(campaign.messages.get()) + self.assertEqual(len(mail.outbox), 1) + html = mail.outbox[0].alternatives[0][0] + self.assertIn("Hello", html) + self.assertIn('src="https://example.com/a.png"', html) + + +class PostcardDesignPickTests(TestCase): + def setUp(self): + User = get_user_model() + self.user = User.objects.create_user( + username="pcm-pick", password="test-pass-123" + ) + self.client = Client() + self.client.login(username="pcm-pick", password="test-pass-123") + self.contact = Contact.objects.create( + email="mail@example.com", + postal_address=Contact.make_postal_address(line1="9 Main"), + ) + + def test_compose_can_pick_pcm_design_id(self): + from unittest.mock import patch + + with patch( + "messaging.views._fetch_pcm_designs", + return_value=( + [{"design_id": "42", "name": "Spring card", "size": "46"}], + "", + ), + ): + url = reverse("messaging:campaign_list") + response = self.client.post( + url, + { + "name": "Mailer", + "subject": "", + "body": "", + "audience": Campaign.Audience.POSTCARD_OPT_IN, + "template_id": "d:42", + }, + ) + campaign = Campaign.objects.get(name="Mailer") + self.assertEqual(response.status_code, 302) + self.assertEqual(campaign.channel, Channel.POSTCARD) + self.assertIsNotNone(campaign.template_id) + self.assertEqual( + campaign.template.postcard_front.get("design_id"), 42 + ) + self.assertEqual(campaign.messages.count(), 1) + + +class StoredFileUploadTests(TestCase): + def setUp(self): + User = get_user_model() + self.user = User.objects.create_user( + username="uploader", password="test-pass-123" + ) + self.client = Client() + self.client.login(username="uploader", password="test-pass-123") + + def test_upload_stores_bytes_in_database(self): + from django.core.files.uploadedfile import SimpleUploadedFile + + from messaging.models import StoredFile + + png = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" + b"\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS\xde\x00\x00" + b"\x00\x0cIDATx\x9cc\xf8\x0f\x00\x00\x01\x01\x00\x05\x18" + b"\xd8N\x00\x00\x00\x00IEND\xaeB`\x82" + ) + upload = SimpleUploadedFile("dot.png", png, content_type="image/png") + response = self.client.post( + reverse("messaging:campaign_image_upload"), + {"image": upload}, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertIn("url", payload) + stored = StoredFile.objects.get() + self.assertEqual(bytes(stored.data), png) + self.assertEqual(stored.content_type, "image/png") + self.assertEqual(stored.uploaded_by, self.user) + + fetch = self.client.get( + reverse("messaging:stored_file", kwargs={"pk": stored.pk}) + ) + self.assertEqual(fetch.status_code, 200) + self.assertEqual(fetch["Content-Type"], "image/png") + self.assertEqual(b"".join(fetch.streaming_content), png) diff --git a/site/messaging/urls.py b/site/messaging/urls.py index 4827c09..2788a3a 100644 --- a/site/messaging/urls.py +++ b/site/messaging/urls.py @@ -18,6 +18,16 @@ urlpatterns = [ views.campaign_test_send, name="campaign_test_send", ), + path( + "campaigns/upload-image/", + views.campaign_image_upload, + name="campaign_image_upload", + ), + path( + "files//", + views.stored_file, + name="stored_file", + ), path("postcard/", views.postcard_designer, name="postcard_designer"), path( "postcard/create/", diff --git a/site/messaging/views.py b/site/messaging/views.py index ca35c89..8fc16b5 100644 --- a/site/messaging/views.py +++ b/site/messaging/views.py @@ -1,5 +1,6 @@ import hashlib import hmac +import io import logging from django.conf import settings @@ -7,18 +8,19 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.http import HttpResponseForbidden, JsonResponse +from django.http import FileResponse, HttpResponseForbidden, JsonResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_http_methods, require_POST from contacts.models import Channel -from messaging.models import Campaign, MessageTemplate, ProviderEvent +from messaging.models import Campaign, MessageTemplate, ProviderEvent, StoredFile from messaging.providers.postcard.pcm import ( PCM_SIZE_CHOICES, PcmApiError, create_custom_design, + design_id_from_template, get_design_embed_url, list_designs, ) @@ -41,6 +43,11 @@ from messaging.webhooks import ( logger = logging.getLogger(__name__) +_ALLOWED_IMAGE_TYPES = frozenset( + {"image/jpeg", "image/png", "image/gif", "image/webp"} +) +_MAX_IMAGE_BYTES = 5 * 1024 * 1024 + def _audience_choices() -> list[tuple[str, str]]: """Labeled audience options with live opted-in counts.""" @@ -65,8 +72,165 @@ def _postcard_templates(): )[:50] +def _fetch_pcm_designs() -> tuple[list[dict], str]: + """Return (normalized design rows, api_error).""" + designs: list[dict] = [] + api_error = "" + try: + remote = list_designs(product_type="postcard") + for item in remote: + if not isinstance(item, dict): + continue + did = item.get("designID") or item.get("design_id") or item.get("id") + if did is None: + continue + size_info = item.get("size") or {} + size_key = ( + size_info.get("key") + if isinstance(size_info, dict) + else size_info + ) or "" + designs.append( + { + "design_id": str(did), + "name": item.get("friendlyName") + or item.get("name") + or f"Design {did}", + "size": str(size_key), + } + ) + except PcmApiError as exc: + api_error = str(exc) + + seen = {d["design_id"] for d in designs} + for tmpl in _postcard_templates(): + front = tmpl.postcard_front or {} + did = front.get("design_id") + if did is None: + continue + did_s = str(did) + if did_s in seen: + continue + designs.insert( + 0, + { + "design_id": did_s, + "name": tmpl.name, + "size": str(front.get("size") or ""), + }, + ) + seen.add(did_s) + return designs, api_error + + +def _postcard_design_choices() -> list[dict]: + """Options for campaign compose: PCM designs + saved templates.""" + designs, _ = _fetch_pcm_designs() + by_id = {d["design_id"]: d for d in designs} + choices: list[dict] = [] + for tmpl in _postcard_templates(): + did = design_id_from_template(tmpl) + if did is None: + continue + did_s = str(did) + choices.append( + { + "value": f"t:{tmpl.pk}", + "label": f"{tmpl.name} (design {did_s})", + "design_id": did_s, + } + ) + by_id.pop(did_s, None) + for did_s, d in by_id.items(): + choices.append( + { + "value": f"d:{did_s}", + "label": f"{d['name']} (design {did_s})", + "design_id": did_s, + "size": d.get("size") or "46", + "name": d["name"], + } + ) + return choices + + +def _resolve_postcard_template(raw: str) -> MessageTemplate | None: + """Resolve compose select value ``t:`` or ``d:``.""" + value = (raw or "").strip() + if not value: + return None + if value.startswith("t:"): + return MessageTemplate.objects.filter( + pk=value[2:], channel=Channel.POSTCARD + ).first() + if value.startswith("d:"): + design_raw = value[2:].strip() + try: + design_id = int(design_raw) + except ValueError: + return None + for tmpl in MessageTemplate.objects.filter(channel=Channel.POSTCARD): + if design_id_from_template(tmpl) == design_id: + return tmpl + name = f"PCM design {design_id}" + designs, _ = _fetch_pcm_designs() + match = next( + (d for d in designs if d["design_id"] == str(design_id)), None + ) + size = (match or {}).get("size") or "46" + if match and match.get("name"): + name = match["name"] + front = { + "design_id": design_id, + "size": size, + "name": name, + "provider": "pcm", + } + return MessageTemplate.objects.create( + channel=Channel.POSTCARD, + name=name[:120], + subject="", + body=f"PCM design {design_id}", + postcard_front=front, + postcard_back={}, + ) + # Legacy: bare MessageTemplate pk + return MessageTemplate.objects.filter( + pk=value, channel=Channel.POSTCARD + ).first() + + +def _format_postal_address(addr: dict | None) -> str: + if not addr: + return "" + line1 = (addr.get("line1") or "").strip() + line2 = (addr.get("line2") or "").strip() + city = (addr.get("city") or "").strip() + state = (addr.get("state") or "").strip() + zip_code = (addr.get("zip") or "").strip() + city_line = ", ".join(p for p in (city, state) if p) + if zip_code: + city_line = f"{city_line} {zip_code}".strip() + return ", ".join(p for p in (line1, line2, city_line) if p) + + +def _message_destination(message) -> str: + """Channel-specific destination shown on the recipients table.""" + contact = message.contact + channel = message.channel or (message.campaign.channel if message.campaign_id else "") + if channel == Channel.EMAIL: + return (contact.email or "").strip() + if channel == Channel.SMS: + return (contact.phone or "").strip() + if channel == Channel.POSTCARD: + return _format_postal_address(contact.postal_address) + return "" + + def _campaign_report(campaign: Campaign) -> dict: messages_qs = list(campaign.messages.select_related("contact").all()[:200]) + for msg in messages_qs: + msg.destination = _message_destination(msg) stats = campaign_engagement_stats(campaign) recent_events = ( ProviderEvent.objects.filter(message__campaign=campaign) @@ -168,7 +332,7 @@ def campaign_list(request): template = None if template_id: - template = MessageTemplate.objects.filter(pk=template_id).first() + template = _resolve_postcard_template(template_id) if not name: form_errors.append("Campaign name is required.") @@ -177,7 +341,7 @@ def campaign_list(request): if audience == Campaign.Audience.POSTCARD_OPT_IN: if not template or template.channel != Channel.POSTCARD: form_errors.append( - "Choose a saved postcard template (design it under Postcard first)." + "Choose a postcard design (create one under Postcard design)." ) if not body: body = "Postcard mailing" @@ -222,9 +386,10 @@ def campaign_list(request): { "campaigns": campaigns, "audience_choices": _audience_choices(), - "postcard_templates": _postcard_templates(), + "postcard_designs": _postcard_design_choices(), "form_data": form_data, "form_errors": form_errors, + "image_upload_url": reverse("messaging:campaign_image_upload"), }, ) @@ -274,6 +439,7 @@ def campaign_status_json(request, pk): { "id": str(m.pk), "contact": str(m.contact), + "destination": getattr(m, "destination", "") or "", "status": m.status, "status_display": m.get_status_display(), "provider_message_id": m.provider_message_id or "", @@ -340,62 +506,62 @@ def campaign_test_send(request, pk): return redirect("messaging:campaign_detail", pk=campaign.pk) +@login_required +@require_POST +def campaign_image_upload(request): + """Upload an image for the email rich editor; store bytes in the DB.""" + upload = request.FILES.get("image") or request.FILES.get("file") + if not upload: + return JsonResponse({"error": "No image uploaded."}, status=400) + content_type = (getattr(upload, "content_type", None) or "").lower() + if content_type not in _ALLOWED_IMAGE_TYPES: + return JsonResponse( + {"error": "Use a JPEG, PNG, GIF, or WebP image."}, status=400 + ) + if upload.size and upload.size > _MAX_IMAGE_BYTES: + return JsonResponse({"error": "Image must be 5 MB or smaller."}, status=400) + + data = upload.read() + if len(data) > _MAX_IMAGE_BYTES: + return JsonResponse({"error": "Image must be 5 MB or smaller."}, status=400) + + original = (getattr(upload, "name", None) or "image")[:255] + stored = StoredFile.objects.create( + kind=StoredFile.Kind.CAMPAIGN_IMAGE, + filename=original, + content_type=content_type, + size=len(data), + data=data, + uploaded_by=request.user if request.user.is_authenticated else None, + ) + path = reverse("messaging:stored_file", kwargs={"pk": stored.pk}) + url = request.build_absolute_uri(path) + return JsonResponse({"url": url, "id": str(stored.pk)}) + + +@require_GET +def stored_file(request, pk): + """Public fetch for email clients / preview (UUID acts as capability token).""" + stored = get_object_or_404(StoredFile, pk=pk) + response = FileResponse( + io.BytesIO(bytes(stored.data)), + content_type=stored.content_type or "application/octet-stream", + ) + if stored.filename: + response["Content-Disposition"] = f'inline; filename="{stored.filename}"' + response["Cache-Control"] = "public, max-age=86400" + return response + + @login_required def postcard_designer(request): """PCM Integrations designer — list designs + embed iframe.""" - api_error = "" - designs: list[dict] = [] + designs, api_error = _fetch_pcm_designs() embed_url = "" active_design_id = (request.GET.get("design_id") or "").strip() active_name = "" active_size = "46" - try: - remote = list_designs(product_type="postcard") - for item in remote: - if not isinstance(item, dict): - continue - did = item.get("designID") or item.get("design_id") or item.get("id") - if did is None: - continue - size_info = item.get("size") or {} - size_key = ( - size_info.get("key") - if isinstance(size_info, dict) - else size_info - ) or "" - designs.append( - { - "design_id": str(did), - "name": item.get("friendlyName") - or item.get("name") - or f"Design {did}", - "size": str(size_key), - } - ) - except PcmApiError as exc: - api_error = str(exc) - - # Merge saved local templates that may not appear in the remote page yet. - seen = {d["design_id"] for d in designs} - for tmpl in _postcard_templates(): - front = tmpl.postcard_front or {} - did = front.get("design_id") - if did is None: - continue - did_s = str(did) - if did_s in seen: - continue - designs.insert( - 0, - { - "design_id": did_s, - "name": tmpl.name, - "size": str(front.get("size") or ""), - }, - ) - seen.add(did_s) - if active_design_id: match = next( (d for d in designs if d["design_id"] == active_design_id), None diff --git a/site/monica_site/settings/base.py b/site/monica_site/settings/base.py index c650d83..7a3095f 100644 --- a/site/monica_site/settings/base.py +++ b/site/monica_site/settings/base.py @@ -186,7 +186,12 @@ STATIC_ROOT = BASE_DIR / "staticfiles" STATICFILES_DIRS = [ BASE_DIR / "monica_site" / "static", ] +# Uploaded blobs live in the DB (messaging.StoredFile). Default storage is +# in-memory only so nothing is written to disk accidentally. STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.memory.InMemoryStorage", + }, "staticfiles": { "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", }, diff --git a/site/monica_site/settings/dev.py b/site/monica_site/settings/dev.py index 02e7e2f..696eb2d 100644 --- a/site/monica_site/settings/dev.py +++ b/site/monica_site/settings/dev.py @@ -9,6 +9,9 @@ SITE_UNDER_CONSTRUCTION = env_bool("SITE_UNDER_CONSTRUCTION", False) # noqa: F4 TIANJI_ENABLED = env_bool("TIANJI_ENABLED", False) # noqa: F405 STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.memory.InMemoryStorage", + }, "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, diff --git a/site/public/email_branding.py b/site/public/email_branding.py index 013786a..0eebe79 100644 --- a/site/public/email_branding.py +++ b/site/public/email_branding.py @@ -39,6 +39,27 @@ def _absolute_static_url(site_url: str, relative: str) -> str: return f"{site_url}{path}" +def _tagline_with_site_link(tagline: str, site_url: str) -> str: + """Turn leading MKDRealtor.com (or similar) into a link to the public site.""" + raw = (tagline or "").strip() + if not raw: + return "" + # Match "MKDRealtor.com" (any case) at the start, optional trailing " · rest" + match = re.match( + r"(?i)^(MKDRealtor\.com)(\s*[·•\-–—]\s*.*)?$", + raw, + ) + if not match: + return html.escape(raw) + label = html.escape(match.group(1)) + rest = html.escape(match.group(2) or "") + href = html.escape(site_url, quote=True) + return ( + f'{label}' + f"{rest}" + ) + + def email_brand_context(**extra): site_url = (getattr(settings, "PUBLIC_SITE_URL", None) or "").rstrip("/") if not site_url: @@ -57,6 +78,7 @@ def email_brand_context(**extra): "brand_name": brand_name, "brand_legal": brand_legal, "brand_tagline": tagline, + "brand_tagline_html": _tagline_with_site_link(tagline, site_url), "host_label": host_label, **extra, } @@ -81,3 +103,44 @@ def plain_text_to_email_html(text: str) -> str: f'line-height:1.6;">{joined}

' ) return "\n".join(blocks) + + +_HTML_TAG_RE = re.compile( + r"<\s*(p|div|br|span|strong|em|b|i|u|a|img|h[1-6]|ul|ol|li|font|table)\b", + re.I, +) + + +def sanitize_email_html(raw: str) -> str: + """Light cleanup for staff-authored HTML (Quill) before sending.""" + text = raw or "" + text = re.sub(r"(?is)]*>.*?", "", text) + text = re.sub(r"(?is)]*>.*?", "", text) + text = re.sub(r"(?is)]*>.*?", "", text) + text = re.sub(r"(?i)\son\w+\s*=\s*([\"']).*?\1", "", text) + text = re.sub(r"(?i)\son\w+\s*=\s*[^\s>]+", "", text) + text = re.sub(r"(?i)javascript:", "", text) + return text.strip() + + +def campaign_body_to_email_html(body: str) -> str: + """Render campaign body for email — HTML as-is when Quill markup, else plain.""" + raw = (body or "").strip() + if not raw: + return "" + if _HTML_TAG_RE.search(raw): + return sanitize_email_html(raw) + return plain_text_to_email_html(raw) + + +def campaign_body_to_plain_text(body: str) -> str: + """Plain-text alternative for multipart emails.""" + from django.utils.html import strip_tags + + raw = (body or "").strip() + if not raw: + return "" + if _HTML_TAG_RE.search(raw): + text = strip_tags(sanitize_email_html(raw)) + return html.unescape(re.sub(r"[ \t]+\n", "\n", text)).strip() + return raw diff --git a/site/public/templates/emails/base_email.html b/site/public/templates/emails/base_email.html index ed541c0..14151e7 100644 --- a/site/public/templates/emails/base_email.html +++ b/site/public/templates/emails/base_email.html @@ -73,7 +73,9 @@ {% if brand_name %}

{{ brand_name }}

{% endif %} - {% if brand_tagline %} + {% if brand_tagline_html %} +

{{ brand_tagline_html|safe }}

+ {% elif brand_tagline %}

{{ brand_tagline }}

{% endif %} {% block header_extra %}{% endblock %} @@ -97,10 +99,14 @@ © {% now "Y" %} {{ brand_name|default:"Monica Dhillon" }}. All rights reserved.

+ {% if brand_tagline_html %} + {{ brand_tagline_html|safe }} + {% else %} {{ host_label|default:"mkdrealtor.com" }} {% if brand_tagline %}  ·  {{ brand_tagline }} {% endif %} + {% endif %}

{% endblock %} diff --git a/site/public/views.py b/site/public/views.py index 4bb1ba8..d7a80b9 100644 --- a/site/public/views.py +++ b/site/public/views.py @@ -8,6 +8,7 @@ from django.views.decorators.http import require_GET, require_http_methods from analytics.services import attribute_lead_from_request from contacts.models import Channel, ConsentRecord, Contact +from contacts.services import upsert_contact from leads.models import Lead from messaging.services import ( channel_preferences, @@ -91,12 +92,6 @@ def contact(request): form = ContactForm(request.POST) if form.is_valid(): data = form.cleaned_data - defaults = { - "first_name": data["first_name"], - "last_name": data.get("last_name") or "", - "phone": data.get("phone") or "", - "source": Contact.Source.CONTACT_FORM, - } postal = Contact.make_postal_address( line1=data.get("address_line1") or "", line2=data.get("address_line2") or "", @@ -104,11 +99,16 @@ def contact(request): state=data.get("address_state") or "", zip_code=data.get("address_zip") or "", ) - if Contact.postal_address_has_content(postal): - defaults["postal_address"] = postal - contact_obj, _ = Contact.objects.update_or_create( - email=data["email"].lower(), - defaults=defaults, + submitted_email = data["email"].lower() + contact_obj, _created, match_reason = upsert_contact( + email=submitted_email, + first_name=data["first_name"], + last_name=data.get("last_name") or "", + phone=data.get("phone") or "", + postal_address=postal + if Contact.postal_address_has_content(postal) + else None, + source=Contact.Source.CONTACT_FORM, ) ConsentRecord.objects.update_or_create( contact=contact_obj, @@ -121,11 +121,26 @@ def contact(request): channel=Channel.SMS, defaults={"opted_in": True, "reason": "contact_form"}, ) + if Contact.postal_address_has_content(postal): + ConsentRecord.objects.get_or_create( + contact=contact_obj, + channel=Channel.POSTCARD, + defaults={"opted_in": True, "reason": "contact_form"}, + ) interest = data.get("interest") or "" interest_label = dict(ContactForm.INTEREST_CHOICES).get(interest, interest) body = data.get("message") or "" if interest_label: body = f"Interest: {interest_label}\n\n{body}".strip() + if ( + match_reason in {"phone", "address"} + and (contact_obj.email or "").lower() != submitted_email + ): + body = ( + f"Submitted email: {submitted_email} " + f"(merged by {match_reason} with " + f"{contact_obj.email or 'existing contact'})\n\n{body}" + ).strip() lead = Lead.objects.create( contact=contact_obj, message=body,