From dc08229f213ec7a2c090581c9c6c2e67d5e9d169 Mon Sep 17 00:00:00 2001
From: Ryan Westfall <ryan@aimloperations.com>
Date: Mon, 19 Oct 2026 08:07:47 -0700
Subject: [PATCH] Serve Hesychia crawler files as static text (#44)

## Summary
- `chat_web_app` nginx now has exact locations for `/robots.txt`, `/sitemap.xml`, and `/llms.txt`.
- Each location serves the static file or returns 404. It does not fall through to `index.html`.
- Other static apps are unchanged. The flag is `exact_root_files` on `chat_web_app` only.
- Merged current `master`, so the Stripe webhook proxy stays in place next to the new locations.
- Closes #43.

The file contents ship in the chat_web_app public-site PR.

## Test plan
- [ ] Render the web-static nginx template and confirm only `chat_web_app` gets the three exact locations
- [ ] Webhook location `/api/finance/webhooks/` is still present
- [ ] After deploy, `curl -I https://hesychia.ai/robots.txt` is `text/plain`, and a missing file would be 404 rather than HTML

Reviewed-on: https://git.aimloperations.com/ai_ml_operations/server-infra/pulls/44
---
 inventory/group_vars/all.yml             | 15 ++++++++
 roles/web-static/templates/nginx.conf.j2 | 46 ++++++++++++++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml
index 4f8ebde..3c75add 100644
--- a/inventory/group_vars/all.yml
+++ b/inventory/group_vars/all.yml
@@ -123,6 +123,21 @@ app_catalog:
     # to chat_backend so POST is not answered 405 by the SPA index.html.
     api_backend: chat_backend
     api_proxy_prefix: /api/finance/webhooks/
+    # /robots.txt, /sitemap.xml, and /llms.txt must not fall through to index.html.
+    exact_root_files: true
+    # Homepage was missing these. Enforcing CSP stays loose so Tianji, the API,
+    # websockets, and inline styles keep working. Report-Only is the strict policy.
+    security_headers: true
+    # Unknown URLs are HTTP 404 with public/404.html, not 200 + the SPA shell.
+    # Slash routes are directory index.html files from scripts/stampCanonicals.js.
+    # These extensionless routes are the matching files (keep in sync with
+    # llm-fe/src/llm-fe/utils/canonicals.js paths that do not end in /).
+    error_page_404: /404.html
+    html_exact_paths:
+      - /legal/terms-of-service
+      - /billing/success
+      - /billing/cancel
+      - /document_storage
   chat_backend:
     type: django
     repo: "{{ git_base_url }}/ai_ml_operations/chat_backend.git"
diff --git a/roles/web-static/templates/nginx.conf.j2 b/roles/web-static/templates/nginx.conf.j2
index a340b0f..771bb53 100644
--- a/roles/web-static/templates/nginx.conf.j2
+++ b/roles/web-static/templates/nginx.conf.j2
@@ -17,6 +17,14 @@ log_format observability escape=json
     '"user_agent":"$http_user_agent"'
   '}';
 
+{% macro security_headers() %}
+    add_header X-Content-Type-Options "nosniff" always;
+    add_header X-Frame-Options "DENY" always;
+    add_header Referrer-Policy "strict-origin-when-cross-origin" always;
+    add_header Content-Security-Policy "default-src 'self' https: wss: data: blob: 'unsafe-inline'; frame-ancestors 'none'; base-uri 'self'; object-src 'none'" always;
+    add_header Content-Security-Policy-Report-Only "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; object-src 'none'; form-action 'self'" always;
+{% endmacro %}
+
 {% for a in _static_apps %}
 server {
     listen {{ a.port }};
@@ -29,6 +37,9 @@ server {
 
     root {{ (app_catalog[a.name].webroot_pattern | default(web_static_root ~ '/{env}_' ~ a.name)) | replace('{env}', a.env) }};
     index index.html;
+{% if app_catalog[a.name].security_headers | default(false) %}
+{{ security_headers() }}
+{% endif %}
 
 {%- set api_backend_name = app_catalog[a.name].api_backend | default('') %}
 {%- if api_backend_name %}
@@ -58,6 +69,38 @@ server {
 {%- endif %}
 {%- endif %}
 
+{% if app_catalog[a.name].exact_root_files | default(false) %}
+    # Crawlers must get these files, never the SPA index.html fallback.
+    location = /robots.txt {
+        default_type text/plain;
+        try_files /robots.txt =404;
+    }
+    location = /sitemap.xml {
+        default_type application/xml;
+        try_files /sitemap.xml =404;
+    }
+    location = /llms.txt {
+        default_type text/plain;
+        try_files /llms.txt =404;
+    }
+{% endif %}
+
+{% if app_catalog[a.name].html_exact_paths | default([]) %}
+{% for p in app_catalog[a.name].html_exact_paths %}
+    location = {{ p }} {
+        default_type text/html;
+        try_files {{ p }} =404;
+    }
+{% endfor %}
+{% endif %}
+
+{% if a.name == 'chat_web_app' and a.env == 'prod' %}
+    # HeyCatch short links. Unknown paths 404, so the SDK never sees /x.
+    location ~ ^/([a-z0-9])$ {
+        return 302 /?utm_source=heycatch&utm_campaign=$1;
+    }
+{% endif %}
+
 {% if app_catalog[a.name].error_page_404 | default('') %}
     error_page 404 {{ app_catalog[a.name].error_page_404 }};
 
@@ -73,6 +116,9 @@ server {
     location ~* \.(?:js|css|woff2?|png|jpg|jpeg|gif|svg|ico)$ {
         expires 7d;
         add_header Cache-Control "public";
+{% if app_catalog[a.name].security_headers | default(false) %}
+{{ security_headers() }}
+{% endif %}
     }
 }
 {% endfor %}