Serve Hesychia crawler files as static text instead of the app shell.

Exact locations return robots.txt, sitemap.xml, and llms.txt, or 404.
Closes #43.

Co-authored-by: Cursor <cursoragent@cursor.com>

inventory/group_vars/all.yml

@@ -119,6 +119,8 @@ app_catalog:
     # Document root served by nginx and written by `npm run build:<env>`.
     # {env} is replaced with the entry's env (beta/prod).
     webroot_pattern: "/var/www/{env}.chat.aimloperations/html"
+    # /robots.txt, /sitemap.xml, and /llms.txt must not fall through to index.html.
+    exact_root_files: true
   chat_backend:
     type: django
     repo: "{{ git_base_url }}/ai_ml_operations/chat_backend.git"

roles/web-static/templates/nginx.conf.j2

@@ -30,6 +30,22 @@ server {
     root {{ (app_catalog[a.name].webroot_pattern | default(web_static_root ~ '/{env}_' ~ a.name)) | replace('{env}', a.env) }};
     index index.html;
 
+{% if app_catalog[a.name].exact_root_files | default(false) %}
+    # Crawlers must get these files, never the SPA index.html fallback.
+    location = /robots.txt {
+        default_type text/plain;
+        try_files /robots.txt =404;
+    }
+    location = /sitemap.xml {
+        default_type application/xml;
+        try_files /sitemap.xml =404;
+    }
+    location = /llms.txt {
+        default_type text/plain;
+        try_files /llms.txt =404;
+    }
+{% endif %}
+
 {% if app_catalog[a.name].error_page_404 | default('') %}
     error_page 404 {{ app_catalog[a.name].error_page_404 }};