Merge master and keep both the webhook proxy and crawler file locations.

Co-authored-by: Cursor <cursoragent@cursor.com>

inventory/group_vars/all.yml

@@ -123,6 +123,8 @@ app_catalog:
     # to chat_backend so POST is not answered 405 by the SPA index.html.
     api_backend: chat_backend
     api_proxy_prefix: /api/finance/webhooks/
+    # /robots.txt, /sitemap.xml, and /llms.txt must not fall through to index.html.
+    exact_root_files: true
   chat_backend:
     type: django
     repo: "{{ git_base_url }}/ai_ml_operations/chat_backend.git"

roles/web-static/templates/nginx.conf.j2

@@ -58,6 +58,22 @@ server {
 {%- endif %}
 {%- endif %}
 
+{% if app_catalog[a.name].exact_root_files | default(false) %}
+    # Crawlers must get these files, never the SPA index.html fallback.
+    location = /robots.txt {
+        default_type text/plain;
+        try_files /robots.txt =404;
+    }
+    location = /sitemap.xml {
+        default_type application/xml;
+        try_files /sitemap.xml =404;
+    }
+    location = /llms.txt {
+        default_type text/plain;
+        try_files /llms.txt =404;
+    }
+{% endif %}
+
 {% if app_catalog[a.name].error_page_404 | default('') %}
     error_page 404 {{ app_catalog[a.name].error_page_404 }};