From dce0879e62286d3c0857006045bc18b80d0ce0da Mon Sep 17 00:00:00 2001 From: Ryan Westfall Date: Sat, 17 Oct 2026 07:19:05 -0500 Subject: [PATCH 1/2] Serve Hesychia crawler files as static text instead of the app shell. Exact locations return robots.txt, sitemap.xml, and llms.txt, or 404. Closes #43. Co-authored-by: Cursor --- inventory/group_vars/all.yml | 2 ++ roles/web-static/templates/nginx.conf.j2 | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index fa5db9b..4d68ce4 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -119,6 +119,8 @@ app_catalog: # Document root served by nginx and written by `npm run build:`. # {env} is replaced with the entry's env (beta/prod). webroot_pattern: "/var/www/{env}.chat.aimloperations/html" + # /robots.txt, /sitemap.xml, and /llms.txt must not fall through to index.html. + exact_root_files: true chat_backend: type: django repo: "{{ git_base_url }}/ai_ml_operations/chat_backend.git" diff --git a/roles/web-static/templates/nginx.conf.j2 b/roles/web-static/templates/nginx.conf.j2 index 686f33c..22c1c60 100644 --- a/roles/web-static/templates/nginx.conf.j2 +++ b/roles/web-static/templates/nginx.conf.j2 @@ -30,6 +30,22 @@ server { root {{ (app_catalog[a.name].webroot_pattern | default(web_static_root ~ '/{env}_' ~ a.name)) | replace('{env}', a.env) }}; index index.html; +{% if app_catalog[a.name].exact_root_files | default(false) %} + # Crawlers must get these files, never the SPA index.html fallback. + location = /robots.txt { + default_type text/plain; + try_files /robots.txt =404; + } + location = /sitemap.xml { + default_type application/xml; + try_files /sitemap.xml =404; + } + location = /llms.txt { + default_type text/plain; + try_files /llms.txt =404; + } +{% endif %} + {% if app_catalog[a.name].error_page_404 | default('') %} error_page 404 {{ app_catalog[a.name].error_page_404 }}; -- 2.54.0 From 5e9878632eee7e7220ee93beb73e7bffff56ef89 Mon Sep 17 00:00:00 2001 From: Ryan Westfall Date: Mon, 19 Oct 2026 08:02:51 -0700 Subject: [PATCH 2/2] Serve Hesychia pages with 404s and security headers (#45) ## Summary - Unknown Hesychia URLs return 404 with `404.html` instead of 200 and the app shell. - Extensionless routes (`/legal/terms-of-service`, billing return paths, `/document_storage`) are served as HTML. - `chat_web_app` sends the security headers. The enforcing CSP stays loose enough for Tianji, the API, and the new analytics hosts. A stricter policy is report-only. - Production only: one-character paths redirect for HeyCatch short links. Depends on #44. Merge that first, then retarget this to `master` if Gitea does not do it automatically. ## Test plan - [ ] After #44 and this change are deployed, `curl -sI https://hesychia.ai/not-a-page` is 404. - [ ] `curl -sI https://hesychia.ai/pricing/` is 200 once the frontend shells are published. - [ ] Response headers include `Content-Security-Policy`, `X-Content-Type-Options: nosniff`, `X-Frame-Options: DENY`, and `Referrer-Policy`. - [ ] `curl -sI https://hesychia.ai/x` is a 302 to `/?utm_source=heycatch&utm_campaign=x`. Reviewed-on: https://git.aimloperations.com/ai_ml_operations/server-infra/pulls/45 --- inventory/group_vars/all.yml | 13 ++++++++++ roles/web-static/templates/nginx.conf.j2 | 30 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 704e99a..3c75add 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -125,6 +125,19 @@ app_catalog: api_proxy_prefix: /api/finance/webhooks/ # /robots.txt, /sitemap.xml, and /llms.txt must not fall through to index.html. exact_root_files: true + # Homepage was missing these. Enforcing CSP stays loose so Tianji, the API, + # websockets, and inline styles keep working. Report-Only is the strict policy. + security_headers: true + # Unknown URLs are HTTP 404 with public/404.html, not 200 + the SPA shell. + # Slash routes are directory index.html files from scripts/stampCanonicals.js. + # These extensionless routes are the matching files (keep in sync with + # llm-fe/src/llm-fe/utils/canonicals.js paths that do not end in /). + error_page_404: /404.html + html_exact_paths: + - /legal/terms-of-service + - /billing/success + - /billing/cancel + - /document_storage chat_backend: type: django repo: "{{ git_base_url }}/ai_ml_operations/chat_backend.git" diff --git a/roles/web-static/templates/nginx.conf.j2 b/roles/web-static/templates/nginx.conf.j2 index 8c810b2..771bb53 100644 --- a/roles/web-static/templates/nginx.conf.j2 +++ b/roles/web-static/templates/nginx.conf.j2 @@ -17,6 +17,14 @@ log_format observability escape=json '"user_agent":"$http_user_agent"' '}'; +{% macro security_headers() %} + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Content-Security-Policy "default-src 'self' https: wss: data: blob: 'unsafe-inline'; frame-ancestors 'none'; base-uri 'self'; object-src 'none'" always; + add_header Content-Security-Policy-Report-Only "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; object-src 'none'; form-action 'self'" always; +{% endmacro %} + {% for a in _static_apps %} server { listen {{ a.port }}; @@ -29,6 +37,9 @@ server { root {{ (app_catalog[a.name].webroot_pattern | default(web_static_root ~ '/{env}_' ~ a.name)) | replace('{env}', a.env) }}; index index.html; +{% if app_catalog[a.name].security_headers | default(false) %} +{{ security_headers() }} +{% endif %} {%- set api_backend_name = app_catalog[a.name].api_backend | default('') %} {%- if api_backend_name %} @@ -74,6 +85,22 @@ server { } {% endif %} +{% if app_catalog[a.name].html_exact_paths | default([]) %} +{% for p in app_catalog[a.name].html_exact_paths %} + location = {{ p }} { + default_type text/html; + try_files {{ p }} =404; + } +{% endfor %} +{% endif %} + +{% if a.name == 'chat_web_app' and a.env == 'prod' %} + # HeyCatch short links. Unknown paths 404, so the SDK never sees /x. + location ~ ^/([a-z0-9])$ { + return 302 /?utm_source=heycatch&utm_campaign=$1; + } +{% endif %} + {% if app_catalog[a.name].error_page_404 | default('') %} error_page 404 {{ app_catalog[a.name].error_page_404 }}; @@ -89,6 +116,9 @@ server { location ~* \.(?:js|css|woff2?|png|jpg|jpeg|gif|svg|ico)$ { expires 7d; add_header Cache-Control "public"; +{% if app_catalog[a.name].security_headers | default(false) %} +{{ security_headers() }} +{% endif %} } } {% endfor %} -- 2.54.0