westfarn
846f1b8a37
## Summary - Add `robots.txt`, `sitemap.xml`, and `llms.txt` endpoints so crawlers and AI agents can discover public pages - Fix accessibility tree issues: contact form labels, semantic nav/profile dropdown buttons, cookie consent dialog ARIA, and dead `href="#"` links - Reduce layout shift on homepage/contact via hero min-heights, trusted-by marquee sizing, and fixed cookie banner - Add regression tests and `docs/agentic-browsing.md` for Lighthouse agentic-browsing audits Closes #5. Also addresses overlap with #3 (sitemap + robots.txt). ## Test plan - [x] `python manage.py test public.tests` (14 tests pass) - [ ] Verify `GET /robots.txt`, `/sitemap.xml`, `/llms.txt` return 200 in staging/prod - [ ] Confirm contact form labels visible and submittable on `/contact` - [ ] Run Lighthouse `--only-categories=agentic-browsing` on homepage and contact page - [ ] Smoke test mobile nav dropdown and cookie consent banner Reviewed-on: #8
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
"""Machine-readable site discovery endpoints for crawlers and AI agents."""
|
|
|
|
from django.http import HttpResponse
|
|
from django.template.loader import render_to_string
|
|
from django.urls import reverse
|
|
|
|
# Public marketing pages included in sitemap and llms.txt.
|
|
PUBLIC_PAGE_ENTRIES = (
|
|
("public_index", "Home", "weekly", "1.0"),
|
|
("forward_deployed", "Forward-Deployed AI", "monthly", "0.9"),
|
|
("bot", "AI Agents", "monthly", "0.9"),
|
|
("ml_model", "ML Models", "monthly", "0.8"),
|
|
("chat", "Secure AI Chat", "monthly", "0.8"),
|
|
("ai_sensor", "AI Sensor Algorithms", "monthly", "0.7"),
|
|
("ai_education", "AI Education", "monthly", "0.7"),
|
|
("computers", "Computer Builds", "monthly", "0.7"),
|
|
("file_hosting", "File Hosting", "monthly", "0.7"),
|
|
("web_design", "Web Design and Hosting", "monthly", "0.8"),
|
|
("contact", "Contact", "monthly", "0.9"),
|
|
("terms_of_service", "Terms of Service and Privacy", "yearly", "0.3"),
|
|
)
|
|
|
|
|
|
def _absolute_url(request, url_name):
|
|
return request.build_absolute_uri(reverse(url_name))
|
|
|
|
|
|
def robots_txt(request):
|
|
sitemap_url = _absolute_url(request, "sitemap_xml")
|
|
content = render_to_string(
|
|
"public/robots.txt",
|
|
{"sitemap_url": sitemap_url},
|
|
)
|
|
return HttpResponse(content, content_type="text/plain; charset=utf-8")
|
|
|
|
|
|
def sitemap_xml(request):
|
|
pages = [
|
|
{
|
|
"loc": _absolute_url(request, url_name),
|
|
"changefreq": changefreq,
|
|
"priority": priority,
|
|
}
|
|
for url_name, _title, changefreq, priority in PUBLIC_PAGE_ENTRIES
|
|
]
|
|
content = render_to_string("public/sitemap.xml", {"pages": pages})
|
|
return HttpResponse(content, content_type="application/xml; charset=utf-8")
|
|
|
|
|
|
def llms_txt(request):
|
|
pages = [
|
|
{
|
|
"title": title,
|
|
"url": _absolute_url(request, url_name),
|
|
}
|
|
for url_name, title, _changefreq, _priority in PUBLIC_PAGE_ENTRIES
|
|
]
|
|
content = render_to_string(
|
|
"public/llms.txt",
|
|
{
|
|
"site_url": request.build_absolute_uri("/"),
|
|
"contact_url": _absolute_url(request, "contact"),
|
|
"pages": pages,
|
|
},
|
|
)
|
|
return HttpResponse(content, content_type="text/plain; charset=utf-8")
|