Add eval harness (#62 P4), status frames (#96), and agentic runs (#63).

Ship the Phase 4 accuracy eval suite with a manual Gitea workflow, emit versioned WS status frames during grounded chat, and introduce opt-in agent infrastructure (Redis/Celery, AgentRun/Step, tools, orchestrator) gated by ALLOW_AGENTIC_TASKS so default chat behaviour stays unchanged.
2026-08-04 06:07:26 -05:00
parent e1e086a474
commit 9c0b648db3
46 changed files with 4669 additions and 117 deletions
@@ -0,0 +1,60 @@
+name: Run Evals
+
+# Manual eval harness — does not gate PRs (#62 Phase 4).
+# Requires self-hosted runner with GPU/Ollama and live search when SKIP_LIVE is unset.
+on:
+  workflow_dispatch: {}
+
+jobs:
+  run-evals:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Validate eval suite (offline)
+        env:
+          DJANGO_ENV: dev
+          DJANGO_SECRET_KEY: test-secret-key
+          DJANGO_DEBUG: "true"
+          DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver
+          DATABASE_URL: ""
+          DB_HOST: ""
+          SKIP_RAG_INIT: "1"
+          SKIP_LIVE: "1"
+          OLLAMA_BASE_URL: http://127.0.0.1:11434
+        working-directory: llm_be
+        run: uv run python manage.py run_evals --dry-run
+
+      - name: Run eval harness
+        env:
+          DJANGO_ENV: dev
+          DJANGO_SECRET_KEY: test-secret-key
+          DJANGO_DEBUG: "true"
+          DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver
+          DATABASE_URL: ""
+          DB_HOST: ""
+          SKIP_RAG_INIT: "1"
+          RUN_EVALS: "1"
+          ALLOW_INTERNET_ACCESS: "true"
+          OLLAMA_BASE_URL: http://127.0.0.1:11434
+        working-directory: llm_be
+        run: |
+          uv run python manage.py run_evals \
+            --runs 3 \
+            --output "../eval-report.json"
+
+      - name: Upload eval report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-report
+          path: eval-report.json