westfarn
9c0b648db3
Ship the Phase 4 accuracy eval suite with a manual Gitea workflow, emit versioned WS status frames during grounded chat, and introduce opt-in agent infrastructure (Redis/Celery, AgentRun/Step, tools, orchestrator) gated by ALLOW_AGENTIC_TASKS so default chat behaviour stays unchanged.
61 lines
1.7 KiB
YAML
61 lines
1.7 KiB
YAML
name: Run Evals
|
|
|
|
# Manual eval harness — does not gate PRs (#62 Phase 4).
|
|
# Requires self-hosted runner with GPU/Ollama and live search when SKIP_LIVE is unset.
|
|
on:
|
|
workflow_dispatch: {}
|
|
|
|
jobs:
|
|
run-evals:
|
|
runs-on: self-hosted
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
run: |
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --frozen
|
|
|
|
- name: Validate eval suite (offline)
|
|
env:
|
|
DJANGO_ENV: dev
|
|
DJANGO_SECRET_KEY: test-secret-key
|
|
DJANGO_DEBUG: "true"
|
|
DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver
|
|
DATABASE_URL: ""
|
|
DB_HOST: ""
|
|
SKIP_RAG_INIT: "1"
|
|
SKIP_LIVE: "1"
|
|
OLLAMA_BASE_URL: http://127.0.0.1:11434
|
|
working-directory: llm_be
|
|
run: uv run python manage.py run_evals --dry-run
|
|
|
|
- name: Run eval harness
|
|
env:
|
|
DJANGO_ENV: dev
|
|
DJANGO_SECRET_KEY: test-secret-key
|
|
DJANGO_DEBUG: "true"
|
|
DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver
|
|
DATABASE_URL: ""
|
|
DB_HOST: ""
|
|
SKIP_RAG_INIT: "1"
|
|
RUN_EVALS: "1"
|
|
ALLOW_INTERNET_ACCESS: "true"
|
|
OLLAMA_BASE_URL: http://127.0.0.1:11434
|
|
working-directory: llm_be
|
|
run: |
|
|
uv run python manage.py run_evals \
|
|
--runs 3 \
|
|
--output "../eval-report.json"
|
|
|
|
- name: Upload eval report
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: eval-report
|
|
path: eval-report.json
|