name: Run Evals

# Manual eval harness — does not gate PRs (#62 Phase 4).
# Requires self-hosted runner with GPU/Ollama and live search when SKIP_LIVE is unset.
on:
  workflow_dispatch: {}

jobs:
  run-evals:
    runs-on: self-hosted
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Install dependencies
        run: uv sync --frozen

      - name: Validate eval suite (offline)
        env:
          DJANGO_ENV: dev
          DJANGO_SECRET_KEY: test-secret-key
          DJANGO_DEBUG: "true"
          DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver
          DATABASE_URL: ""
          DB_HOST: ""
          SKIP_RAG_INIT: "1"
          SKIP_LIVE: "1"
          OLLAMA_BASE_URL: http://127.0.0.1:11434
        working-directory: llm_be
        run: uv run python manage.py run_evals --dry-run

      - name: Run eval harness
        env:
          DJANGO_ENV: dev
          DJANGO_SECRET_KEY: test-secret-key
          DJANGO_DEBUG: "true"
          DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver
          DATABASE_URL: ""
          DB_HOST: ""
          SKIP_RAG_INIT: "1"
          RUN_EVALS: "1"
          ALLOW_INTERNET_ACCESS: "true"
          OLLAMA_BASE_URL: http://127.0.0.1:11434
        working-directory: llm_be
        run: |
          uv run python manage.py run_evals \
            --runs 3 \
            --output "../eval-report.json"

      - name: Upload eval report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-report
          path: eval-report.json