From 02cee6c2202763f5c5f9cc8d77fe7f422d7d9c8d Mon Sep 17 00:00:00 2001 From: Ryan Westfall Date: Mon, 27 Jul 2026 10:26:21 -0500 Subject: [PATCH] Enable beta auto-deploy on master; prod via manual button MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #26. Split deploy.yml into deploy-beta (push→master) and deploy-prod (workflow_dispatch), wire beta hosts/CORS defaults, and document beta secret + Tianji coordination with chat_web_app. --- .env.prod.example | 17 ++++ .gitea/workflows/deploy-beta.yml | 82 +++++++++++++++++++ .../workflows/{deploy.yml => deploy-prod.yml} | 52 ++++++++---- README.md | 30 +++++-- .../chat_backend/tests/test_settings_cors.py | 8 ++ llm_be/llm_be/settings.py | 7 +- scripts/validate-env.sh | 8 +- 7 files changed, 176 insertions(+), 28 deletions(-) create mode 100644 .gitea/workflows/deploy-beta.yml rename .gitea/workflows/{deploy.yml => deploy-prod.yml} (58%) diff --git a/.env.prod.example b/.env.prod.example index 63cbdde..5810340 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -77,10 +77,27 @@ GUNICORN_BIND=0.0.0.0:8000 # ============================================================================= # BETA overrides (use separate file: chat_backend_beta.env) +# Control node: ~/Documents/secrets/chat_backend/chat_backend_beta.env +# Infra: server-infra#7 (host_apps beta :8013, Postgres chat_backend_beta, NPM) # ============================================================================= # DJANGO_ENV=beta +# DJANGO_DEBUG=false # DJANGO_SECRET_KEY=replace-with-a-different-beta-secret # DJANGO_ALLOWED_HOSTS=beta.chatbackend.aimloperations.com +# Optional; when unset, https:// origins are derived from DJANGO_ALLOWED_HOSTS. +# DJANGO_CSRF_TRUSTED_ORIGINS=https://beta.chatbackend.aimloperations.com,https://beta.chat.aimloperations.com +# CORS_ALLOWED_ORIGINS=https://beta.chat.aimloperations.com +# CORS_ORIGIN_ALLOW_ALL=false +# USE_TLS_PROXY=true # DATABASE_URL=postgres://westfarn:replace-db-password@10.0.0.230:5432/chat_backend_beta # WEB_PORT=8013 # OLLAMA_BASE_URL=http://10.0.0.128:11434 +# OAUTH_CALLBACK_BASE_URL=https://beta.chatbackend.aimloperations.com +# FRONTEND_BASE_URL=https://beta.chat.aimloperations.com +# Register beta OAuth redirect URIs in each IdP console: +# https://beta.chatbackend.aimloperations.com/api/auth/oauth/google/callback/ +# https://beta.chatbackend.aimloperations.com/api/auth/oauth/microsoft/callback/ +# +# Tianji: backend does not load tracker.js. Beta SPA uses a distinct website ID +# (chat_web_app .env.beta REACT_APP_TIANJI_WEBSITE_ID). Ensure CORS allows the +# beta frontend origin so Tianji-instrumented pages can call this API. diff --git a/.gitea/workflows/deploy-beta.yml b/.gitea/workflows/deploy-beta.yml new file mode 100644 index 0000000..7bd360e --- /dev/null +++ b/.gitea/workflows/deploy-beta.yml @@ -0,0 +1,82 @@ +name: Deploy Beta + +# Auto-deploy beta after push to master (mirrors dta_service / chat_web_app). +# Prod is manual via Deploy Prod (workflow_dispatch). +on: + push: + branches: + - master + +jobs: + unit-tests: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install dependencies + run: uv sync --frozen + + - name: Run unit tests + env: + DJANGO_ENV: dev + DJANGO_SECRET_KEY: test-secret-key + DJANGO_DEBUG: "true" + DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver + DATABASE_URL: "" + DB_HOST: "" + SKIP_RAG_INIT: "1" + OLLAMA_BASE_URL: http://127.0.0.1:11434 + working-directory: llm_be + run: uv run python manage.py test + + docker: + needs: unit-tests + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build Docker image + run: docker compose build + + # Ephemeral local Postgres only — never inherit host DATABASE_URL (prod/beta). + - name: Run containerized tests + run: | + set -euo pipefail + unset DATABASE_URL DB_HOST DB_NAME DB_USER DB_PASSWORD DB_PORT \ + COMPOSE_DATABASE_URL DJANGO_ENV DJANGO_SECRET_KEY DJANGO_DEBUG \ + DJANGO_ALLOWED_HOSTS OLLAMA_BASE_URL || true + + PROJECT="chat-backend-ci-${{ gitea.sha }}" + cleanup() { docker compose -p "$PROJECT" down -v --remove-orphans || true; } + trap cleanup EXIT + + docker compose -p "$PROJECT" up -d --wait db + docker compose -p "$PROJECT" run --rm --no-deps --entrypoint "" \ + -e DJANGO_ENV=dev \ + -e DJANGO_SECRET_KEY=test-secret-key \ + -e DJANGO_DEBUG=true \ + -e DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,testserver \ + -e DATABASE_URL=postgres://chat_backend:chat_backend@db:5432/chat_backend \ + -e SKIP_RAG_INIT=1 \ + -e OLLAMA_BASE_URL=http://127.0.0.1:11434 \ + web uv run python manage.py test + + deploy-beta: + needs: docker + runs-on: self-hosted + env: + SERVER_INFRA_ROOT: /home/westfarn/Documents/repos/server-infra + steps: + - name: Deploy chat_backend beta + run: | + "$SERVER_INFRA_ROOT/scripts/deploy.sh" \ + --app chat_backend \ + --env beta \ + --ref "${{ gitea.sha }}" diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy-prod.yml similarity index 58% rename from .gitea/workflows/deploy.yml rename to .gitea/workflows/deploy-prod.yml index 6438cd1..c640a8a 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy-prod.yml @@ -1,26 +1,49 @@ -name: Deploy Chat Backend +name: Deploy Prod -# Runs after Unit Tests completes on master. Direct pushes only (not PRs). +# Manual prod deploy only (mirrors dta_service / chat_web_app). +# Push to master deploys beta via Deploy Beta. on: - workflow_run: - workflows: [Unit Tests] - types: [completed] - branches: [master] + workflow_dispatch: {} jobs: + unit-tests: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install dependencies + run: uv sync --frozen + + - name: Run unit tests + env: + DJANGO_ENV: dev + DJANGO_SECRET_KEY: test-secret-key + DJANGO_DEBUG: "true" + DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,testserver + DATABASE_URL: "" + DB_HOST: "" + SKIP_RAG_INIT: "1" + OLLAMA_BASE_URL: http://127.0.0.1:11434 + working-directory: llm_be + run: uv run python manage.py test + docker: - if: gitea.event.workflow_run.conclusion == 'success' && gitea.event.workflow_run.event == 'push' + needs: unit-tests runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v4 - with: - ref: ${{ gitea.event.workflow_run.head_sha }} - name: Build Docker image run: docker compose build - # Ephemeral local Postgres only — never inherit host DATABASE_URL (prod). + # Ephemeral local Postgres only — never inherit host DATABASE_URL (prod/beta). - name: Run containerized tests run: | set -euo pipefail @@ -28,7 +51,7 @@ jobs: COMPOSE_DATABASE_URL DJANGO_ENV DJANGO_SECRET_KEY DJANGO_DEBUG \ DJANGO_ALLOWED_HOSTS OLLAMA_BASE_URL || true - PROJECT="chat-backend-ci-${{ gitea.event.workflow_run.head_sha }}" + PROJECT="chat-backend-ci-${{ gitea.sha }}" cleanup() { docker compose -p "$PROJECT" down -v --remove-orphans || true; } trap cleanup EXIT @@ -43,10 +66,9 @@ jobs: -e OLLAMA_BASE_URL=http://127.0.0.1:11434 \ web uv run python manage.py test - deploy: - if: gitea.event.workflow_run.conclusion == 'success' && gitea.event.workflow_run.event == 'push' - runs-on: self-hosted + deploy-prod: needs: docker + runs-on: self-hosted env: SERVER_INFRA_ROOT: /home/westfarn/Documents/repos/server-infra steps: @@ -55,4 +77,4 @@ jobs: "$SERVER_INFRA_ROOT/scripts/deploy.sh" \ --app chat_backend \ --env prod \ - --ref "${{ gitea.event.workflow_run.head_sha }}" + --ref "${{ gitea.sha }}" diff --git a/README.md b/README.md index 6747fff..1b6038c 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ with `COMPOSE_DATABASE_URL` if needed. | `ENABLE_ACCOUNT_REGISTRATION` | `false` | optional | Self-serve sign-up; keep false until ready | | `STRIPE_SECRET_KEY` / `STRIPE_PUBLISHABLE_KEY` / `STRIPE_WEBHOOK_SECRET` | empty | yes for billing | Stripe API + webhook | | `STRIPE_PRICE_ID` | empty | optional | Pre-created Price; else `$10/mo` from settings | -| `FRONTEND_BASE_URL` | `http://localhost:3000` | set in prod | Checkout success/cancel base | -| `CORS_ALLOWED_ORIGINS` | local + chat FE | set in prod | Frontend origin | +| `FRONTEND_BASE_URL` | `http://localhost:3000` | set in prod/beta | Checkout success/cancel + OAuth return | +| `CORS_ALLOWED_ORIGINS` | local + chat FE (+ beta FE default) | set in prod/beta | Frontend origin(s) | | `USE_TLS_PROXY` | false (dev) | true behind NPM | Sets `SECURE_PROXY_SSL_HEADER` | | `GUNICORN_WORKERS` / `GUNICORN_BIND` | 2 / `0.0.0.0:8000` | optional | Entrypoint | | `SKIP_RAG_INIT` | unset | CI/migrate often `1` | Skip Chroma/Ollama boot work | @@ -105,16 +105,18 @@ generation prompts (chat, RAG, data analysis). Not env-configurable. Templates: `.env.example` (local), `.env.prod.example` (control-node secret). -Control-node secret path (server-infra on ai-server-4080): +Control-node secret paths (server-infra on ai-server-4080): ```text ~/Documents/secrets/chat_backend/chat_backend_prod.env +~/Documents/secrets/chat_backend/chat_backend_beta.env ``` Validate with: ```bash ./scripts/validate-env.sh ~/Documents/secrets/chat_backend/chat_backend_prod.env +./scripts/validate-env.sh ~/Documents/secrets/chat_backend/chat_backend_beta.env ``` If `DATABASE_URL` password contains `$`, escape each as `$$` for Compose. @@ -141,29 +143,41 @@ RAG loaders that need a path materialize a short-lived temp file, then delete it Chroma’s vector index may still use a volume (`chroma_db`); that is embeddings metadata, not the original upload. -## Production (docker-compose.prod.yml) +## Production / beta (docker-compose.prod.yml) - Single `web` service; **no** bundled DB — `DATABASE_URL` → shared Postgres (`10.0.0.230`). -- Host port from `WEB_PORT` (catalog: **8003**; beta reserved **8013**). +- Host port from `WEB_PORT` (prod **8003**; beta **8013**). - Entrypoint: wait DB → migrate → collectstatic → `gunicorn` + `UvicornWorker` (ASGI for HTTP **and** WebSockets). - Active/active on **adama + roslin + ai-server-4080**; NPM balances upstreams. -- Deployed by: +- Manual / local deploy: ```bash +# beta (day-to-day) +~/Documents/repos/server-infra/scripts/deploy.sh \ + --app chat_backend --env beta --ref + +# prod (intentional) ~/Documents/repos/server-infra/scripts/deploy.sh \ --app chat_backend --env prod --ref ``` +Beta hosts / CORS: `beta.chatbackend.aimloperations.com` API + +`https://beta.chat.aimloperations.com` SPA (see `.env.prod.example` beta block). +DB: `chat_backend_beta`. Pair with [server-infra#7](https://git.aimloperations.com/ai_ml_operations/server-infra/issues/7) +and frontend [chat_web_app#35](https://git.aimloperations.com/ai_ml_operations/chat_web_app/issues/35). + ## CI / CD (Gitea Actions) | Workflow | Trigger | Action | |----------|---------|--------| | `unittests.yml` | push + PR → `master` | `uv sync` + `manage.py test` | | `ci.yml` | PR → `master` | same unit tests | -| `deploy.yml` | after Unit Tests succeeds on `master` **push** | docker build + tests on **ephemeral compose Postgres** → `deploy.sh` | +| `deploy-beta.yml` | **push** to `master` | unit tests → docker compose tests → `deploy.sh --env beta` | +| `deploy-prod.yml` | **manual** `workflow_dispatch` only | unit tests → docker compose tests → `deploy.sh --env prod` | -Deploy never runs on PRs. +Push/merge to `master` auto-deploys **beta** only. Prod requires the Gitea +**Run workflow** button on **Deploy Prod**. Deploy never runs on PRs. ## Security note diff --git a/llm_be/chat_backend/tests/test_settings_cors.py b/llm_be/chat_backend/tests/test_settings_cors.py index 1bde2cb..29eeb88 100644 --- a/llm_be/chat_backend/tests/test_settings_cors.py +++ b/llm_be/chat_backend/tests/test_settings_cors.py @@ -59,3 +59,11 @@ class CapacitorWebviewOriginTests(SimpleTestCase): from django.conf import settings self.assertFalse(settings.CORS_ALLOW_CREDENTIALS) + + def test_default_cors_includes_beta_frontend(self): + from django.conf import settings + + self.assertIn( + "https://beta.chat.aimloperations.com", + settings.CORS_ALLOWED_ORIGINS, + ) diff --git a/llm_be/llm_be/settings.py b/llm_be/llm_be/settings.py index 79237a7..176b513 100644 --- a/llm_be/llm_be/settings.py +++ b/llm_be/llm_be/settings.py @@ -122,7 +122,9 @@ DEBUG = env_bool("DJANGO_DEBUG", DJANGO_ENV == "dev") allowed_hosts = env_list( "DJANGO_ALLOWED_HOSTS", - "localhost,127.0.0.1,0.0.0.0,chatbackend.aimloperations.com,chat.aimloperations.com", + "localhost,127.0.0.1,0.0.0.0," + "chatbackend.aimloperations.com,chat.aimloperations.com," + "beta.chatbackend.aimloperations.com,beta.chat.aimloperations.com", ) ALLOWED_HOSTS = allowed_hosts if allowed_hosts else ["*"] @@ -139,7 +141,8 @@ CORS_ORIGIN_ALLOW_ALL = env_bool("CORS_ORIGIN_ALLOW_ALL", True) CORS_ALLOWED_ORIGINS = with_capacitor_webview_origins( env_list( "CORS_ALLOWED_ORIGINS", - "http://localhost:3000,http://127.0.0.1:3000,https://chat.aimloperations.com", + "http://localhost:3000,http://127.0.0.1:3000," + "https://chat.aimloperations.com,https://beta.chat.aimloperations.com", ) ) diff --git a/scripts/validate-env.sh b/scripts/validate-env.sh index 038b46f..ed676eb 100755 --- a/scripts/validate-env.sh +++ b/scripts/validate-env.sh @@ -44,9 +44,11 @@ if ((${#missing[@]} > 0)); then exit 1 fi -if [[ "$DJANGO_ENV" == "prod" && ( "$DJANGO_SECRET_KEY" == change-me* || "$DJANGO_SECRET_KEY" == *dev-only* ) ]]; then - echo "DJANGO_SECRET_KEY must be changed from the example value for production." >&2 - exit 1 +if [[ "$DJANGO_ENV" == "prod" || "$DJANGO_ENV" == "beta" ]]; then + if [[ "$DJANGO_SECRET_KEY" == change-me* || "$DJANGO_SECRET_KEY" == *dev-only* || "$DJANGO_SECRET_KEY" == django-insecure* ]]; then + echo "DJANGO_SECRET_KEY must be a real secret for $DJANGO_ENV (not an example/dev value)." >&2 + exit 1 + fi fi echo "Environment validation passed (DJANGO_ENV=$DJANGO_ENV)."