Fix SearxNG crash loop: make settings.yml readable inside the container.

Container runs as non-root; mode 0640 owned by westfarn caused
PermissionError on /etc/searxng/settings.yml and a restart loop.
Also lengthen the default secret_key to >= 32 chars.

IMPLEMENTATION.md

@@ -35,7 +35,7 @@ Both pipelines share the same inventory (`inventory/hosts.yml`).
 |------|-----|------|
 | adama | 10.0.0.77 | Ubuntu Server VM (Proxmox) — app host |
 | roslin | 10.0.0.176 | Ubuntu Server VM (Proxmox) — app host |
-| ai-server-4080 | 10.0.0.128 | Control node + Gitea act runner (no app workloads) |
+| ai-server-4080 | 10.0.0.128 | Control node + Gitea act runner + Ollama + SearxNG + observability; also runs app replicas |
 
 Hostname on this machine: `ryan-development-1`
 
@@ -54,7 +54,7 @@ server-infra/
 │   └── host_vars/
 │       ├── adama.yml       # host_apps (django + dta_webapp)
 │       ├── roslin.yml      # host_apps (mirrors adama)
-│       └── ai-server-4080.yml  # control node / act runner, no workloads
+│       └── ai-server-4080.yml  # control node / act runner / SearxNG / observability
 ├── playbooks/
 │   ├── site.yml            # Phase 1: provision
 │   └── deploy-apps.yml     # Phase 2: CI deploy
@@ -65,6 +65,9 @@ server-infra/
 │   ├── nodejs/             # Node.js + npm + npx (NodeSource)
 │   ├── gitea-key/          # per-server SSH key + Gitea access probe
 │   ├── tianji/             # Monitoring reporter
+│   ├── observability/      # Loki + Prometheus + Grafana (ai-server-4080)
+│   ├── searxng/            # SearxNG JSON API for chat_backend (#10)
+│   ├── alloy/              # log/metrics shipper
 │   ├── app-deploy/         # django (docker) + node-static deploy
 │   └── web-static/         # nginx container serving /var/www builds
 └── scripts/
@@ -194,7 +197,7 @@ After Docker install, re-SSH so the `docker` group membership takes effect.
 | `dta_webapp` | node/vite static | adama + roslin (+ ai-server-4080) | beta + prod | active/active; built to `/var/www/<env>.app.ditchtheagent/html`, served by web-static nginx |
 | `scha` | django (docker) | adama + roslin + ai-server-4080 | prod | active/active behind NPM; beta port reserved |
 | `chat_web_app` | node-static (CRA) | adama + roslin + ai-server-4080 | beta + prod | active/active; built to `/var/www/<env>.chat.aimloperations/html`, served by web-static nginx |
-| `chat_backend` | django (docker) | adama + roslin + ai-server-4080 | beta + prod | active/active behind NPM; Ollama via `OLLAMA_BASE_URL=http://10.0.0.128:11434` |
+| `chat_backend` | django (docker) | adama + roslin + ai-server-4080 | beta + prod | active/active behind NPM; Ollama `http://10.0.0.128:11434`; SearxNG `http://10.0.0.128:8088` (`SEARXNG_BASE_URL`) |
 
 Django apps use a **shared external Postgres** (via `DATABASE_URL` in each host's
 env file) so active/active replicas share one database. Beta and prod never share
@@ -221,6 +224,20 @@ future beta replica.
 | chat_backend | 8013 | 8003 | adama, roslin, ai-server-4080 |
 | dta_webapp (nginx) | 8081 | 8080 | adama, roslin, ai-server-4080 |
 | chat_web_app (nginx) | 8083 | 8082 | adama, roslin, ai-server-4080 |
+| SearxNG (LAN only) | — | **8088** | ai-server-4080 only (`searxng_stack`); not an NPM upstream |
+
+Host-local services on ai-server-4080 (not balanced by NPM):
+
+| Service | Port | Notes |
+|---------|------|-------|
+| Ollama | 11434 | Not Ansible-managed today; GPU host |
+| SearxNG | 8088 | `roles/searxng` (#10); JSON API for chat_backend grounded search |
+| Loki | 3100 | `roles/observability` |
+| Prometheus | 9090 | `roles/observability` |
+| Grafana | 3000 | `roles/observability` |
+
+**Port clash warning:** do **not** bind SearxNG to `8080` — that is `dta_webapp` prod.
+chat_backend secrets must use `SEARXNG_BASE_URL=http://10.0.0.128:8088`.
 
 ### Flow
 

inventory/host_vars/ai-server-4080.yml

@@ -11,6 +11,11 @@ act_runner_enabled: true
 # Central Loki + Prometheus + Grafana (roles/observability).
 observability_stack: true
 
+# SearxNG JSON search API for chat_backend grounded retrieval (#10).
+# Host port 8088 — 8080 is dta_webapp on this host. chat_backend secret:
+#   SEARXNG_BASE_URL=http://10.0.0.128:8088
+searxng_stack: true
+
 # This host's pre-existing ~/.ssh/id_ed25519 is a personal key WITH a passphrase,
 # which hangs the (non-BatchMode) gitea access probe. Use a dedicated,
 # passphrase-less deploy key here instead.

playbooks/site.yml

@@ -21,6 +21,13 @@
     - role: observability
       when: observability_stack | default(false) | bool
 
+- name: Provision SearxNG (chat_backend grounded search)
+  hosts: ai-server-4080
+  become: true
+  roles:
+    - role: searxng
+      when: searxng_stack | default(false) | bool
+
 - name: Provision Alloy agents
   hosts: webservers
   become: true

roles/searxng/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+# SearxNG for chat_backend grounded web search (#10).
+# Hosted on ai-server-4080 only (next to Ollama). LAN-only — not NPM public.
+
+searxng_dir: "{{ apps_base_dir }}/searxng"
+searxng_image: "searxng/searxng:latest"
+
+# Host port 8088 — 8080 is already dta_webapp prod on ai-server-4080.
+searxng_host_port: 8088
+searxng_container_port: 8080
+
+# Public base URL as seen by chat_backend containers on the LAN.
+searxng_base_url: "http://{{ ansible_host }}:{{ searxng_host_port }}/"
+
+# Override via host_vars or vault; must be stable across restarts.
+# SearxNG expects a long random string (use >= 32 chars).
+searxng_secret_key: "change-me-searxng-secret-key-32chars"
+
+# LAN CIDR allowed to hit the JSON API (same pattern as observability).
+searxng_ufw_from: "{{ ufw_ssh_allowed_network }}"

roles/searxng/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+# SearxNG JSON search API for chat_backend grounded retrieval.
+# Enabled on ai-server-4080 via searxng_stack: true (issue #10).
+
+- name: searxng | ensure project directory
+  ansible.builtin.file:
+    path: "{{ searxng_dir }}"
+    state: directory
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0755"
+
+- name: searxng | settings.yml
+  ansible.builtin.template:
+    src: settings.yml.j2
+    dest: "{{ searxng_dir }}/settings.yml"
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    # Container runs as non-root searxng UID — must be world-readable (#10).
+    mode: "0644"
+  register: _searxng_settings
+
+- name: searxng | compose file
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ searxng_dir }}/docker-compose.yml"
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0644"
+  register: _searxng_compose
+
+- name: searxng | allow JSON API from LAN
+  community.general.ufw:
+    rule: allow
+    port: "{{ searxng_host_port }}"
+    proto: tcp
+    from_ip: "{{ searxng_ufw_from }}"
+    comment: SearxNG for chat_backend grounded search
+
+- name: searxng | start stack
+  ansible.builtin.command:
+    cmd: >-
+      docker compose up -d --remove-orphans
+      {{ '--force-recreate' if (
+        _searxng_compose is changed
+        or _searxng_settings is changed
+      ) else '' }}
+    chdir: "{{ searxng_dir }}"
+  become: true
+  become_user: "{{ admin_user }}"
+  register: _searxng_up
+  changed_when: >-
+    _searxng_up.rc == 0 and (
+      'Started' in (_searxng_up.stdout | default(''))
+      or 'Recreated' in (_searxng_up.stdout | default(''))
+      or 'Created' in (_searxng_up.stdout | default(''))
+      or _searxng_compose is changed
+      or _searxng_settings is changed
+    )
+  failed_when: _searxng_up.rc != 0
+
+- name: searxng | show compose failure output
+  ansible.builtin.debug:
+    msg: "{{ _searxng_up.stderr_lines | default(_searxng_up.stdout_lines) }}"
+  when: _searxng_up is failed

roles/searxng/templates/docker-compose.yml.j2

@@ -0,0 +1,23 @@
+# Managed by Ansible (roles/searxng). Do not edit by hand.
+services:
+  searxng:
+    image: {{ searxng_image }}
+    container_name: searxng
+    restart: unless-stopped
+    ports:
+      - "{{ searxng_host_port }}:{{ searxng_container_port }}"
+    volumes:
+      - ./settings.yml:/etc/searxng/settings.yml:rw
+    environment:
+      - SEARXNG_BASE_URL={{ searxng_base_url }}
+    cap_drop:
+      - ALL
+    cap_add:
+      - CHOWN
+      - SETGID
+      - SETUID
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"

roles/searxng/templates/settings.yml.j2

@@ -0,0 +1,20 @@
+# Managed by Ansible (roles/searxng). Do not edit by hand.
+# Minimal overlay on SearxNG defaults — JSON format required by chat_backend (#62).
+
+use_default_settings: true
+
+server:
+  secret_key: "{{ searxng_secret_key }}"
+  limiter: false
+  image_proxy: false
+  port: {{ searxng_container_port }}
+  bind_address: "0.0.0.0"
+  base_url: "{{ searxng_base_url }}"
+
+search:
+  safe_search: 0
+  autocomplete: ""
+  default_lang: "en"
+  formats:
+    - html
+    - json

scripts/provision.sh

@@ -15,7 +15,8 @@ Provision server(s) with site.yml.
 
 Applies: common, ufw, docker, nodejs, gitea-key, tianji, alloy (every host).
 On ai-server-4080 also: observability (Loki + Prometheus + Grafana) when
-observability_stack is true in host_vars.
+observability_stack is true, and SearxNG when searxng_stack is true
+(chat_backend grounded search on :8088).
 
   HOST              Optional. Limit to one host: adama, roslin, or ai-server-4080.
                     Omit to run against all webservers.
@@ -31,7 +32,7 @@ Examples:
   $(basename "$0") adama --check    # dry run on adama only
   $(basename "$0") adama            # provision adama (includes Alloy)
   $(basename "$0") adama --ask-pass  # first SSH login before ssh-copy-id
-  $(basename "$0") ai-server-4080   # control node + Loki/Prometheus/Grafana
+  $(basename "$0") ai-server-4080   # control node + Loki/Prometheus/Grafana + SearxNG
   $(basename "$0")                  # provision all hosts
 EOF
 }