From f37ec41530c3ca329fda7fe759660b929326c8f3 Mon Sep 17 00:00:00 2001 From: Ryan Westfall Date: Fri, 10 Jul 2026 06:50:52 -0500 Subject: [PATCH] Updates for grafana stack --- README.md | 8 +- docs/OBSERVABILITY.md | 118 +++------- inventory/group_vars/all.yml | 11 + inventory/host_vars/ai-server-4080.yml | 3 + playbooks/site.yml | 19 +- roles/alloy/defaults/main.yml | 5 + roles/alloy/tasks/main.yml | 50 +++++ roles/alloy/templates/config.alloy.j2 | 206 ++++++++++++++++++ roles/alloy/templates/docker-compose.yml.j2 | 29 +++ roles/observability/defaults/main.yml | 23 ++ roles/observability/tasks/main.yml | 128 +++++++++++ .../templates/docker-compose.yml.j2 | 60 +++++ .../templates/grafana-datasources.yml.j2 | 21 ++ .../templates/loki-config.yml.j2 | 47 ++++ .../observability/templates/prometheus.yml.j2 | 13 ++ scripts/provision.sh | 8 +- 16 files changed, 652 insertions(+), 97 deletions(-) create mode 100644 roles/alloy/defaults/main.yml create mode 100644 roles/alloy/tasks/main.yml create mode 100644 roles/alloy/templates/config.alloy.j2 create mode 100644 roles/alloy/templates/docker-compose.yml.j2 create mode 100644 roles/observability/defaults/main.yml create mode 100644 roles/observability/tasks/main.yml create mode 100644 roles/observability/templates/docker-compose.yml.j2 create mode 100644 roles/observability/templates/grafana-datasources.yml.j2 create mode 100644 roles/observability/templates/loki-config.yml.j2 create mode 100644 roles/observability/templates/prometheus.yml.j2 diff --git a/README.md b/README.md index 4038f8e..65fec9f 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,13 @@ exit # Test connectivity to one host ansible adama -m ping -# Provision one host (dry run first) +# Provision one host (dry run first) — includes Alloy log/metrics agent ./scripts/provision.sh adama --check ./scripts/provision.sh adama +# Control node: Alloy + Loki + Prometheus + Grafana +./scripts/provision.sh ai-server-4080 + # Provision all hosts ./scripts/provision.sh ``` @@ -34,6 +37,9 @@ See [IMPLEMENTATION.md](IMPLEMENTATION.md) for full architecture, CI/CD plan, an Observability (Alloy → Loki / Prometheus → Grafana): [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md) · [docs/GRAFANA_USAGE.md](docs/GRAFANA_USAGE.md) +Provision installs **Alloy** on every host. On **ai-server-4080** it also starts +the central **Loki / Prometheus / Grafana** stack (`observability_stack: true`). + ## Servers | Host | IP | Role | diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index dce0778..1e08aaa 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -760,13 +760,18 @@ Dashboard build steps (container health, system status, imports) are in --- -## Part 4 — Ansible provision (make Alloy automatic) +## Part 4 — Ansible provision (implemented) -Manual steps above prove the pipeline. Next, encode Alloy into provision so -`./scripts/provision.sh` installs/updates it on every host, and keep Loki + -Prometheus + Grafana only on the control node. +Roles are in the repo. `./scripts/provision.sh` runs `site.yml`, which: -### 4.1 Target layout (recommended) +1. Base roles on every host (`common` → `tianji`) +2. **`observability`** on `ai-server-4080` when `observability_stack: true` + (Loki + Prometheus + Grafana + UFW for `3100`/`9090`/`3000`) +3. **`alloy`** on every host (logs → Loki, metrics → Prometheus) + +Alloy is Grafana’s Promtail successor; one agent covers logs **and** metrics. + +### 4.1 Layout ``` roles/ @@ -786,107 +791,32 @@ roles/ └── grafana-datasources.yml.j2 ``` -### 4.2 Inventory vars (`inventory/group_vars/all.yml`) +### 4.2 Inventory -Add something like: +Vars in `inventory/group_vars/all.yml` (`loki_url`, `prometheus_url`, +`grafana_public_url`, …). Flag in `inventory/host_vars/ai-server-4080.yml`: ```yaml -# Observability -loki_url: "http://10.0.0.128:3100" -loki_push_url: "{{ loki_url }}/loki/api/v1/push" -prometheus_url: "http://10.0.0.128:9090" -prometheus_remote_write_url: "{{ prometheus_url }}/api/v1/write" -alloy_image: "grafana/alloy:v1.7.1" -alloy_dir: "{{ apps_base_dir }}/observability/alloy" -observability_dir: "{{ apps_base_dir }}/observability" -loki_image: "grafana/loki:3.4.2" -prometheus_image: "prom/prometheus:v3.2.1" -grafana_image: "grafana/grafana:11.5.2" -grafana_public_url: "https://grafana.aimloperations.com" +observability_stack: true ``` -### 4.3 Host flag (`inventory/host_vars/ai-server-4080.yml`) +`playbooks/site.yml` order: base → observability (4080) → alloy (all). -```yaml -observability_stack: true # run Loki + Prometheus + Grafana on this host -``` +### 4.3 Secrets -Other hosts omit the flag (or set `false`). +- Default Grafana password is `CHANGE_ME_ON_FIRST_LOGIN` (role default). Change + on first login, or set `observability_grafana_admin_password` in host_vars / + vault (never commit real passwords). +- SMTP2GO: see [GRAFANA_USAGE.md](GRAFANA_USAGE.md). -### 4.4 Playbook wiring (`playbooks/site.yml`) - -```yaml ---- -- name: Provision webservers - hosts: webservers - become: true - roles: - - common - - ufw - - docker - - nodejs - - gitea-key - - tianji - - alloy - -- name: Provision central observability stack - hosts: ai-server-4080 - become: true - roles: - - role: observability - when: observability_stack | default(false) -``` - -Order matters: **docker** before **alloy** / **observability**. - -### 4.5 Alloy template essentials - -In `config.alloy.j2`, set host from inventory — never hardcode: - -```jinja -labels = { - job = "systemd", - host = "{{ inventory_hostname }}", -} -``` - -Same for Docker log labels and every `prometheus.relabel` `replacement` -for `host`. That is how `adama` / `roslin` / `ai-server-4080` stay correct -without editing each file by hand. - -### 4.6 UFW for Loki + Prometheus (control node only) - -Do **not** put `3100` / `9090` in global `ufw_allowed_tcp_ports` (that opens -on every host). Prefer a small task in `roles/observability` or a host_vars -list: - -```yaml -# host_vars/ai-server-4080.yml -ufw_extra_rules: - - { port: 3100, proto: tcp, from_ip: "10.0.0.0/24", comment: "Loki" } - - { port: 9090, proto: tcp, from_ip: "10.0.0.0/24", comment: "Prometheus" } - - { port: 3000, proto: tcp, from_ip: "10.0.0.0/24", comment: "Grafana" } -``` - -(Extend `roles/ufw` to loop `ufw_extra_rules` when you implement this.) - -### 4.7 Secrets - -- Grafana admin password: store under `~/Documents/secrets/observability/` - (same pattern as app env files), push or template at provision time. **Never - commit** passwords or SMTP2GO keys to git. -- SMTP2GO credentials: configure in Grafana UI or via env file on the host — - see [GRAFANA_USAGE.md](GRAFANA_USAGE.md). - -### 4.8 Apply +### 4.4 Apply ```bash -# After roles exist: ./scripts/provision.sh ai-server-4080 --check ./scripts/provision.sh ai-server-4080 ./scripts/provision.sh adama ./scripts/provision.sh roslin -# or all: +# or all (site.yml orders stack before Alloy): ./scripts/provision.sh ``` @@ -1002,8 +932,10 @@ du -sh /opt/apps/observability/loki/data \ ### Follow-ups -- [ ] Encode `roles/alloy` + `roles/observability` and add to `site.yml` +- [x] Encode `roles/alloy` + `roles/observability` and add to `site.yml` - [ ] Optional: drop or keep Tianji side-by-side (they do not conflict) +- [ ] NPM proxy + change Grafana admin password +- [ ] SMTP2GO + dashboards — [GRAFANA_USAGE.md](GRAFANA_USAGE.md) --- diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 193b5e8..e8307ab 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -21,6 +21,17 @@ tianji_server_url: https://tianji.aimloperations.com tianji_workspace_id: cm7w8087y020lddswyhamadj2 tianji_install_script_url: "https://tianji.aimloperations.com/serverStatus/{{ tianji_workspace_id }}/install.sh?url={{ tianji_server_url }}" +# --------------------------------------------------------------------------- +# Observability (Alloy on every host → Loki/Prometheus/Grafana on control node) +# See docs/OBSERVABILITY.md +# --------------------------------------------------------------------------- +loki_url: "http://10.0.0.128:3100" +loki_push_url: "{{ loki_url }}/loki/api/v1/push" +prometheus_url: "http://10.0.0.128:9090" +prometheus_remote_write_url: "{{ prometheus_url }}/api/v1/write" +grafana_public_url: "https://grafana.aimloperations.com" +grafana_public_domain: "grafana.aimloperations.com" + # --------------------------------------------------------------------------- # App deployment (Phase 2) # --------------------------------------------------------------------------- diff --git a/inventory/host_vars/ai-server-4080.yml b/inventory/host_vars/ai-server-4080.yml index 4c440f1..88e2f74 100644 --- a/inventory/host_vars/ai-server-4080.yml +++ b/inventory/host_vars/ai-server-4080.yml @@ -8,6 +8,9 @@ ansible_control_node: true ansible_connection: local act_runner_enabled: true +# Central Loki + Prometheus + Grafana (roles/observability). +observability_stack: true + # This host's pre-existing ~/.ssh/id_ed25519 is a personal key WITH a passphrase, # which hangs the (non-BatchMode) gitea access probe. Use a dedicated, # passphrase-less deploy key here instead. diff --git a/playbooks/site.yml b/playbooks/site.yml index d628409..afb625a 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -1,5 +1,9 @@ --- -- name: Provision webservers +# Order: base OS → central Loki/Prometheus/Grafana → Alloy agents. +# Alloy on every host pushes to the stack on ai-server-4080, so the stack +# must exist before (or at least as Alloy starts against) those endpoints. + +- name: Provision webservers (base) hosts: webservers become: true roles: @@ -9,3 +13,16 @@ - nodejs - gitea-key - tianji + +- name: Provision central observability stack + hosts: ai-server-4080 + become: true + roles: + - role: observability + when: observability_stack | default(false) | bool + +- name: Provision Alloy agents + hosts: webservers + become: true + roles: + - alloy diff --git a/roles/alloy/defaults/main.yml b/roles/alloy/defaults/main.yml new file mode 100644 index 0000000..ffe9191 --- /dev/null +++ b/roles/alloy/defaults/main.yml @@ -0,0 +1,5 @@ +--- +alloy_image: "grafana/alloy:v1.7.1" +alloy_dir: "{{ apps_base_dir }}/observability/alloy" +alloy_loki_push_url: "{{ loki_push_url }}" +alloy_prometheus_remote_write_url: "{{ prometheus_remote_write_url }}" diff --git a/roles/alloy/tasks/main.yml b/roles/alloy/tasks/main.yml new file mode 100644 index 0000000..a43d25e --- /dev/null +++ b/roles/alloy/tasks/main.yml @@ -0,0 +1,50 @@ +--- +# Ship host/container logs → Loki and metrics → Prometheus via Grafana Alloy +# (Promtail successor; also collects node + cAdvisor metrics). + +- name: alloy | ensure project directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0750" + loop: + - "{{ apps_base_dir }}/observability" + - "{{ alloy_dir }}" + +- name: alloy | config + ansible.builtin.template: + src: config.alloy.j2 + dest: "{{ alloy_dir }}/config.alloy" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + register: _alloy_config + +- name: alloy | compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ alloy_dir }}/docker-compose.yml" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + register: _alloy_compose + +- name: alloy | start / recreate container + ansible.builtin.command: + cmd: >- + docker compose up -d --remove-orphans + {{ '--force-recreate' if (_alloy_compose is changed or _alloy_config is changed) else '' }} + chdir: "{{ alloy_dir }}" + become: true + become_user: "{{ admin_user }}" + register: _alloy_up + changed_when: >- + _alloy_up.rc == 0 and ( + 'Started' in (_alloy_up.stdout | default('')) + or 'Recreated' in (_alloy_up.stdout | default('')) + or 'Created' in (_alloy_up.stdout | default('')) + or _alloy_compose is changed + or _alloy_config is changed + ) diff --git a/roles/alloy/templates/config.alloy.j2 b/roles/alloy/templates/config.alloy.j2 new file mode 100644 index 0000000..e1cb2d3 --- /dev/null +++ b/roles/alloy/templates/config.alloy.j2 @@ -0,0 +1,206 @@ +// Managed by Ansible (roles/alloy). Do not edit by hand. +// host label = inventory hostname: {{ inventory_hostname }} + +// ===================================================================== +// LOGS +// ===================================================================== + +loki.source.journal "system" { + forward_to = [loki.process.journal_labels.receiver] + relabel_rules = discovery.relabel.journal.rules + labels = { + job = "systemd", + host = "{{ inventory_hostname }}", + } +} + +discovery.relabel "journal" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } +} + +loki.process "journal_labels" { + forward_to = [loki.write.default.receiver] + + stage.static_labels { + values = { + env = "host", + app = "system", + } + } +} + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "docker" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "compose_project" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.docker.output + forward_to = [loki.process.docker_labels.receiver] + labels = { + job = "docker", + host = "{{ inventory_hostname }}", + } +} + +loki.process "docker_labels" { + forward_to = [loki.write.default.receiver] + + stage.regex { + source = "compose_project" + expression = "^(?P[a-z0-9_]+)_(?Pbeta|prod)$" + } + + stage.labels { + values = { + app = "", + env = "", + } + } + +{% raw %} + stage.template { + source = "env" + template = `{{ if .env }}{{ .env }}{{ else }}infra{{ end }}` + } + + stage.template { + source = "app" + template = `{{ if .app }}{{ .app }}{{ else }}{{ .compose_project }}{{ end }}` + } +{% endraw %} + + stage.labels { + values = { + app = "", + env = "", + } + } +} + +loki.write "default" { + endpoint { + url = "{{ alloy_loki_push_url }}" + } +} + +// ===================================================================== +// METRICS — host (CPU / RAM / disk / load / network) +// ===================================================================== + +prometheus.exporter.unix "node" { + procfs_path = "/host/proc" + sysfs_path = "/host/sys" + rootfs_path = "/host/root" +} + +prometheus.scrape "node" { + targets = prometheus.exporter.unix.node.targets + forward_to = [prometheus.relabel.add_host.receiver] + scrape_interval = "15s" + job_name = "node" +} + +// ===================================================================== +// METRICS — Docker containers (cAdvisor) +// ===================================================================== + +prometheus.exporter.cadvisor "docker" { + docker_host = "unix:///var/run/docker.sock" + docker_only = true + storage_duration = "5m" + store_container_labels = false + allowlisted_container_labels = [ + "com.docker.compose.project", + "com.docker.compose.service", + ] +} + +prometheus.scrape "cadvisor" { + targets = prometheus.exporter.cadvisor.docker.targets + forward_to = [prometheus.relabel.cadvisor_labels.receiver] + scrape_interval = "15s" + job_name = "cadvisor" +} + +prometheus.relabel "cadvisor_labels" { + forward_to = [prometheus.remote_write.default.receiver] + + rule { + target_label = "host" + replacement = "{{ inventory_hostname }}" + } + + rule { + source_labels = ["container_label_com_docker_compose_project"] + target_label = "compose_project" + } + + rule { + source_labels = ["compose_project"] + regex = "^([a-z0-9_]+)_(beta|prod)$" + target_label = "app" + replacement = "${1}" + } + + rule { + source_labels = ["compose_project"] + regex = "^([a-z0-9_]+)_(beta|prod)$" + target_label = "env" + replacement = "${2}" + } + + rule { + source_labels = ["env"] + regex = "^$" + target_label = "env" + replacement = "infra" + } + + rule { + source_labels = ["app"] + regex = "^$" + target_label = "app" + replacement = "infra" + } +} + +prometheus.relabel "add_host" { + forward_to = [prometheus.remote_write.default.receiver] + + rule { + target_label = "host" + replacement = "{{ inventory_hostname }}" + } +} + +prometheus.remote_write "default" { + endpoint { + url = "{{ alloy_prometheus_remote_write_url }}" + } +} diff --git a/roles/alloy/templates/docker-compose.yml.j2 b/roles/alloy/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..17f6168 --- /dev/null +++ b/roles/alloy/templates/docker-compose.yml.j2 @@ -0,0 +1,29 @@ +# Managed by Ansible (roles/alloy). Do not edit by hand. +services: + alloy: + image: {{ alloy_image }} + container_name: alloy + restart: unless-stopped + privileged: true + pid: host + command: + - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy/data + - --server.http.listen-addr=0.0.0.0:12345 + volumes: + - ./config.alloy:/etc/alloy/config.alloy:ro + - alloy-data:/var/lib/alloy/data + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log/journal:/var/log/journal:ro + - /etc/machine-id:/etc/machine-id:ro + - /run/systemd/journal:/run/systemd/journal:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/host/root:ro + - /var/run:/var/run:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + +volumes: + alloy-data: diff --git a/roles/observability/defaults/main.yml b/roles/observability/defaults/main.yml new file mode 100644 index 0000000..292f82d --- /dev/null +++ b/roles/observability/defaults/main.yml @@ -0,0 +1,23 @@ +--- +observability_dir: "{{ apps_base_dir }}/observability" +observability_loki_image: "grafana/loki:3.4.2" +observability_prometheus_image: "prom/prometheus:v3.2.1" +observability_grafana_image: "grafana/grafana:11.5.2" + +# Container process UIDs for bind-mounted data dirs (official images). +observability_prometheus_uid: 65534 # nobody +observability_prometheus_gid: 65534 +observability_grafana_uid: 472 +observability_grafana_gid: 472 + +# Override in host_vars or secrets; change on first login if left default. +observability_grafana_admin_user: admin +observability_grafana_admin_password: "CHANGE_ME_ON_FIRST_LOGIN" +observability_grafana_public_url: "{{ grafana_public_url }}" +observability_grafana_domain: "{{ grafana_public_domain }}" + +observability_loki_retention: 744h +observability_prometheus_retention: 31d + +# LAN CIDR allowed to reach Loki / Prometheus / Grafana on this host. +observability_ufw_from: "{{ ufw_ssh_allowed_network }}" diff --git a/roles/observability/tasks/main.yml b/roles/observability/tasks/main.yml new file mode 100644 index 0000000..50647a7 --- /dev/null +++ b/roles/observability/tasks/main.yml @@ -0,0 +1,128 @@ +--- +# Central Loki + Prometheus + Grafana stack (ai-server-4080 only). + +- name: observability | ensure project directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0750" + loop: + - "{{ observability_dir }}" + - "{{ observability_dir }}/loki" + - "{{ observability_dir }}/loki/data" + - "{{ observability_dir }}/loki/rules" + - "{{ observability_dir }}/prometheus" + - "{{ observability_dir }}/grafana" + - "{{ observability_dir }}/grafana/provisioning" + - "{{ observability_dir }}/grafana/provisioning/datasources" + +# Official images drop privileges; bind mounts must match container UIDs. +# Mode 0755 — container UIDs must write even when parent dirs are 0750 admin-owned. +- name: observability | Prometheus data dir (nobody) + ansible.builtin.file: + path: "{{ observability_dir }}/prometheus/data" + state: directory + owner: "{{ observability_prometheus_uid }}" + group: "{{ observability_prometheus_gid }}" + mode: "0755" + +- name: observability | Grafana data dir + ansible.builtin.file: + path: "{{ observability_dir }}/grafana/data" + state: directory + owner: "{{ observability_grafana_uid }}" + group: "{{ observability_grafana_gid }}" + mode: "0755" + +- name: observability | Loki config + ansible.builtin.template: + src: loki-config.yml.j2 + dest: "{{ observability_dir }}/loki/loki-config.yml" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + register: _obs_loki_cfg + +- name: observability | Prometheus config + ansible.builtin.template: + src: prometheus.yml.j2 + dest: "{{ observability_dir }}/prometheus/prometheus.yml" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + register: _obs_prom_cfg + +- name: observability | Grafana datasources + ansible.builtin.template: + src: grafana-datasources.yml.j2 + dest: "{{ observability_dir }}/grafana/provisioning/datasources/datasources.yml" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + register: _obs_grafana_ds + +- name: observability | compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ observability_dir }}/docker-compose.yml" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + register: _obs_compose + +- name: observability | allow Loki from LAN + community.general.ufw: + rule: allow + port: "3100" + proto: tcp + from_ip: "{{ observability_ufw_from }}" + comment: Loki ingest from Alloy + +- name: observability | allow Prometheus from LAN + community.general.ufw: + rule: allow + port: "9090" + proto: tcp + from_ip: "{{ observability_ufw_from }}" + comment: Prometheus remote-write from Alloy + +- name: observability | allow Grafana from LAN + community.general.ufw: + rule: allow + port: "3000" + proto: tcp + from_ip: "{{ observability_ufw_from }}" + comment: Grafana for NPM / LAN + +- name: observability | start stack + ansible.builtin.command: + cmd: >- + docker compose up -d --remove-orphans + {{ '--force-recreate' if ( + _obs_compose is changed + or _obs_loki_cfg is changed + or _obs_prom_cfg is changed + or _obs_grafana_ds is changed + ) else '' }} + chdir: "{{ observability_dir }}" + become: true + become_user: "{{ admin_user }}" + register: _obs_up + changed_when: >- + _obs_up.rc == 0 and ( + 'Started' in (_obs_up.stdout | default('')) + or 'Recreated' in (_obs_up.stdout | default('')) + or 'Created' in (_obs_up.stdout | default('')) + or _obs_compose is changed + or _obs_loki_cfg is changed + or _obs_prom_cfg is changed + or _obs_grafana_ds is changed + ) + failed_when: _obs_up.rc != 0 + +- name: observability | show compose failure output + ansible.builtin.debug: + msg: "{{ _obs_up.stderr_lines | default(_obs_up.stdout_lines) }}" + when: _obs_up is failed diff --git a/roles/observability/templates/docker-compose.yml.j2 b/roles/observability/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..938452a --- /dev/null +++ b/roles/observability/templates/docker-compose.yml.j2 @@ -0,0 +1,60 @@ +# Managed by Ansible (roles/observability). Do not edit by hand. +services: + loki: + image: {{ observability_loki_image }} + container_name: loki + restart: unless-stopped + user: "0:0" + command: -config.file=/etc/loki/loki-config.yml + ports: + - "3100:3100" + volumes: + - ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro + - ./loki/data:/loki + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3100/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + + prometheus: + image: {{ observability_prometheus_image }} + container_name: prometheus + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time={{ observability_prometheus_retention }} + - --web.enable-remote-write-receiver + - --web.enable-lifecycle + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/data:/prometheus + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9090/-/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + + grafana: + image: {{ observability_grafana_image }} + container_name: grafana + restart: unless-stopped + depends_on: + loki: + condition: service_healthy + prometheus: + condition: service_healthy + ports: + - "3000:3000" + environment: + GF_SECURITY_ADMIN_USER: "{{ observability_grafana_admin_user }}" + GF_SECURITY_ADMIN_PASSWORD: "{{ observability_grafana_admin_password }}" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_SERVER_ROOT_URL: "{{ observability_grafana_public_url }}" + GF_SERVER_DOMAIN: "{{ observability_grafana_domain }}" + volumes: + - ./grafana/data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro diff --git a/roles/observability/templates/grafana-datasources.yml.j2 b/roles/observability/templates/grafana-datasources.yml.j2 new file mode 100644 index 0000000..c3c745a --- /dev/null +++ b/roles/observability/templates/grafana-datasources.yml.j2 @@ -0,0 +1,21 @@ +# Managed by Ansible (roles/observability). Do not edit by hand. +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: false + editable: false + jsonData: + maxLines: 1000 + + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 15s diff --git a/roles/observability/templates/loki-config.yml.j2 b/roles/observability/templates/loki-config.yml.j2 new file mode 100644 index 0000000..0cf92b0 --- /dev/null +++ b/roles/observability/templates/loki-config.yml.j2 @@ -0,0 +1,47 @@ +# Managed by Ansible (roles/observability). Do not edit by hand. +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + max_query_series: 500 + retention_period: {{ observability_loki_retention }} + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + delete_request_store: filesystem + +ruler: + alertmanager_url: http://localhost:9093 diff --git a/roles/observability/templates/prometheus.yml.j2 b/roles/observability/templates/prometheus.yml.j2 new file mode 100644 index 0000000..3dcf993 --- /dev/null +++ b/roles/observability/templates/prometheus.yml.j2 @@ -0,0 +1,13 @@ +# Managed by Ansible (roles/observability). Do not edit by hand. +global: + scrape_interval: 15s + evaluation_interval: 15s + +# Alloy on each host pushes metrics via remote_write. +# Local scrape keeps Prometheus self-health visible. +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + labels: + host: "{{ inventory_hostname }}" diff --git a/scripts/provision.sh b/scripts/provision.sh index ea6cf79..7ee4f6c 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -13,6 +13,10 @@ Usage: $(basename "$0") [HOST] [OPTIONS] Provision server(s) with site.yml. +Applies: common, ufw, docker, nodejs, gitea-key, tianji, alloy (every host). +On ai-server-4080 also: observability (Loki + Prometheus + Grafana) when +observability_stack is true in host_vars. + HOST Optional. Limit to one host: adama, roslin, or ai-server-4080. Omit to run against all webservers. @@ -25,9 +29,9 @@ Options: Examples: $(basename "$0") adama --check # dry run on adama only - $(basename "$0") adama # provision adama + $(basename "$0") adama # provision adama (includes Alloy) $(basename "$0") adama --ask-pass # first SSH login before ssh-copy-id - $(basename "$0") ai-server-4080 # provision the control node + $(basename "$0") ai-server-4080 # control node + Loki/Prometheus/Grafana $(basename "$0") # provision all hosts EOF }