Updates for grafana stack

2026-07-10 06:50:52 -05:00
parent 728a0c60f2
commit f37ec41530
16 changed files with 652 additions and 97 deletions
@@ -0,0 +1,128 @@
+---
+# Central Loki + Prometheus + Grafana stack (ai-server-4080 only).
+
+- name: observability | ensure project directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0750"
+  loop:
+    - "{{ observability_dir }}"
+    - "{{ observability_dir }}/loki"
+    - "{{ observability_dir }}/loki/data"
+    - "{{ observability_dir }}/loki/rules"
+    - "{{ observability_dir }}/prometheus"
+    - "{{ observability_dir }}/grafana"
+    - "{{ observability_dir }}/grafana/provisioning"
+    - "{{ observability_dir }}/grafana/provisioning/datasources"
+
+# Official images drop privileges; bind mounts must match container UIDs.
+# Mode 0755 — container UIDs must write even when parent dirs are 0750 admin-owned.
+- name: observability | Prometheus data dir (nobody)
+  ansible.builtin.file:
+    path: "{{ observability_dir }}/prometheus/data"
+    state: directory
+    owner: "{{ observability_prometheus_uid }}"
+    group: "{{ observability_prometheus_gid }}"
+    mode: "0755"
+
+- name: observability | Grafana data dir
+  ansible.builtin.file:
+    path: "{{ observability_dir }}/grafana/data"
+    state: directory
+    owner: "{{ observability_grafana_uid }}"
+    group: "{{ observability_grafana_gid }}"
+    mode: "0755"
+
+- name: observability | Loki config
+  ansible.builtin.template:
+    src: loki-config.yml.j2
+    dest: "{{ observability_dir }}/loki/loki-config.yml"
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0644"
+  register: _obs_loki_cfg
+
+- name: observability | Prometheus config
+  ansible.builtin.template:
+    src: prometheus.yml.j2
+    dest: "{{ observability_dir }}/prometheus/prometheus.yml"
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0644"
+  register: _obs_prom_cfg
+
+- name: observability | Grafana datasources
+  ansible.builtin.template:
+    src: grafana-datasources.yml.j2
+    dest: "{{ observability_dir }}/grafana/provisioning/datasources/datasources.yml"
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0644"
+  register: _obs_grafana_ds
+
+- name: observability | compose file
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ observability_dir }}/docker-compose.yml"
+    owner: "{{ admin_user }}"
+    group: "{{ admin_user }}"
+    mode: "0644"
+  register: _obs_compose
+
+- name: observability | allow Loki from LAN
+  community.general.ufw:
+    rule: allow
+    port: "3100"
+    proto: tcp
+    from_ip: "{{ observability_ufw_from }}"
+    comment: Loki ingest from Alloy
+
+- name: observability | allow Prometheus from LAN
+  community.general.ufw:
+    rule: allow
+    port: "9090"
+    proto: tcp
+    from_ip: "{{ observability_ufw_from }}"
+    comment: Prometheus remote-write from Alloy
+
+- name: observability | allow Grafana from LAN
+  community.general.ufw:
+    rule: allow
+    port: "3000"
+    proto: tcp
+    from_ip: "{{ observability_ufw_from }}"
+    comment: Grafana for NPM / LAN
+
+- name: observability | start stack
+  ansible.builtin.command:
+    cmd: >-
+      docker compose up -d --remove-orphans
+      {{ '--force-recreate' if (
+        _obs_compose is changed
+        or _obs_loki_cfg is changed
+        or _obs_prom_cfg is changed
+        or _obs_grafana_ds is changed
+      ) else '' }}
+    chdir: "{{ observability_dir }}"
+  become: true
+  become_user: "{{ admin_user }}"
+  register: _obs_up
+  changed_when: >-
+    _obs_up.rc == 0 and (
+      'Started' in (_obs_up.stdout | default(''))
+      or 'Recreated' in (_obs_up.stdout | default(''))
+      or 'Created' in (_obs_up.stdout | default(''))
+      or _obs_compose is changed
+      or _obs_loki_cfg is changed
+      or _obs_prom_cfg is changed
+      or _obs_grafana_ds is changed
+    )
+  failed_when: _obs_up.rc != 0
+
+- name: observability | show compose failure output
+  ansible.builtin.debug:
+    msg: "{{ _obs_up.stderr_lines | default(_obs_up.stdout_lines) }}"
+  when: _obs_up is failed