Updates for grafana stack
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
---
|
||||
observability_dir: "{{ apps_base_dir }}/observability"
|
||||
observability_loki_image: "grafana/loki:3.4.2"
|
||||
observability_prometheus_image: "prom/prometheus:v3.2.1"
|
||||
observability_grafana_image: "grafana/grafana:11.5.2"
|
||||
|
||||
# Container process UIDs for bind-mounted data dirs (official images).
|
||||
observability_prometheus_uid: 65534 # nobody
|
||||
observability_prometheus_gid: 65534
|
||||
observability_grafana_uid: 472
|
||||
observability_grafana_gid: 472
|
||||
|
||||
# Override in host_vars or secrets; change on first login if left default.
|
||||
observability_grafana_admin_user: admin
|
||||
observability_grafana_admin_password: "CHANGE_ME_ON_FIRST_LOGIN"
|
||||
observability_grafana_public_url: "{{ grafana_public_url }}"
|
||||
observability_grafana_domain: "{{ grafana_public_domain }}"
|
||||
|
||||
observability_loki_retention: 744h
|
||||
observability_prometheus_retention: 31d
|
||||
|
||||
# LAN CIDR allowed to reach Loki / Prometheus / Grafana on this host.
|
||||
observability_ufw_from: "{{ ufw_ssh_allowed_network }}"
|
||||
@@ -0,0 +1,128 @@
|
||||
---
|
||||
# Central Loki + Prometheus + Grafana stack (ai-server-4080 only).
|
||||
|
||||
- name: observability | ensure project directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0750"
|
||||
loop:
|
||||
- "{{ observability_dir }}"
|
||||
- "{{ observability_dir }}/loki"
|
||||
- "{{ observability_dir }}/loki/data"
|
||||
- "{{ observability_dir }}/loki/rules"
|
||||
- "{{ observability_dir }}/prometheus"
|
||||
- "{{ observability_dir }}/grafana"
|
||||
- "{{ observability_dir }}/grafana/provisioning"
|
||||
- "{{ observability_dir }}/grafana/provisioning/datasources"
|
||||
|
||||
# Official images drop privileges; bind mounts must match container UIDs.
|
||||
# Mode 0755 — container UIDs must write even when parent dirs are 0750 admin-owned.
|
||||
- name: observability | Prometheus data dir (nobody)
|
||||
ansible.builtin.file:
|
||||
path: "{{ observability_dir }}/prometheus/data"
|
||||
state: directory
|
||||
owner: "{{ observability_prometheus_uid }}"
|
||||
group: "{{ observability_prometheus_gid }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: observability | Grafana data dir
|
||||
ansible.builtin.file:
|
||||
path: "{{ observability_dir }}/grafana/data"
|
||||
state: directory
|
||||
owner: "{{ observability_grafana_uid }}"
|
||||
group: "{{ observability_grafana_gid }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: observability | Loki config
|
||||
ansible.builtin.template:
|
||||
src: loki-config.yml.j2
|
||||
dest: "{{ observability_dir }}/loki/loki-config.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_loki_cfg
|
||||
|
||||
- name: observability | Prometheus config
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ observability_dir }}/prometheus/prometheus.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_prom_cfg
|
||||
|
||||
- name: observability | Grafana datasources
|
||||
ansible.builtin.template:
|
||||
src: grafana-datasources.yml.j2
|
||||
dest: "{{ observability_dir }}/grafana/provisioning/datasources/datasources.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_grafana_ds
|
||||
|
||||
- name: observability | compose file
|
||||
ansible.builtin.template:
|
||||
src: docker-compose.yml.j2
|
||||
dest: "{{ observability_dir }}/docker-compose.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_compose
|
||||
|
||||
- name: observability | allow Loki from LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "3100"
|
||||
proto: tcp
|
||||
from_ip: "{{ observability_ufw_from }}"
|
||||
comment: Loki ingest from Alloy
|
||||
|
||||
- name: observability | allow Prometheus from LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "9090"
|
||||
proto: tcp
|
||||
from_ip: "{{ observability_ufw_from }}"
|
||||
comment: Prometheus remote-write from Alloy
|
||||
|
||||
- name: observability | allow Grafana from LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "3000"
|
||||
proto: tcp
|
||||
from_ip: "{{ observability_ufw_from }}"
|
||||
comment: Grafana for NPM / LAN
|
||||
|
||||
- name: observability | start stack
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
docker compose up -d --remove-orphans
|
||||
{{ '--force-recreate' if (
|
||||
_obs_compose is changed
|
||||
or _obs_loki_cfg is changed
|
||||
or _obs_prom_cfg is changed
|
||||
or _obs_grafana_ds is changed
|
||||
) else '' }}
|
||||
chdir: "{{ observability_dir }}"
|
||||
become: true
|
||||
become_user: "{{ admin_user }}"
|
||||
register: _obs_up
|
||||
changed_when: >-
|
||||
_obs_up.rc == 0 and (
|
||||
'Started' in (_obs_up.stdout | default(''))
|
||||
or 'Recreated' in (_obs_up.stdout | default(''))
|
||||
or 'Created' in (_obs_up.stdout | default(''))
|
||||
or _obs_compose is changed
|
||||
or _obs_loki_cfg is changed
|
||||
or _obs_prom_cfg is changed
|
||||
or _obs_grafana_ds is changed
|
||||
)
|
||||
failed_when: _obs_up.rc != 0
|
||||
|
||||
- name: observability | show compose failure output
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ _obs_up.stderr_lines | default(_obs_up.stdout_lines) }}"
|
||||
when: _obs_up is failed
|
||||
@@ -0,0 +1,60 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
services:
|
||||
loki:
|
||||
image: {{ observability_loki_image }}
|
||||
container_name: loki
|
||||
restart: unless-stopped
|
||||
user: "0:0"
|
||||
command: -config.file=/etc/loki/loki-config.yml
|
||||
ports:
|
||||
- "3100:3100"
|
||||
volumes:
|
||||
- ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro
|
||||
- ./loki/data:/loki
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3100/ready || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
prometheus:
|
||||
image: {{ observability_prometheus_image }}
|
||||
container_name: prometheus
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ observability_prometheus_retention }}
|
||||
- --web.enable-remote-write-receiver
|
||||
- --web.enable-lifecycle
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/data:/prometheus
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9090/-/ready || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
grafana:
|
||||
image: {{ observability_grafana_image }}
|
||||
container_name: grafana
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_healthy
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: "{{ observability_grafana_admin_user }}"
|
||||
GF_SECURITY_ADMIN_PASSWORD: "{{ observability_grafana_admin_password }}"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_SERVER_ROOT_URL: "{{ observability_grafana_public_url }}"
|
||||
GF_SERVER_DOMAIN: "{{ observability_grafana_domain }}"
|
||||
volumes:
|
||||
- ./grafana/data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
@@ -0,0 +1,21 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
isDefault: false
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
@@ -0,0 +1,47 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: info
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
ingestion_rate_mb: 16
|
||||
ingestion_burst_size_mb: 32
|
||||
max_query_series: 500
|
||||
retention_period: {{ observability_loki_retention }}
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
compaction_interval: 10m
|
||||
retention_enabled: true
|
||||
retention_delete_delay: 2h
|
||||
delete_request_store: filesystem
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
@@ -0,0 +1,13 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
# Alloy on each host pushes metrics via remote_write.
|
||||
# Local scrape keeps Prometheus self-health visible.
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
labels:
|
||||
host: "{{ inventory_hostname }}"
|
||||
Reference in New Issue
Block a user