Updates for grafana stack
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
---
|
||||
alloy_image: "grafana/alloy:v1.7.1"
|
||||
alloy_dir: "{{ apps_base_dir }}/observability/alloy"
|
||||
alloy_loki_push_url: "{{ loki_push_url }}"
|
||||
alloy_prometheus_remote_write_url: "{{ prometheus_remote_write_url }}"
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
# Ship host/container logs → Loki and metrics → Prometheus via Grafana Alloy
|
||||
# (Promtail successor; also collects node + cAdvisor metrics).
|
||||
|
||||
- name: alloy | ensure project directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0750"
|
||||
loop:
|
||||
- "{{ apps_base_dir }}/observability"
|
||||
- "{{ alloy_dir }}"
|
||||
|
||||
- name: alloy | config
|
||||
ansible.builtin.template:
|
||||
src: config.alloy.j2
|
||||
dest: "{{ alloy_dir }}/config.alloy"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _alloy_config
|
||||
|
||||
- name: alloy | compose file
|
||||
ansible.builtin.template:
|
||||
src: docker-compose.yml.j2
|
||||
dest: "{{ alloy_dir }}/docker-compose.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _alloy_compose
|
||||
|
||||
- name: alloy | start / recreate container
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
docker compose up -d --remove-orphans
|
||||
{{ '--force-recreate' if (_alloy_compose is changed or _alloy_config is changed) else '' }}
|
||||
chdir: "{{ alloy_dir }}"
|
||||
become: true
|
||||
become_user: "{{ admin_user }}"
|
||||
register: _alloy_up
|
||||
changed_when: >-
|
||||
_alloy_up.rc == 0 and (
|
||||
'Started' in (_alloy_up.stdout | default(''))
|
||||
or 'Recreated' in (_alloy_up.stdout | default(''))
|
||||
or 'Created' in (_alloy_up.stdout | default(''))
|
||||
or _alloy_compose is changed
|
||||
or _alloy_config is changed
|
||||
)
|
||||
@@ -0,0 +1,206 @@
|
||||
// Managed by Ansible (roles/alloy). Do not edit by hand.
|
||||
// host label = inventory hostname: {{ inventory_hostname }}
|
||||
|
||||
// =====================================================================
|
||||
// LOGS
|
||||
// =====================================================================
|
||||
|
||||
loki.source.journal "system" {
|
||||
forward_to = [loki.process.journal_labels.receiver]
|
||||
relabel_rules = discovery.relabel.journal.rules
|
||||
labels = {
|
||||
job = "systemd",
|
||||
host = "{{ inventory_hostname }}",
|
||||
}
|
||||
}
|
||||
|
||||
discovery.relabel "journal" {
|
||||
targets = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
}
|
||||
|
||||
loki.process "journal_labels" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
|
||||
stage.static_labels {
|
||||
values = {
|
||||
env = "host",
|
||||
app = "system",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
discovery.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
}
|
||||
|
||||
discovery.relabel "docker" {
|
||||
targets = discovery.docker.containers.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.*)"
|
||||
target_label = "container"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
|
||||
target_label = "compose_project"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
target_label = "compose_service"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.relabel.docker.output
|
||||
forward_to = [loki.process.docker_labels.receiver]
|
||||
labels = {
|
||||
job = "docker",
|
||||
host = "{{ inventory_hostname }}",
|
||||
}
|
||||
}
|
||||
|
||||
loki.process "docker_labels" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
|
||||
stage.regex {
|
||||
source = "compose_project"
|
||||
expression = "^(?P<app>[a-z0-9_]+)_(?P<env>beta|prod)$"
|
||||
}
|
||||
|
||||
stage.labels {
|
||||
values = {
|
||||
app = "",
|
||||
env = "",
|
||||
}
|
||||
}
|
||||
|
||||
{% raw %}
|
||||
stage.template {
|
||||
source = "env"
|
||||
template = `{{ if .env }}{{ .env }}{{ else }}infra{{ end }}`
|
||||
}
|
||||
|
||||
stage.template {
|
||||
source = "app"
|
||||
template = `{{ if .app }}{{ .app }}{{ else }}{{ .compose_project }}{{ end }}`
|
||||
}
|
||||
{% endraw %}
|
||||
|
||||
stage.labels {
|
||||
values = {
|
||||
app = "",
|
||||
env = "",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "{{ alloy_loki_push_url }}"
|
||||
}
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
// METRICS — host (CPU / RAM / disk / load / network)
|
||||
// =====================================================================
|
||||
|
||||
prometheus.exporter.unix "node" {
|
||||
procfs_path = "/host/proc"
|
||||
sysfs_path = "/host/sys"
|
||||
rootfs_path = "/host/root"
|
||||
}
|
||||
|
||||
prometheus.scrape "node" {
|
||||
targets = prometheus.exporter.unix.node.targets
|
||||
forward_to = [prometheus.relabel.add_host.receiver]
|
||||
scrape_interval = "15s"
|
||||
job_name = "node"
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
// METRICS — Docker containers (cAdvisor)
|
||||
// =====================================================================
|
||||
|
||||
prometheus.exporter.cadvisor "docker" {
|
||||
docker_host = "unix:///var/run/docker.sock"
|
||||
docker_only = true
|
||||
storage_duration = "5m"
|
||||
store_container_labels = false
|
||||
allowlisted_container_labels = [
|
||||
"com.docker.compose.project",
|
||||
"com.docker.compose.service",
|
||||
]
|
||||
}
|
||||
|
||||
prometheus.scrape "cadvisor" {
|
||||
targets = prometheus.exporter.cadvisor.docker.targets
|
||||
forward_to = [prometheus.relabel.cadvisor_labels.receiver]
|
||||
scrape_interval = "15s"
|
||||
job_name = "cadvisor"
|
||||
}
|
||||
|
||||
prometheus.relabel "cadvisor_labels" {
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
|
||||
rule {
|
||||
target_label = "host"
|
||||
replacement = "{{ inventory_hostname }}"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["container_label_com_docker_compose_project"]
|
||||
target_label = "compose_project"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["compose_project"]
|
||||
regex = "^([a-z0-9_]+)_(beta|prod)$"
|
||||
target_label = "app"
|
||||
replacement = "${1}"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["compose_project"]
|
||||
regex = "^([a-z0-9_]+)_(beta|prod)$"
|
||||
target_label = "env"
|
||||
replacement = "${2}"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["env"]
|
||||
regex = "^$"
|
||||
target_label = "env"
|
||||
replacement = "infra"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["app"]
|
||||
regex = "^$"
|
||||
target_label = "app"
|
||||
replacement = "infra"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.relabel "add_host" {
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
|
||||
rule {
|
||||
target_label = "host"
|
||||
replacement = "{{ inventory_hostname }}"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.remote_write "default" {
|
||||
endpoint {
|
||||
url = "{{ alloy_prometheus_remote_write_url }}"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# Managed by Ansible (roles/alloy). Do not edit by hand.
|
||||
services:
|
||||
alloy:
|
||||
image: {{ alloy_image }}
|
||||
container_name: alloy
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
pid: host
|
||||
command:
|
||||
- run
|
||||
- /etc/alloy/config.alloy
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
volumes:
|
||||
- ./config.alloy:/etc/alloy/config.alloy:ro
|
||||
- alloy-data:/var/lib/alloy/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /var/log/journal:/var/log/journal:ro
|
||||
- /etc/machine-id:/etc/machine-id:ro
|
||||
- /run/systemd/journal:/run/systemd/journal:ro
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/host/root:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /var/lib/docker:/var/lib/docker:ro
|
||||
- /dev/disk:/dev/disk:ro
|
||||
|
||||
volumes:
|
||||
alloy-data:
|
||||
@@ -0,0 +1,23 @@
|
||||
---
|
||||
observability_dir: "{{ apps_base_dir }}/observability"
|
||||
observability_loki_image: "grafana/loki:3.4.2"
|
||||
observability_prometheus_image: "prom/prometheus:v3.2.1"
|
||||
observability_grafana_image: "grafana/grafana:11.5.2"
|
||||
|
||||
# Container process UIDs for bind-mounted data dirs (official images).
|
||||
observability_prometheus_uid: 65534 # nobody
|
||||
observability_prometheus_gid: 65534
|
||||
observability_grafana_uid: 472
|
||||
observability_grafana_gid: 472
|
||||
|
||||
# Override in host_vars or secrets; change on first login if left default.
|
||||
observability_grafana_admin_user: admin
|
||||
observability_grafana_admin_password: "CHANGE_ME_ON_FIRST_LOGIN"
|
||||
observability_grafana_public_url: "{{ grafana_public_url }}"
|
||||
observability_grafana_domain: "{{ grafana_public_domain }}"
|
||||
|
||||
observability_loki_retention: 744h
|
||||
observability_prometheus_retention: 31d
|
||||
|
||||
# LAN CIDR allowed to reach Loki / Prometheus / Grafana on this host.
|
||||
observability_ufw_from: "{{ ufw_ssh_allowed_network }}"
|
||||
@@ -0,0 +1,128 @@
|
||||
---
|
||||
# Central Loki + Prometheus + Grafana stack (ai-server-4080 only).
|
||||
|
||||
- name: observability | ensure project directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0750"
|
||||
loop:
|
||||
- "{{ observability_dir }}"
|
||||
- "{{ observability_dir }}/loki"
|
||||
- "{{ observability_dir }}/loki/data"
|
||||
- "{{ observability_dir }}/loki/rules"
|
||||
- "{{ observability_dir }}/prometheus"
|
||||
- "{{ observability_dir }}/grafana"
|
||||
- "{{ observability_dir }}/grafana/provisioning"
|
||||
- "{{ observability_dir }}/grafana/provisioning/datasources"
|
||||
|
||||
# Official images drop privileges; bind mounts must match container UIDs.
|
||||
# Mode 0755 — container UIDs must write even when parent dirs are 0750 admin-owned.
|
||||
- name: observability | Prometheus data dir (nobody)
|
||||
ansible.builtin.file:
|
||||
path: "{{ observability_dir }}/prometheus/data"
|
||||
state: directory
|
||||
owner: "{{ observability_prometheus_uid }}"
|
||||
group: "{{ observability_prometheus_gid }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: observability | Grafana data dir
|
||||
ansible.builtin.file:
|
||||
path: "{{ observability_dir }}/grafana/data"
|
||||
state: directory
|
||||
owner: "{{ observability_grafana_uid }}"
|
||||
group: "{{ observability_grafana_gid }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: observability | Loki config
|
||||
ansible.builtin.template:
|
||||
src: loki-config.yml.j2
|
||||
dest: "{{ observability_dir }}/loki/loki-config.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_loki_cfg
|
||||
|
||||
- name: observability | Prometheus config
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ observability_dir }}/prometheus/prometheus.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_prom_cfg
|
||||
|
||||
- name: observability | Grafana datasources
|
||||
ansible.builtin.template:
|
||||
src: grafana-datasources.yml.j2
|
||||
dest: "{{ observability_dir }}/grafana/provisioning/datasources/datasources.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_grafana_ds
|
||||
|
||||
- name: observability | compose file
|
||||
ansible.builtin.template:
|
||||
src: docker-compose.yml.j2
|
||||
dest: "{{ observability_dir }}/docker-compose.yml"
|
||||
owner: "{{ admin_user }}"
|
||||
group: "{{ admin_user }}"
|
||||
mode: "0644"
|
||||
register: _obs_compose
|
||||
|
||||
- name: observability | allow Loki from LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "3100"
|
||||
proto: tcp
|
||||
from_ip: "{{ observability_ufw_from }}"
|
||||
comment: Loki ingest from Alloy
|
||||
|
||||
- name: observability | allow Prometheus from LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "9090"
|
||||
proto: tcp
|
||||
from_ip: "{{ observability_ufw_from }}"
|
||||
comment: Prometheus remote-write from Alloy
|
||||
|
||||
- name: observability | allow Grafana from LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "3000"
|
||||
proto: tcp
|
||||
from_ip: "{{ observability_ufw_from }}"
|
||||
comment: Grafana for NPM / LAN
|
||||
|
||||
- name: observability | start stack
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
docker compose up -d --remove-orphans
|
||||
{{ '--force-recreate' if (
|
||||
_obs_compose is changed
|
||||
or _obs_loki_cfg is changed
|
||||
or _obs_prom_cfg is changed
|
||||
or _obs_grafana_ds is changed
|
||||
) else '' }}
|
||||
chdir: "{{ observability_dir }}"
|
||||
become: true
|
||||
become_user: "{{ admin_user }}"
|
||||
register: _obs_up
|
||||
changed_when: >-
|
||||
_obs_up.rc == 0 and (
|
||||
'Started' in (_obs_up.stdout | default(''))
|
||||
or 'Recreated' in (_obs_up.stdout | default(''))
|
||||
or 'Created' in (_obs_up.stdout | default(''))
|
||||
or _obs_compose is changed
|
||||
or _obs_loki_cfg is changed
|
||||
or _obs_prom_cfg is changed
|
||||
or _obs_grafana_ds is changed
|
||||
)
|
||||
failed_when: _obs_up.rc != 0
|
||||
|
||||
- name: observability | show compose failure output
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ _obs_up.stderr_lines | default(_obs_up.stdout_lines) }}"
|
||||
when: _obs_up is failed
|
||||
@@ -0,0 +1,60 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
services:
|
||||
loki:
|
||||
image: {{ observability_loki_image }}
|
||||
container_name: loki
|
||||
restart: unless-stopped
|
||||
user: "0:0"
|
||||
command: -config.file=/etc/loki/loki-config.yml
|
||||
ports:
|
||||
- "3100:3100"
|
||||
volumes:
|
||||
- ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro
|
||||
- ./loki/data:/loki
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3100/ready || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
prometheus:
|
||||
image: {{ observability_prometheus_image }}
|
||||
container_name: prometheus
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ observability_prometheus_retention }}
|
||||
- --web.enable-remote-write-receiver
|
||||
- --web.enable-lifecycle
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/data:/prometheus
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9090/-/ready || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
grafana:
|
||||
image: {{ observability_grafana_image }}
|
||||
container_name: grafana
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_healthy
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: "{{ observability_grafana_admin_user }}"
|
||||
GF_SECURITY_ADMIN_PASSWORD: "{{ observability_grafana_admin_password }}"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_SERVER_ROOT_URL: "{{ observability_grafana_public_url }}"
|
||||
GF_SERVER_DOMAIN: "{{ observability_grafana_domain }}"
|
||||
volumes:
|
||||
- ./grafana/data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
@@ -0,0 +1,21 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
isDefault: false
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
@@ -0,0 +1,47 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: info
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
ingestion_rate_mb: 16
|
||||
ingestion_burst_size_mb: 32
|
||||
max_query_series: 500
|
||||
retention_period: {{ observability_loki_retention }}
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
compaction_interval: 10m
|
||||
retention_enabled: true
|
||||
retention_delete_delay: 2h
|
||||
delete_request_store: filesystem
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
@@ -0,0 +1,13 @@
|
||||
# Managed by Ansible (roles/observability). Do not edit by hand.
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
# Alloy on each host pushes metrics via remote_write.
|
||||
# Local scrape keeps Prometheus self-health visible.
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
labels:
|
||||
host: "{{ inventory_hostname }}"
|
||||
Reference in New Issue
Block a user