Add observability platform Ansible project
Deploys the Grafana stack (Prometheus/Alertmanager/Loki/Tempo/Grafana with Alloy as collector) and the ELK stack (Filebeat/Logstash/Elasticsearch/ Kibana) via Docker Compose on a target host. External to the application repo, pre-built by the mentor.
This commit is contained in:
Movses Aslanyan
@@ -0,0 +1,7 @@
|
||||
---
|
||||
- name: Restart observability stack
|
||||
community.docker.docker_compose_v2:
|
||||
project_src: "{{ stack_root }}"
|
||||
state: restarted
|
||||
when: stack_root is defined
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
---
|
||||
- name: Validate selected stack
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- observability_stack in ['elk-grafana', 'grafana-stack']
|
||||
fail_msg: observability_stack must be elk-grafana or grafana-stack.
|
||||
|
||||
- name: Set stack deployment directory
|
||||
ansible.builtin.set_fact:
|
||||
stack_root: "{{ observability_root }}/{{ observability_stack }}"
|
||||
|
||||
- name: Create stack directory tree
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop: >-
|
||||
{{
|
||||
[stack_root, stack_root + '/config']
|
||||
+ (
|
||||
[stack_root + '/config/filebeat', stack_root + '/config/logstash/pipeline', stack_root + '/config/grafana/provisioning/datasources']
|
||||
if observability_stack == 'elk-grafana'
|
||||
else [stack_root + '/config/prometheus/rules', stack_root + '/config/alertmanager', stack_root + '/config/loki', stack_root + '/config/tempo', stack_root + '/config/alloy', stack_root + '/config/grafana/provisioning/datasources']
|
||||
)
|
||||
}}
|
||||
|
||||
- name: Apply Elasticsearch kernel requirement
|
||||
ansible.posix.sysctl:
|
||||
name: vm.max_map_count
|
||||
value: "262144"
|
||||
state: present
|
||||
sysctl_set: true
|
||||
reload: true
|
||||
when: observability_stack == 'elk-grafana'
|
||||
|
||||
- name: Render ELK and Grafana files
|
||||
ansible.builtin.template:
|
||||
src: "elk-grafana/{{ item.src }}"
|
||||
dest: "{{ stack_root }}/{{ item.dest }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "{{ item.mode | default('0644') }}"
|
||||
loop:
|
||||
- { src: docker-compose.yml.j2, dest: docker-compose.yml }
|
||||
- { src: config/filebeat/filebeat.yml.j2, dest: config/filebeat/filebeat.yml }
|
||||
- { src: config/logstash/pipeline/logstash.conf.j2, dest: config/logstash/pipeline/logstash.conf }
|
||||
- { src: config/grafana/provisioning/datasources/datasources.yml.j2, dest: config/grafana/provisioning/datasources/datasources.yml }
|
||||
when: observability_stack == 'elk-grafana'
|
||||
notify: Restart observability stack
|
||||
|
||||
- name: Render Grafana stack files
|
||||
ansible.builtin.template:
|
||||
src: "grafana-stack/{{ item.src }}"
|
||||
dest: "{{ stack_root }}/{{ item.dest }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "{{ item.mode | default('0644') }}"
|
||||
loop:
|
||||
- { src: docker-compose.yml.j2, dest: docker-compose.yml }
|
||||
- { src: config/prometheus/prometheus.yml.j2, dest: config/prometheus/prometheus.yml }
|
||||
- { src: config/prometheus/rules/host.rules.yml.j2, dest: config/prometheus/rules/host.rules.yml }
|
||||
- { src: config/alertmanager/alertmanager.yml.j2, dest: config/alertmanager/alertmanager.yml }
|
||||
- { src: config/loki/loki.yml.j2, dest: config/loki/loki.yml }
|
||||
- { src: config/tempo/tempo.yml.j2, dest: config/tempo/tempo.yml }
|
||||
- { src: config/alloy/config.alloy.j2, dest: config/alloy/config.alloy }
|
||||
- { src: config/grafana/provisioning/datasources/datasources.yml.j2, dest: config/grafana/provisioning/datasources/datasources.yml }
|
||||
when: observability_stack == 'grafana-stack'
|
||||
notify: Restart observability stack
|
||||
|
||||
- name: Start selected observability stack
|
||||
community.docker.docker_compose_v2:
|
||||
project_src: "{{ stack_root }}"
|
||||
state: present
|
||||
pull: missing
|
||||
remove_orphans: true
|
||||
|
||||
- name: Flush restart handler before health checks
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Wait for Grafana
|
||||
ansible.builtin.uri:
|
||||
url: http://127.0.0.1:3000/api/health
|
||||
status_code: 200
|
||||
register: grafana_health
|
||||
retries: 30
|
||||
delay: 5
|
||||
until: grafana_health.status == 200
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
filebeat.inputs:
|
||||
- type: filestream
|
||||
id: docker-containers
|
||||
enabled: true
|
||||
paths:
|
||||
- /var/lib/docker/containers/*/*.log
|
||||
parsers:
|
||||
- container:
|
||||
stream: all
|
||||
format: docker
|
||||
|
||||
processors:
|
||||
- add_docker_metadata:
|
||||
host: unix:///var/run/docker.sock
|
||||
- add_host_metadata: ~
|
||||
|
||||
output.logstash:
|
||||
hosts: ["logstash:5044"]
|
||||
|
||||
logging.level: info
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Elasticsearch
|
||||
uid: elasticsearch
|
||||
type: elasticsearch
|
||||
access: proxy
|
||||
url: http://elasticsearch:9200
|
||||
database: "logs-*"
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
index: "logs-*"
|
||||
interval: Daily
|
||||
timeField: "@timestamp"
|
||||
logMessageField: message
|
||||
logLevelField: log.level
|
||||
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
}
|
||||
tcp {
|
||||
port => 5000
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if [message] =~ /^\s*\{/ {
|
||||
json {
|
||||
source => "message"
|
||||
skip_on_invalid_json => true
|
||||
}
|
||||
}
|
||||
mutate {
|
||||
add_field => { "[@metadata][target_index]" => "logs-%{+YYYY.MM.dd}" }
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["http://elasticsearch:9200"]
|
||||
index => "%{[@metadata][target_index]}"
|
||||
}
|
||||
stdout { codec => dots }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
services:
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:{{ elastic_version }}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
xpack.security.enabled: "false"
|
||||
xpack.security.enrollment.enabled: "false"
|
||||
ES_JAVA_OPTS: "-Xms{{ elasticsearch_heap }} -Xmx{{ elasticsearch_heap }}"
|
||||
volumes:
|
||||
- elasticsearch-data:/usr/share/elasticsearch/data
|
||||
ports:
|
||||
- "9200:9200"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
networks: [observability]
|
||||
|
||||
logstash:
|
||||
image: docker.elastic.co/logstash/logstash:{{ elastic_version }}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./config/logstash/pipeline:/usr/share/logstash/pipeline:ro
|
||||
- logstash-data:/usr/share/logstash/data
|
||||
ports:
|
||||
- "5044:5044"
|
||||
- "5000:5000/tcp"
|
||||
networks: [observability]
|
||||
|
||||
filebeat:
|
||||
image: docker.elastic.co/beats/filebeat:{{ elastic_version }}
|
||||
user: root
|
||||
restart: unless-stopped
|
||||
command: ["filebeat", "-e", "--strict.perms=false"]
|
||||
depends_on: [logstash]
|
||||
volumes:
|
||||
- ./config/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- filebeat-data:/usr/share/filebeat/data
|
||||
networks: [observability]
|
||||
|
||||
kibana:
|
||||
image: docker.elastic.co/kibana/kibana:{{ elastic_version }}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||
ports:
|
||||
- "5601:5601"
|
||||
networks: [observability]
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:{{ grafana_version }}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: "{{ grafana_admin_user }}"
|
||||
GF_SECURITY_ADMIN_PASSWORD: "{{ grafana_admin_password }}"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./config/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks: [observability]
|
||||
|
||||
volumes:
|
||||
elasticsearch-data:
|
||||
logstash-data:
|
||||
filebeat-data:
|
||||
grafana-data:
|
||||
|
||||
networks:
|
||||
observability:
|
||||
driver: bridge
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: default
|
||||
group_by: [alertname, instance]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
|
||||
receivers:
|
||||
- name: default
|
||||
{% if alertmanager_webhook_url | length > 0 %}
|
||||
webhook_configs:
|
||||
- url: "{{ alertmanager_webhook_url }}"
|
||||
send_resolved: true
|
||||
{% endif %}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
logging {
|
||||
level = "info"
|
||||
}
|
||||
|
||||
discovery.docker "local" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
}
|
||||
|
||||
discovery.relabel "docker" {
|
||||
targets = discovery.docker.local.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.*)"
|
||||
target_label = "container"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "local" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.relabel.docker.output
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.receiver.otlp "default" {
|
||||
grpc {
|
||||
endpoint = "0.0.0.0:4317"
|
||||
}
|
||||
http {
|
||||
endpoint = "0.0.0.0:4318"
|
||||
}
|
||||
output {
|
||||
traces = [otelcol.exporter.otlp.tempo.input]
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.exporter.otlp "tempo" {
|
||||
client {
|
||||
endpoint = "tempo:4317"
|
||||
tls {
|
||||
insecure = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
exemplarTraceIdDestinations:
|
||||
- datasourceUid: tempo
|
||||
name: trace_id
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: true
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: 'trace[_-]?id["=: ]+([a-fA-F0-9]+)'
|
||||
name: TraceID
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: true
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
spanStartTimeShift: -1m
|
||||
spanEndTimeShift: 1m
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
|
||||
- name: Alertmanager
|
||||
uid: alertmanager
|
||||
type: alertmanager
|
||||
access: proxy
|
||||
url: http://alertmanager:9093
|
||||
editable: true
|
||||
jsonData:
|
||||
implementation: prometheus
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /var/loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
instance_addr: 127.0.0.1
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
filesystem:
|
||||
directory: /var/loki/chunks
|
||||
|
||||
compactor:
|
||||
working_directory: /var/loki/compactor
|
||||
|
||||
limits_config:
|
||||
retention_period: 336h
|
||||
allow_structured_metadata: true
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: ["alertmanager:9093"]
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["prometheus:9090"]
|
||||
|
||||
- job_name: node-exporter
|
||||
static_configs:
|
||||
- targets: ["node-exporter:9100"]
|
||||
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ["loki:3100"]
|
||||
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: ["tempo:3200"]
|
||||
|
||||
- job_name: alloy
|
||||
static_configs:
|
||||
- targets: ["alloy:12345"]
|
||||
|
||||
- job_name: grafana
|
||||
static_configs:
|
||||
- targets: ["grafana:3000"]
|
||||
|
||||
{% for job in prometheus_extra_scrape_jobs %}
|
||||
- job_name: "{{ job.job_name }}"
|
||||
static_configs:
|
||||
- targets: {{ job.targets | to_json }}
|
||||
{% endfor %}
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
groups:
|
||||
- name: host.rules
|
||||
rules:
|
||||
- alert: HostExporterDown
|
||||
expr: up{job="node-exporter"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node exporter is unavailable"
|
||||
description: "Prometheus has failed to scrape {{ '{{' }} $labels.instance {{ '}}' }} for more than two minutes."
|
||||
|
||||
- alert: HostHighCPU
|
||||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU utilization"
|
||||
description: "CPU utilization on {{ '{{' }} $labels.instance {{ '}}' }} has exceeded 85% for ten minutes."
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 336h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics, local-blocks]
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:{{ prometheus_version }}
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --web.enable-lifecycle
|
||||
- --web.enable-remote-write-receiver
|
||||
volumes:
|
||||
- ./config/prometheus:/etc/prometheus:ro
|
||||
- prometheus-data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks: [observability]
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:{{ alertmanager_version }}
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --config.file=/etc/alertmanager/alertmanager.yml
|
||||
- --storage.path=/alertmanager
|
||||
volumes:
|
||||
- ./config/alertmanager:/etc/alertmanager:ro
|
||||
- alertmanager-data:/alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
networks: [observability]
|
||||
|
||||
loki:
|
||||
image: grafana/loki:{{ loki_version }}
|
||||
restart: unless-stopped
|
||||
command: ["-config.file=/etc/loki/loki.yml"]
|
||||
volumes:
|
||||
- ./config/loki/loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki-data:/var/loki
|
||||
ports:
|
||||
- "3100:3100"
|
||||
networks: [observability]
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:{{ tempo_version }}
|
||||
restart: unless-stopped
|
||||
command: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
volumes:
|
||||
- ./config/tempo/tempo.yml:/etc/tempo/tempo.yml:ro
|
||||
- tempo-data:/var/tempo
|
||||
ports:
|
||||
- "3200:3200"
|
||||
networks: [observability]
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:{{ node_exporter_version }}
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --path.rootfs=/host
|
||||
pid: host
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
ports:
|
||||
- "9100:9100"
|
||||
networks: [observability]
|
||||
|
||||
alloy:
|
||||
image: grafana/alloy:{{ alloy_version }}
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- run
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- /etc/alloy/config.alloy
|
||||
volumes:
|
||||
- ./config/alloy/config.alloy:/etc/alloy/config.alloy:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- alloy-data:/var/lib/alloy/data
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "4317:4317"
|
||||
- "4318:4318"
|
||||
networks: [observability]
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:{{ grafana_version }}
|
||||
restart: unless-stopped
|
||||
depends_on: [prometheus, loki, tempo]
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: "{{ grafana_admin_user }}"
|
||||
GF_SECURITY_ADMIN_PASSWORD: "{{ grafana_admin_password }}"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./config/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks: [observability]
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
alertmanager-data:
|
||||
loki-data:
|
||||
tempo-data:
|
||||
alloy-data:
|
||||
grafana-data:
|
||||
|
||||
networks:
|
||||
observability:
|
||||
driver: bridge
|
||||
|
||||
Reference in New Issue
Block a user