commit b36fe93ed7a6cd274d5303314009b612e2986152 parent 49d66050cafd50efd6fef84bcc54b7e3b25123c8 Author: Florian Dold <dold@taler.net> Date: Fri, 4 Sep 2026 18:00:12 +0200 monitoring: retire legacy stack and add teardown playbook Diffstat:
30 files changed, 205 insertions(+), 1342 deletions(-)
diff --git a/README b/README @@ -70,6 +70,19 @@ host and referenced from the exchange configuration. NOTE: this should still be further automated. +### remove-monitoring.sh + +The legacy monitoring stack is no longer deployed by this repository. To +permanently remove its services, packages, configuration, local metrics data, +nginx site, certificate, and database user from an existing deployment, run: + +``` +$ ./remove-monitoring.sh $DEPLOYMENT +``` + +This is destructive. It does not back up the local Prometheus or Alloy data. + + ### Setting up backups (TOPS-only for now) First run: @@ -167,6 +180,10 @@ pixel still needs to be setup to send the backups. Safely reboots the system by first stopping all Taler services, then making a backup and only then actually rebooting it. +### remove-monitoring + +Permanently removes the legacy monitoring infrastructure and its local data. + ### sanctionlist-check Imports the latest sanction list and checks all records against @@ -242,7 +259,7 @@ Deploys libeufin-nexus which connects us to the bank. ### monitoring -Deploys Alloy and Promethesus exporters for host monitoring. +Contains the cleanup tasks used by the remove-monitoring playbook. ### pixel_borg diff --git a/inventories/group_vars/all/defaults.yml b/inventories/group_vars/all/defaults.yml @@ -6,9 +6,6 @@ deploy_auditor: true # Deploy challenger? deploy_challenger: false -# Deploy monitoring? -deploy_monitoring: true - # If true, use EBICS keys from that were externally created. ebics_keys_external: false @@ -49,9 +46,6 @@ nexus_domain: "nexus.{{ domain_name }}" # one (for example "trixie-testing"). taler_repo_suites: "{{ ansible_facts['distribution_release'] }}" -# Name identifying this host towards the monitoring backends. -target_host_name: "{{ inventory_hostname }}" - # Use letsencrypt by default exchange_use_letsencrypt: true nexus_use_letsencrypt: true diff --git a/inventories/group_vars/testing/test-secrets.yml b/inventories/group_vars/testing/test-secrets.yml @@ -23,9 +23,3 @@ exchange_kycaid_access_token: FIXME # Bearer access token for the auditor auditor_access_token: secret-token:FIXME - -# Bearer access token for monitoring -prometheus_access_token: secret-token:FIXME - -# Bearer access token for loki.taler-systems.com -loki_access_token: secret-token:FIXME diff --git a/inventories/host_vars/fdold-acai-tops/test-public.yml b/inventories/host_vars/fdold-acai-tops/test-public.yml @@ -10,8 +10,6 @@ deployment_kind: "tops" # Main domain name. domain_name: "topstest.fdold.eu" exchange_domain: "exchange.{{ domain_name }}" -# Our internal hostname -target_host_name: "acai.box.fdold.eu" # Deploy EBICS configuration (true/false). use_ebics: false # Our currency. diff --git a/inventories/host_vars/fdold-acai-tops/test-secrets.yml b/inventories/host_vars/fdold-acai-tops/test-secrets.yml @@ -22,9 +22,3 @@ exchange_kycaid_access_token: FIXME # Bearer access token for the auditor auditor_access_token: secret-token:FIXME - -# Bearer access token for monitoring -prometheus_access_token: secret-token:FIXME - -# Bearer access token for loki.taler-systems.com -loki_access_token: secret-token:FIXME diff --git a/inventories/host_vars/podman-localhost/test-secrets.yml b/inventories/host_vars/podman-localhost/test-secrets.yml @@ -22,9 +22,3 @@ exchange_kycaid_access_token: FIXME # Bearer access token for the auditor auditor_access_token: secret-token:FIXME - -# Bearer access token for monitoring -prometheus_access_token: secret-token:FIXME - -# Bearer access token for loki.taler-systems.com -loki_access_token: secret-token:FIXME diff --git a/inventories/host_vars/rusty/vars.yml b/inventories/host_vars/rusty/vars.yml @@ -3,13 +3,8 @@ deployment_kind: "tops" # Public variables for a "test" deployment # Deploy challenger? deploy_challenger: true -# No monitoring for rusty, -# monitoring stack will be replaced. -deploy_monitoring: false # Main external domain name. domain_name: "stage.taler-ops.ch" -# Our internal hostname -target_host_name: "rusty.taler-ops.ch" # Suite for taler packages. taler_repo_suites: trixie-testing # Deploy EBICS configuration (true/false). @@ -126,9 +121,3 @@ exchange_kycaid_access_token: "{{ vault_exchange_kycaid_access_token }}" # Bearer access token for the auditor SPA (set via browser extension to set Authorization HTTP header on auditor.$DOMAIN!) auditor_access_token: "{{ vault_auditor_access_token }}" - -# Bearer access token for monitoring.$DOMAIN (must be given to grafana) -prometheus_access_token: "{{ vault_prometheus_access_token }}" - -# Bearer access token for loki.taler-systems.com (see that nginx config) -loki_access_token: "{{ vault_loki_access_token }}" diff --git a/inventories/host_vars/spec/vars.yml b/inventories/host_vars/spec/vars.yml @@ -8,8 +8,6 @@ configure_ebics: true # Main domain name. domain_name: "taler-ops.ch" exchange_domain: "exchange.{{ domain_name }}" -# Our internal hostname -target_host_name: "spec.taler-ops.ch" # Suite for taler packages. taler_repo_suites: trixie # Deploy EBICS configuration (true/false). @@ -100,9 +98,3 @@ exchange_kycaid_access_token: "{{ vault_exchange_kycaid_access_token }}" # Bearer access token for the auditor SPA (set via browser extension to set Authorization HTTP header on auditor.$DOMAIN!) auditor_access_token: "{{ vault_auditor_access_token }}" - -# Bearer access token for monitoring.$DOMAIN (must be given to grafana) -prometheus_access_token: "{{ vault_prometheus_access_token }}" - -# Bearer access token for loki.taler-systems.com (see that nginx config) -loki_access_token: "{{ vault_loki_access_token }}" diff --git a/playbooks/remove-monitoring.yml b/playbooks/remove-monitoring.yml @@ -0,0 +1,9 @@ +--- +- name: Remove the legacy monitoring infrastructure + hosts: all + any_errors_fatal: true + tasks: + - name: Remove the legacy monitoring infrastructure + ansible.builtin.include_role: + name: monitoring + tasks_from: remove diff --git a/playbooks/setup.yml b/playbooks/setup.yml @@ -34,20 +34,6 @@ that: auditor_access_token is defined quiet: true - - name: "Check the monitoring secrets" - when: deploy_monitoring | bool - ansible.builtin.assert: - that: - - loki_access_token is defined - - prometheus_access_token is defined - quiet: true - - - name: Disable existing monitoring stack - ansible.builtin.include_role: - name: monitoring - tasks_from: disable - when: not (deploy_monitoring | bool) - - name: "Check the challenger secrets" when: deploy_challenger | bool ansible.builtin.assert: @@ -83,8 +69,6 @@ when: deploy_challenger | bool - role: auditor when: deploy_auditor | bool - - role: monitoring - when: deploy_monitoring | bool - role: devtesting when: dangerously_enable_devtesting | bool diff --git a/remove-monitoring.sh b/remove-monitoring.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu + +if [ -z "${1:-}" ] +then + echo "Call with 'spec' or another host/group to select target" + exit 1 +fi + +ansible-playbook \ + --inventory inventories/default \ + --limit "$1" \ + playbooks/remove-monitoring.yml + +exit 0 diff --git a/roles/monitoring/files/etc/apt/keyrings/grafana.gpg b/roles/monitoring/files/etc/apt/keyrings/grafana.gpg Binary files differ. diff --git a/roles/monitoring/files/etc/default/alloy b/roles/monitoring/files/etc/default/alloy @@ -1,9 +0,0 @@ - -# The configuration file holding the Alloy config. -CONFIG_FILE="/etc/alloy/config.alloy" - -# User-defined arguments to pass to the run command. -CUSTOM_ARGS="--server.http.listen-addr=127.0.0.1:12345 --disable-reporting" - -# Restart on system upgrade. Defaults to true. -RESTART_ON_UPGRADE=true diff --git a/roles/monitoring/files/etc/default/prometheus b/roles/monitoring/files/etc/default/prometheus @@ -1,5 +0,0 @@ -# Set the command-line arguments to pass to the server. -# Due to shell escaping, to pass backslashes for regexes, you need to double -# them (\\d for \d). If running under systemd, you need to double them again -# (\\\\d to mean \d), and escape newlines too. -ARGS="--web.listen-address=127.0.0.1:9090 --web.enable-remote-write-receiver" diff --git a/roles/monitoring/files/etc/default/prometheus-alertmanager b/roles/monitoring/files/etc/default/prometheus-alertmanager @@ -1,5 +0,0 @@ -# Set the command-line arguments to pass to the server. -# Due to shell escaping, to pass backslashes for regexes, you need to double -# them (\\d for \d). If running under systemd, you need to double them again -# (\\\\d to mean \d), and escape newlines too. -ARGS="--web.listen-address=127.0.0.1:9093 --cluster.listen-address=127.0.0.1:9094" diff --git a/roles/monitoring/files/etc/default/prometheus-nginx-exporter b/roles/monitoring/files/etc/default/prometheus-nginx-exporter @@ -1,69 +0,0 @@ -# Set the command-line arguments to pass to the server. -# Due to shell escaping, to pass backslashes for regexes, you need to double -# them (\\d for \d). If running under systemd, you need to double them again -# (\\\\d to mean \d), and escape newlines too. -ARGS="-web.listen-address 127.0.0.1:9113 -nginx.scrape-uri http://127.0.0.1:80/metrics" - -# See monitoring-nginx.conf for the scrape-uri export! - - - -# prometheus-nginx-exporter supports the following options: -# -# -nginx.plus -# Start the exporter for NGINX Plus. By default, the exporter is started for -# NGINX. The default value can be overwritten by NGINX_PLUS environment -# variable. -# -nginx.retries uint -# A number of retries the exporter will make on start to connect to the -# NGINX stub_status page/NGINX Plus API before exiting with an error. The -# default value can be overwritten by NGINX_RETRIES environment variable. -# -nginx.retry-interval value -# An interval between retries to connect to the NGINX stub_status page/NGINX -# Plus API on start. The default value can be overwritten by -# NGINX_RETRY_INTERVAL environment variable. (default 5s). -# -nginx.scrape-uri string -# A URI or unix domain socket path for scraping NGINX or NGINX Plus metrics. -# For NGINX, the stub_status page must be available through the URI. For -# NGINX Plus -- the API. The default value can be overwritten by SCRAPE_URI -# environment variable. (default "http://127.0.0.1:8080/stub_status"). -# -nginx.ssl-ca-cert string -# Path to the PEM encoded CA certificate file used to validate the servers -# SSL certificate. The default value can be overwritten by SSL_CA_CERT -# environment variable. -# -nginx.ssl-client-cert string -# Path to the PEM encoded client certificate file to use when connecting to -# the server. The default value can be overwritten by SSL_CLIENT_CERT -# environment variable. -# -nginx.ssl-client-key string -# Path to the PEM encoded client certificate key file to use when connecting -# to the server. The default value can be overwritten by SSL_CLIENT_KEY -# environment variable. -# -nginx.ssl-verify -# Perform SSL certificate verification. The default value can be overwritten -# by SSL_VERIFY environment variable. (default true). -# -nginx.timeout value -# A timeout for scraping metrics from NGINX or NGINX Plus. The default value -# can be overwritten by TIMEOUT environment variable. (default 5s). -# -prometheus.const-labels value -# A comma separated list of constant labels that will be used in every -# metric. Format is label1=value1,label2=value2... The default value can be -# overwritten by CONST_LABELS environment variable. -# -web.listen-address string -# An address or unix domain socket path to listen on for web interface and -# telemetry. The default value can be overwritten by LISTEN_ADDRESS -# environment variable. (default ":9113"). -# -web.secured-metrics -# Expose metrics using https. The default value can be overwritten by -# SECURED_METRICS variable. -# -web.ssl-server-cert string -# Path to the PEM encoded certificate for the nginx-exporter metrics -# server(when web.secured-metrics=true). The default value can be -# overwritten by SSL_SERVER_CERT variable. -# -web.ssl-server-key string -# Path to the PEM encoded key for the nginx-exporter metrics server (when -# web.secured-metrics=true). The default value can be overwritten by -# SSL_SERVER_KEY variable. -# -web.telemetry-path string -# A path under which to expose metrics. The default value can be overwritten -# by TELEMETRY_PATH environment variable. (default "/metrics"). diff --git a/roles/monitoring/files/etc/default/prometheus-node-exporter b/roles/monitoring/files/etc/default/prometheus-node-exporter @@ -1,5 +0,0 @@ -# Set the command-line arguments to pass to the server. -# Due to shell escaping, to pass backslashes for regexes, you need to double -# them (\\d for \d). If running under systemd, you need to double them again -# (\\\\d to mean \d), and escape newlines too. -ARGS="--web.listen-address 127.0.0.1:9100" diff --git a/roles/monitoring/files/etc/default/prometheus-postgres-exporter b/roles/monitoring/files/etc/default/prometheus-postgres-exporter @@ -1,13 +0,0 @@ -# Connection string for the PostgreSQL database. You need to either connect as -# superuser, or create a user with enough rights, as described in -# /usr/share/doc/prometheus-postgres-exporter/README.Debian - -# DATA_SOURCE_NAME='postgresql://login:password@hostname:port/' -DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres' -# DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=taler-exchange' - -# Set the command-line arguments to pass to the server. -# Due to shell escaping, to pass backslashes for regexes, you need to double -# them (\\d for \d). If running under systemd, you need to double them again -# (\\\\d to mean \d), and escape newlines too. -ARGS="--web.listen-address=127.0.0.1:9187" diff --git a/roles/monitoring/files/etc/keyrings/grafana.gpg b/roles/monitoring/files/etc/keyrings/grafana.gpg Binary files differ. diff --git a/roles/monitoring/files/etc/prometheus/alert_rules.yml b/roles/monitoring/files/etc/prometheus/alert_rules.yml @@ -1,29 +0,0 @@ -groups: -- name: node_exporter_alerts - rules: - - alert: HighCPULatency - expr: sum(rate(node_cpu_seconds_total{mode="system"}[1m])) / count(node_cpu_seconds_total{mode="system"}) * 100 > 80 - for: 1m - labels: - severity: warning - annotations: - summary: "High CPU Latency detected" - description: "CPU latency is above 80% for more than 1 minute." - - - alert: LowDiskSpace - expr: (node_filesystem_free_bytes / node_filesystem_size_bytes) * 100 < 50 - for: 1m - labels: - severity: critical - annotations: - summary: "Low Disk Space detected" - description: "Disk space is below 50% for more than 1 minute." - - - alert: HighMemoryUsage - expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 80 - for: 1m - labels: - severity: warning - annotations: - summary: "High Memory Usage detected" - description: "Memory usage is above 80% for more than 1 minute." diff --git a/roles/monitoring/files/etc/prometheus/node-exporter-rules.yml b/roles/monitoring/files/etc/prometheus/node-exporter-rules.yml @@ -1,311 +0,0 @@ -groups: - -- name: NodeExporter - - rules: - - - alert: HostOutOfMemory - expr: '(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .20)' - for: 2m - labels: - severity: warning - annotations: - summary: Host out of memory (instance {{ $labels.instance }}) - description: "Node memory is filling up (< 20% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostMemoryUnderMemoryPressure - expr: '(rate(node_vmstat_pgmajfault[5m]) > 1000)' - for: 0m - labels: - severity: warning - annotations: - summary: Host memory under memory pressure (instance {{ $labels.instance }}) - description: "The node is under heavy memory pressure. High rate of loading memory pages from disk.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostUnusualNetworkThroughputIn - expr: '((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)' - for: 0m - labels: - severity: warning - annotations: - summary: Host unusual network throughput in (instance {{ $labels.instance }}) - description: "Host receive bandwidth is high (>80%).\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostUnusualNetworkThroughputOut - expr: '((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)' - for: 0m - labels: - severity: warning - annotations: - summary: Host unusual network throughput out (instance {{ $labels.instance }}) - description: "Host transmit bandwidth is high (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostUnusualDiskReadRate - expr: '(rate(node_disk_io_time_seconds_total[5m]) > .80)' - for: 0m - labels: - severity: warning - annotations: - summary: Host unusual disk read rate (instance {{ $labels.instance }}) - description: "Disk is too busy (IO wait > 80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostOutOfDiskSpace - expr: '(node_filesystem_avail_bytes{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"} / node_filesystem_size_bytes < .50 and on (instance, device, mountpoint) node_filesystem_readonly == 0)' - for: 2m - labels: - severity: critical - annotations: - summary: Host out of disk space (instance {{ $labels.instance }}) - description: "Disk is almost full (< 50% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostDiskMayFillIn24Hours - expr: 'predict_linear(node_filesystem_avail_bytes{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"}[1h], 86400) <= 0 and node_filesystem_avail_bytes > 0' - for: 2m - labels: - severity: warning - annotations: - summary: Host disk may fill in 24 hours (instance {{ $labels.instance }}) - description: "Filesystem will likely run out of space within the next 24 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostOutOfInodes - expr: '(node_filesystem_files_free / node_filesystem_files < .50 and ON (instance, device, mountpoint) node_filesystem_readonly == 0)' - for: 2m - labels: - severity: critical - annotations: - summary: Host out of inodes (instance {{ $labels.instance }}) - description: "Disk is almost running out of available inodes (< 50% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostFilesystemDeviceError - expr: 'node_filesystem_device_error{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"} == 1' - for: 2m - labels: - severity: critical - annotations: - summary: Host filesystem device error (instance {{ $labels.instance }}) - description: "Error stat-ing the {{ $labels.mountpoint }} filesystem\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostInodesMayFillIn24Hours - expr: 'predict_linear(node_filesystem_files_free{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"}[1h], 86400) <= 0 and node_filesystem_files_free > 0' - for: 2m - labels: - severity: warning - annotations: - summary: Host inodes may fill in 24 hours (instance {{ $labels.instance }}) - description: "Filesystem will likely run out of inodes within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostUnusualDiskReadLatency - expr: '(rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0)' - for: 2m - labels: - severity: warning - annotations: - summary: Host unusual disk read latency (instance {{ $labels.instance }}) - description: "Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostUnusualDiskWriteLatency - expr: '(rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0)' - for: 2m - labels: - severity: warning - annotations: - summary: Host unusual disk write latency (instance {{ $labels.instance }}) - description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostHighCpuLoad - expr: '(avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > .80' - for: 10m - labels: - severity: warning - annotations: - summary: Host high CPU load (instance {{ $labels.instance }}) - description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostCpuStealNoisyNeighbor - expr: 'avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10' - for: 0m - labels: - severity: warning - annotations: - summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }}) - description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostCpuHighIowait - expr: 'avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) > .10' - for: 0m - labels: - severity: warning - annotations: - summary: Host CPU high iowait (instance {{ $labels.instance }}) - description: "CPU iowait > 10%. Your CPU is idling waiting for storage to respond.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostUnusualDiskIo - expr: 'rate(node_disk_io_time_seconds_total[5m]) > 0.8' - for: 5m - labels: - severity: warning - annotations: - summary: Host unusual disk IO (instance {{ $labels.instance }}) - description: "Disk usage >80%. Check storage for issues or increase IOPS capabilities. Check storage for issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostContextSwitchingHigh - expr: '(rate(node_context_switches_total[15m])/count without(mode,cpu) (node_cpu_seconds_total{mode="idle"})) / (rate(node_context_switches_total[1d])/count without(mode,cpu) (node_cpu_seconds_total{mode="idle"})) > 2' - for: 0m - labels: - severity: warning - annotations: - summary: Host context switching high (instance {{ $labels.instance }}) - description: "Context switching is growing on the node (twice the daily average during the last 15m)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostSwapIsFillingUp - expr: '((1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80)' - for: 2m - labels: - severity: warning - annotations: - summary: Host swap is filling up (instance {{ $labels.instance }}) - description: "Swap is filling up (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostSystemdServiceCrashed - expr: '(node_systemd_unit_state{state="failed"} == 1)' - for: 0m - labels: - severity: warning - annotations: - summary: Host systemd service crashed (instance {{ $labels.instance }}) - description: "systemd service crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostPhysicalComponentTooHot - expr: 'node_hwmon_temp_celsius > node_hwmon_temp_max_celsius' - for: 5m - labels: - severity: warning - annotations: - summary: Host physical component too hot (instance {{ $labels.instance }}) - description: "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostNodeOvertemperatureAlarm - expr: '((node_hwmon_temp_crit_alarm_celsius == 1) or (node_hwmon_temp_alarm == 1))' - for: 0m - labels: - severity: critical - annotations: - summary: Host node overtemperature alarm (instance {{ $labels.instance }}) - description: "Physical node temperature alarm triggered\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostSoftwareRaidInsufficientDrives - expr: '((node_md_disks_required - on(device, instance) node_md_disks{state="active"}) > 0)' - for: 0m - labels: - severity: critical - annotations: - summary: Host software RAID insufficient drives (instance {{ $labels.instance }}) - description: "MD RAID array {{ $labels.device }} on {{ $labels.instance }} has insufficient drives remaining.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostSoftwareRaidDiskFailure - expr: '(node_md_disks{state="failed"} > 0)' - for: 2m - labels: - severity: warning - annotations: - summary: Host software RAID disk failure (instance {{ $labels.instance }}) - description: "MD RAID array {{ $labels.device }} on {{ $labels.instance }} needs attention.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostKernelVersionDeviations - expr: 'changes(node_uname_info[1h]) > 0' - for: 0m - labels: - severity: info - annotations: - summary: Host kernel version deviations (instance {{ $labels.instance }}) - description: "Kernel version for {{ $labels.instance }} has changed.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostOomKillDetected - expr: '(increase(node_vmstat_oom_kill[1m]) > 0)' - for: 0m - labels: - severity: warning - annotations: - summary: Host OOM kill detected (instance {{ $labels.instance }}) - description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostEdacCorrectableErrorsDetected - expr: '(increase(node_edac_correctable_errors_total[1m]) > 0)' - for: 0m - labels: - severity: info - annotations: - summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }}) - description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostEdacUncorrectableErrorsDetected - expr: '(node_edac_uncorrectable_errors_total > 0)' - for: 0m - labels: - severity: warning - annotations: - summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }}) - description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostNetworkReceiveErrors - expr: '(rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01)' - for: 2m - labels: - severity: warning - annotations: - summary: Host Network Receive Errors (instance {{ $labels.instance }}) - description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostNetworkTransmitErrors - expr: '(rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01)' - for: 2m - labels: - severity: warning - annotations: - summary: Host Network Transmit Errors (instance {{ $labels.instance }}) - description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostNetworkBondDegraded - expr: '((node_bonding_active - node_bonding_slaves) != 0)' - for: 2m - labels: - severity: warning - annotations: - summary: Host Network Bond Degraded (instance {{ $labels.instance }}) - description: "Bond \"{{ $labels.device }}\" degraded on \"{{ $labels.instance }}\".\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostConntrackLimit - expr: '(node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8)' - for: 5m - labels: - severity: warning - annotations: - summary: Host conntrack limit (instance {{ $labels.instance }}) - description: "The number of conntrack is approaching limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostClockSkew - expr: '((node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0))' - for: 10m - labels: - severity: warning - annotations: - summary: Host clock skew (instance {{ $labels.instance }}) - description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostClockNotSynchronising - expr: '(min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16)' - for: 2m - labels: - severity: warning - annotations: - summary: Host clock not synchronising (instance {{ $labels.instance }}) - description: "Clock not synchronising. Ensure NTP is configured on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - - alert: HostRequiresReboot - expr: '(node_reboot_required > 0)' - for: 4h - labels: - severity: info - annotations: - summary: Host requires reboot (instance {{ $labels.instance }}) - description: "{{ $labels.instance }} requires a reboot.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" diff --git a/roles/monitoring/files/etc/prometheus/prometheus.yml b/roles/monitoring/files/etc/prometheus/prometheus.yml @@ -1,57 +0,0 @@ -# my global config -global: - scrape_interval: 60s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 60s # Evaluate rules every 15 seconds. The default is every 1 minute. - # scrape_timeout is set to the global default (10s). - -# Alertmanager configuration -- FIXME: not yet setup! -alerting: - alertmanagers: - - static_configs: - - targets: ["localhost:9093"] - # - alertmanager:9093 - -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. -rule_files: - - "alert_rules.yml" - - "node-exporter-rules.yml" - # - "second_rules.yml" - -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. -scrape_configs: - # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. - - job_name: "prometheus" - - # metrics_path defaults to '/metrics' - # scheme defaults to 'http'. - - static_configs: - - targets: ["localhost:9090"] - - # Job, for local node exporter - - job_name: 'node_exporter_metrics' - scrape_interval: 60s - static_configs: - - targets: ['localhost:9100'] - - # Job, for local nginx exporter - - job_name: 'nginx_exporter_metrics' - scrape_interval: 60s - static_configs: - - targets: ['localhost:9113'] - - # Job, for local postgres_exporter - - job_name: 'postgres_exporter' - static_configs: - - targets: ['localhost:9187'] - - # Job, for prometheus_process_exporter - - job_name: 'process_exporter' - static_configs: - - targets: ['localhost:9256'] - - # Job, for Alloy - - job_name: 'alloy_exporter' - static_configs: - - targets: ['localhost:12345'] diff --git a/roles/monitoring/handlers/main.yml b/roles/monitoring/handlers/main.yml @@ -1,15 +1,5 @@ --- -- name: Restart postgres-exporter - service: - name: prometheus-postgres-exporter - state: restarted - -- name: Restart node-exporter - service: - name: prometheus-node-exporter - state: restarted - -- name: Restart nginx-exporter - service: - name: prometheus-nginx-exporter +- name: Restart nginx + ansible.builtin.service: + name: nginx state: restarted diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml @@ -1,232 +0,0 @@ ---- -- name: Get the list of services - service_facts: - -- name: Deploy grafana signing key - copy: - src: etc/apt/keyrings/grafana.gpg - dest: /etc/apt/keyrings/grafana.gpg - owner: root - group: root - mode: "0644" - -- name: Add grafana repo - deb822_repository: - name: Grafana - types: deb - uris: https://apt.grafana.com - suites: stable - components: - - main - architectures: amd64 - signed_by: /etc/apt/keyrings/grafana.gpg - -- name: Update caches - apt: - state: latest - update_cache: true - autoclean: true - autoremove: true - upgrade: safe - when: ansible_facts["os_family"] == 'Debian' - -- name: Stop log export service before update or reconfiguration - service: - name: alloy.service - state: stopped - enabled: false - when: '"alloy.service" in ansible_facts["services"]' - -- name: Stop monitoring services before update or reconfiguration - service: - name: "{{ item }}" - state: stopped - enabled: false - with_items: - - prometheus-node-exporter.service - - prometheus-nginx-exporter.service - - prometheus-postgres-exporter.service - - prometheus-alertmanager.service - - prometheus.service - when: '"prometheus-node-exporter.service" in ansible_facts["services"]' - -- name: Install prometheus and its exporters - apt: - name: - - prometheus-nginx-exporter - - prometheus-node-exporter - - prometheus-postgres-exporter - - prometheus-alertmanager - - prometheus - - alloy - install_recommends: false - -- name: Ensure Taler monitoring HTTP virtualhost configuration file exists - template: - src: templates/etc/nginx/sites-available/monitoring-http.conf.j2 - dest: /etc/nginx/sites-available/monitoring-http.conf - owner: root - group: root - mode: "0644" - notify: Restart nginx - -- name: Ensure Taler monitoring HTTPS configuration file exists - template: - src: templates/etc/nginx/sites-available/monitoring-nginx.conf.j2 - dest: /etc/nginx/sites-available/monitoring-nginx.conf - owner: root - group: root - mode: "0644" - notify: Restart nginx - -- name: Secure the monitoring site with Letsencrypt - ansible.builtin.include_role: - name: cert - vars: - cert_name: monitoring - wanted_cert_domains: - - "monitoring.{{ domain_name }}" - nginx_sites: - - monitoring-nginx.conf - - monitoring-http.conf - -# We need to make sure that our handler notifies nginx to restart NOW -- name: Flush handlers - meta: flush_handlers - -- name: Create prometheus database user - community.postgresql.postgresql_user: - name: prometheus - become: true - become_user: postgres - -- name: Grant access to canonical postgres databases to the postgres-exporter - become: true - become_user: postgres - community.postgresql.postgresql_query: - login_user: postgres - db: postgres - query: - GRANT CONNECT ON DATABASE libeufin TO prometheus; - GRANT CONNECT ON DATABASE postgres TO prometheus; - GRANT CONNECT ON DATABASE "taler-auditor" TO prometheus; - GRANT CONNECT ON DATABASE "taler-exchange" TO prometheus; - GRANT USAGE ON SCHEMA pg_catalog TO prometheus; - GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO prometheus; - -- name: Grant access to challengers databases to the postgres-exporter - become: true - become_user: postgres - community.postgresql.postgresql_query: - login_user: postgres - db: postgres - query: - GRANT CONNECT ON DATABASE "challenger-email" TO prometheus; - GRANT CONNECT ON DATABASE "challenger-postal" TO prometheus; - GRANT CONNECT ON DATABASE "challenger-sms" TO prometheus; - when: deploy_challenger | bool - -- name: Configure node-exporter - copy: - src: etc/default/prometheus-node-exporter - dest: /etc/default/prometheus-node-exporter - owner: root - group: root - mode: "0644" - notify: Restart node-exporter - -- name: Configure node-exporter - copy: - src: etc/default/prometheus-alertmanager - dest: /etc/default/prometheus-alertmanager - owner: root - group: root - mode: "0644" - notify: Restart node-exporter - -- name: Configure postgres-exporter - copy: - src: etc/default/prometheus-postgres-exporter - dest: /etc/default/prometheus-postgres-exporter - owner: root - group: root - mode: "0644" - notify: Restart postgres-exporter - -- name: Configure nginx-exporter - copy: - src: etc/default/prometheus-nginx-exporter - dest: /etc/default/prometheus-nginx-exporter - owner: root - group: root - mode: "0644" - notify: Restart nginx-exporter - -- name: Configure prometheus master - copy: - src: etc/default/prometheus - dest: /etc/default/prometheus - owner: root - group: root - mode: "0644" - -- name: Configure prometheus - copy: - src: etc/prometheus/prometheus.yml - dest: /etc/prometheus/prometheus.yml - owner: root - group: root - mode: "0644" - -- name: Configure alloy service - copy: - src: etc/default/alloy - dest: /etc/default/alloy - owner: root - group: root - mode: "0644" - -- name: Configure alloy log export - template: - src: templates/etc/alloy/config.alloy - dest: /etc/alloy/config.alloy - owner: root - group: root - mode: "0644" - -- name: Configure prometheus alertmanager - template: - src: templates/etc/prometheus/alertmanager.yml - dest: /etc/prometheus/alertmanager.yml - owner: root - group: root - mode: "0644" - -- name: Configure node-exporter rules for alertmanager - copy: - src: etc/prometheus/node-exporter-rules.yml - dest: /etc/prometheus/node-exporter-rules.yml - owner: root - group: root - mode: "0644" - -- name: Configure node-exporter rules for alertmanager - copy: - src: etc/prometheus/alert_rules.yml - dest: /etc/prometheus/alert_rules.yml - owner: root - group: root - mode: "0644" - -- name: Ensure exporter services are enabled and started - service: - name: "{{ item }}" - state: started - enabled: true - with_items: - - prometheus-node-exporter.service - - prometheus-nginx-exporter.service - - prometheus-postgres-exporter.service - - prometheus-alertmanager.service - - prometheus.service - - alloy.service diff --git a/roles/monitoring/tasks/remove.yml b/roles/monitoring/tasks/remove.yml @@ -0,0 +1,159 @@ +--- +- name: Stop the legacy monitoring services + ansible.builtin.include_tasks: disable.yml + +- name: Remove legacy monitoring nginx sites + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/nginx/sites-enabled/monitoring-nginx.conf + - /etc/nginx/sites-enabled/monitoring-http.conf + - /etc/nginx/sites-available/monitoring-nginx.conf + - /etc/nginx/sites-available/monitoring-http.conf + notify: Restart nginx + +# Reload nginx while the monitoring certificate still exists. This keeps a +# pending handler from trying to load a configuration that references a +# certificate which has already been removed. +- name: Apply removal of legacy monitoring nginx sites + ansible.builtin.meta: flush_handlers + +- name: Delete the legacy monitoring certificate with certbot + ansible.builtin.command: + argv: + - certbot + - delete + - --cert-name + - monitoring + - --non-interactive + removes: /etc/letsencrypt/renewal/monitoring.conf + +- name: Remove leftover legacy monitoring certificate files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/letsencrypt/live/monitoring + - /etc/letsencrypt/archive/monitoring + - /etc/letsencrypt/renewal/monitoring.conf + +- name: Remove the Grafana package repository + ansible.builtin.deb822_repository: + name: Grafana + state: absent + when: ansible_facts["os_family"] == 'Debian' + +- name: Remove the Grafana package repository signing key + ansible.builtin.file: + path: /etc/apt/keyrings/grafana.gpg + state: absent + +- name: Purge legacy monitoring packages + ansible.builtin.apt: + name: + - alloy + - prometheus + - prometheus-alertmanager + - prometheus-nginx-exporter + - prometheus-node-exporter + - prometheus-postgres-exporter + state: absent + purge: true + autoremove: true + when: ansible_facts["os_family"] == 'Debian' + +- name: Remove legacy monitoring configuration and data + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/default/alloy + - /etc/default/prometheus + - /etc/default/prometheus-alertmanager + - /etc/default/prometheus-nginx-exporter + - /etc/default/prometheus-node-exporter + - /etc/default/prometheus-postgres-exporter + - /etc/alloy + - /etc/prometheus + - /var/lib/alloy + - /var/lib/prometheus + +- name: Find legacy monitoring nginx logs + ansible.builtin.find: + paths: /var/log/nginx + patterns: monitoring.* + file_type: file + register: monitoring_nginx_logs + +- name: Remove legacy monitoring nginx logs + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ monitoring_nginx_logs.files }}" + loop_control: + label: "{{ item.path }}" + +- name: Check for the legacy Prometheus database user + become: true + become_user: postgres + community.postgresql.postgresql_query: + login_user: postgres + login_db: postgres + query: SELECT 1 FROM pg_roles WHERE rolname = 'prometheus' + register: monitoring_prometheus_role + changed_when: false + +- name: Find databases that received legacy monitoring access + become: true + become_user: postgres + community.postgresql.postgresql_query: + login_user: postgres + login_db: postgres + query: >- + SELECT datname + FROM pg_database + WHERE datname IN ( + 'libeufin', + 'postgres', + 'taler-auditor', + 'taler-exchange', + 'challenger-email', + 'challenger-postal', + 'challenger-sms' + ) + register: monitoring_databases + changed_when: false + +- name: Revoke database access granted to the Prometheus user + become: true + become_user: postgres + community.postgresql.postgresql_query: + login_user: postgres + login_db: postgres + query: >- + REVOKE CONNECT ON DATABASE "{{ item }}" FROM prometheus + loop: >- + {{ monitoring_databases.query_result | map(attribute='datname') | list }} + when: monitoring_prometheus_role.rowcount > 0 + +- name: Revoke PostgreSQL catalog access granted to the Prometheus user + become: true + become_user: postgres + community.postgresql.postgresql_query: + login_user: postgres + login_db: postgres + query: >- + REVOKE USAGE ON SCHEMA pg_catalog FROM prometheus; + REVOKE SELECT ON ALL TABLES IN SCHEMA pg_catalog FROM prometheus + when: monitoring_prometheus_role.rowcount > 0 + +- name: Remove the legacy Prometheus database user + become: true + become_user: postgres + community.postgresql.postgresql_user: + login_user: postgres + login_db: postgres + name: prometheus + state: absent + when: monitoring_prometheus_role.rowcount > 0 diff --git a/roles/monitoring/templates/etc/alloy/config.alloy b/roles/monitoring/templates/etc/alloy/config.alloy @@ -1,337 +0,0 @@ -// For a full configuration reference, see https://grafana.com/docs/alloy -logging { - level = "warn" -} - -// Push the logs to loki -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.write "grafana_loki" { - endpoint { - url = "https://loki.taler-systems.com/loki/api/v1/push" - tenant_id = "{{ target_host_name }}" - authorization { - type = "Bearer" - credentials = "{{ loki_access_token }}" - } - } -} - - -// Which log files to monitor: all regular log files with errors -local.file_match "local_files" { - path_targets = [ - { - "__path__" = "/var/log/*.log", - "job" = "system logs", - "hostname" = "{{ target_host_name }}", - }, - ] - sync_period = "5s" -} - - -// Connect local_files as source to filter_generic_logs -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.source.file "log_scrape" { - targets = local.file_match.local_files.targets - forward_to = [loki.process.filter_generic_logs.receiver] - tail_from_end = true -} - -// Which log files to monitor: all postgres logs -local.file_match "postgres_log_files" { - path_targets = [ - { - "__path__" = "/var/log/postgresql/*.log", - "job" = "postgres logs", - "hostname" = "{{ target_host_name }}", - }, - ] - sync_period = "5s" -} - - -// Connect postgres_log_files as source to filter_generic_logs -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.source.file "postgres_log_scrape" { - targets = local.file_match.postgres_log_files.targets - forward_to = [loki.process.filter_generic_logs.receiver] - tail_from_end = true -} - -// Which log files to monitor: all regular log files with errors -local.file_match "nginx_errors" { - path_targets = [ - { - "__path__" = "/var/log/nginx/*.err", - "job" = "nginx errors", - "hostname" = "{{ target_host_name }}", - }, - ] - sync_period = "5s" -} - -// Connect nginx_errors directly to loki -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.source.file "nginx_error_scrape" { - targets = local.file_match.nginx_errors.targets - forward_to = [loki.write.grafana_loki.receiver] - tail_from_end = true -} - - -// Which log files to monitor: nginx regular logs -local.file_match "http_logs" { - path_targets = [ - { - "__path__" = "/var/log/nginx/*.log", - "job" = "nginx logs", - "hostname" = "{{ target_host_name }}", - }, - ] - sync_period = "5s" -} - -// Connect http_files as source to filter_http -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.source.file "http_scrape" { - targets = local.file_match.http_logs.targets - forward_to = [loki.process.filter_http.receiver] - tail_from_end = true -} - -// Filter the HTTP logs -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.process "filter_http" { - - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#stageregex-block - stage.regex { - expression = "(?P<ip>\\S+) (?P<identd>\\S+) (?P<user>\\S+) \\[(?P<timestamp>[\\w:\\/]+\\s[+\\\\-]\\d{4})\\] \"(?P<action>\\S+)\\s?(?P<path>\\S+)\\s?(?P<protocol>\\S+)?\" (?P<status>\\d{3}|-) (?P<size>\\d+|-)\\s?\"?(?P<referrer>[^\\\"]*)\"?\\s?\"?(?P<useragent>[^\\\"]*)?\"?" - } - - // exported via http://localhost:12345/metrics to Prometheus - stage.metrics { - metric.histogram { - name = "http_status_codes" - prefix = "taler_requests_" - description = "HTTP status codes, reported from Nginx (all requests)" - source = "status" - max_idle_duration = "24h" - buckets = [100,199,200,201,202,203,299,300,399,400,401,402,403,404,405,406,407,408,409,410,411,418,419,420,450,451,452,499,500,599] - } - - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metriccounter-block - metric.counter { - name = "total_requests" - prefix = "taler_requests_" - description = "Total Requests" - match_all = true - action = "inc" - } - } - - stage.drop { - source = "http_logs" - expression = ".*GET.* 200 .*" - drop_counter_reason = "successful HTTP GETs" - } - forward_to = [loki.write.grafana_loki.receiver] -} - - -// Monitor the logs with the latency statistics -local.file_match "nginx_taler_performance_logs" { - path_targets = [ - { - "__path__" = "/var/log/nginx/*.tal", - "job" = "nginx/performance", - "hostname" = "{{ target_host_name }}", - }, - ] - sync_period = "5s" -} - - -// Connect nginx_taler_performance_logs as source to perf_logs -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.source.file "perf_scrape" { - targets = local.file_match.nginx_taler_performance_logs.targets - forward_to = [loki.process.perf_logs.receiver] - tail_from_end = true -} - - - -// Here we export the *.tal logs with the Nginx latency data. -// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/ -loki.process "perf_logs" { - - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#stageregex-block - stage.regex { - expression = "uri=/(?P<ep>[a-zA-Z]+)(?:/\\w+)?(?:/(?P<act>[a-zA-Z-]+))? s=(?P<status>\\d{3}).*urt=(?P<urt>\\d+\\.\\d+|-) rt=(?P<response_time>\\d+\\.\\d+) rl=(?P<request_length>\\d+) bs=(?P<bytes_sent>\\d+)" - } - - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#stagetemplate-block - stage.template { - source = "endpoint" - template = "{{ '{{' }} printf \"%s-%s\" .ep .act | trimSuffix \"-\" {{ '}}' }}" - } - - stage.template { - source = "upstream_response_time" - template = "{{ '{{' }} .urt | replace \"-\" \"0\" {{ '}}' }}" - } - - // exported via http://localhost:12345/metrics to Prometheus - stage.metrics { - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metriccounter-block - metric.gauge { - name = "response_time" - prefix = "taler_requests_" - description = "Time taken for Nginx to respond (non-GET requests)" - source = "response_time" - max_idle_duration = "24h" - action = "set" - } - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metrichistogram-block - // https://www.robustperception.io/how-does-a-prometheus-histogram-work - metric.histogram { - name = "request_length_hist" - prefix = "taler_requests_" - description = "Request Length reported from Nginx (non-GET requests)" - source = "request_length" - max_idle_duration = "24h" - buckets = [1,10,50,100,200,500,1000,2000,5000] - } - - metric.histogram { - name = "bytes_sent_hist" - prefix = "taler_requests_" - description = "Number of bytes sent, reported from Nginx (non-GET requests)" - source = "bytes_sent" - max_idle_duration = "24h" - buckets = [1,10,50,100,200,500,1000,2000,5000] - } - metric.histogram { - name = "response_time_hist" - prefix = "taler_requests_" - description = "Time taken for Nginx to respond (non-GET requests)" - source = "response_time" - max_idle_duration = "24h" - buckets = [0.001,0.0025,0.005,0.010,0.025,0.050,0.1,0.25,0.5,1,2,5] - } - metric.histogram { - name = "upstream_response_time_hist" - prefix = "taler_requests_" - description = "Time taken for the Exchange to respond to Nginx (non-GET requests)" - source = "upstream_response_time" - max_idle_duration = "24h" - buckets = [0.001,0.0025,0.005,0.010,0.025,0.050,0.1,0.25,0.5,1,2,5] - } - - } - // Finally, pass on to Loki - forward_to = [loki.write.grafana_loki.receiver] - -} - - -// Monitor journald logs -// Export journald logs to our generic filter -// but first pass to our generic filter to change labels -loki.source.journal "read" { - forward_to = [loki.process.filter_generic_logs.receiver] - relabel_rules = loki.relabel.journal.rules - max_age = "12h" - labels = {component = "loki.source.journal"} -} - - -// https://community.grafana.com/t/scrape-journald-log-with-alloy-docker-container/119896 -loki.relabel "journal" { - forward_to = [] - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "systemd_unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "systemd_hostname" - } - rule { - source_labels = ["__journal__transport"] - target_label = "systemd_transport" - } -} - - -// Generic filter for logs -// See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/ -loki.process "filter_generic_logs" { - // Determine log level: - // https://community.grafana.com/t/extract-log-level-via-regex-and-set-it-as-a-label/134938/5 - stage.regex { - expression = `(?P<level>(?i)\b(info|debug|error|warn|warning|trace|fatal)\b)` - } - - // https://grafana.com/docs/alloy/latest/tutorials/processing-logs/ - // Drop debug - stage.drop { - source = "level" - value = "debug" - drop_counter_reason = "boring debugging data" - } - // Drop info - stage.drop { - source = "level" - value = "info" - drop_counter_reason = "boring info logs" - } - // Drop trace - stage.drop { - source = "level" - value = "trace" - drop_counter_reason = "boring info logs" - } - - stage.metrics { - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metriccounter-block - - // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metrichistogram-block - // https://www.robustperception.io/how-does-a-prometheus-histogram-work - metric.counter { - name = "warn_log_level" - prefix = "system_logs_" - description = "Warnings in system logs" - source = "level" - value = "warn" - action = "inc" - } - metric.counter { - name = "warn_log_level" - prefix = "system_logs_" - description = "Warnings in system logs" - source = "level" - value = "warning" - action = "inc" - } - metric.counter { - name = "error_log_level" - prefix = "system_logs_" - description = "Errors in system logs" - source = "level" - value = "error" - action = "inc" - } - metric.counter { - name = "fatal_log_level" - prefix = "system_logs_" - description = "Fatal reports in system logs" - source = "level" - value = "fatal" - action = "inc" - } - } - - forward_to = [loki.write.grafana_loki.receiver] -} diff --git a/roles/monitoring/templates/etc/nginx/sites-available/monitoring-http.conf.j2 b/roles/monitoring/templates/etc/nginx/sites-available/monitoring-http.conf.j2 @@ -1,14 +0,0 @@ -server { - - listen 80; - listen [::]:80; - - server_name monitoring.{{ domain_name }}; - - error_log /var/log/nginx/monitoring.{{ domain_name }}-http.err; - access_log /var/log/nginx/monitoring.{{ domain_name }}-http.log; - - location / { - return 301 https://$host$request_uri; - } -} diff --git a/roles/monitoring/templates/etc/nginx/sites-available/monitoring-nginx.conf.j2 b/roles/monitoring/templates/etc/nginx/sites-available/monitoring-nginx.conf.j2 @@ -1,92 +0,0 @@ -server { - - include conf.d/listen.conf.inc; - - server_name monitoring.{{ domain_name }}; - - ssl_certificate /etc/letsencrypt/live/monitoring/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/monitoring/privkey.pem; - ssl_trusted_certificate /etc/letsencrypt/live/monitoring/chain.pem; - - error_log /var/log/nginx/monitoring.{{ domain_name }}.err; - access_log /var/log/nginx/monitoring.{{ domain_name }}.log; - - location /prometheus/ { - if ($http_authorization != "Bearer {{ prometheus_access_token }}") { - return 401; - } - rewrite ^/prometheus/(.*)$ /$1 break; - proxy_pass http://127.0.0.1:9090/; - } - - location /node/ { - # Put API behind simple access control. - if ($http_authorization != "Bearer {{ prometheus_access_token }}") { - return 401; - } - rewrite ^/node/(.*)$ /$1 break; - proxy_pass http://127.0.0.1:9100/; - } - - location /nginx/ { - # Put API behind simple access control. - if ($http_authorization != "Bearer {{ prometheus_access_token }}") { - return 401; - } - rewrite ^/nginx/(.*)$ /$1 break; - proxy_pass http://127.0.0.1:9113/; - } - - location /postgres/ { - # Put API behind simple access control. - if ($http_authorization != "Bearer {{ prometheus_access_token }}") { - return 401; - } - rewrite ^/postgres/(.*)$ /$1 break; - proxy_pass http://127.0.0.1:9187/; - } - -# location /systemd/ { - # Put API behind simple access control. -# if ($http_authorization != "Bearer {{ prometheus_access_token }}") { -# return 401; -# } -# rewrite ^/systemd/(.*)$ /$1 break; -# proxy_pass http://127.0.0.1:9116/; -# } - - # See /etc/default/alloy for the export - location /alloy/ { - # Put API behind simple access control. - if ($http_authorization != "Bearer {{ prometheus_access_token }}") { - return 401; - } - rewrite ^/alloy/(.*)$ /$1 break; - proxy_pass http://127.0.0.1:12345/; - } - - location /alertmanager/ { - if ($http_authorization != "Bearer {{ prometheus_access_token }}") { - return 401; - } - rewrite ^/alertmanager/(.*)$ /$1 break; - proxy_pass http://127.0.0.1:9093/; - } -} - - - -# Expose Nginx /metrics on loopback for the node exporter -# See /etc/default/prometheus-nginx-exporter for the use -server { - listen localhost; - root /dev/null; - server_name localhost; - - location /metrics { - stub_status on; - access_log off; - allow 127.0.0.1; - deny all; - } -} diff --git a/roles/monitoring/templates/etc/prometheus/alertmanager.yml b/roles/monitoring/templates/etc/prometheus/alertmanager.yml @@ -1,88 +0,0 @@ -# Sample configuration. -# See https://prometheus.io/docs/alerting/configuration/ for documentation. - -global: - # The smarthost and SMTP sender used for mail notifications. - smtp_smarthost: 'firefly.gnunet.org:25' - smtp_from: 'alertmanager@taler.net' - smtp_require_tls: false - #smtp_auth_username: 'alertmanager' - #smtp_auth_password: 'password' - -# The directory from which notification templates are read. -templates: -- '/etc/prometheus/alertmanager_templates/*.tmpl' - -# The root route on which each incoming alert enters. -route: - # The labels by which incoming alerts are grouped together. For example, - # multiple alerts coming in for cluster=A and alertname=LatencyHigh would - # be batched into a single group. - group_by: ['alertname', 'cluster', 'service'] - - # When a new group of alerts is created by an incoming alert, wait at - # least 'group_wait' to send the initial notification. - # This way ensures that you get multiple alerts for the same group that start - # firing shortly after another are batched together on the first - # notification. - group_wait: 30s - - # When the first notification was sent, wait 'group_interval' to send a batch - # of new alerts that started firing for that group. - group_interval: 5m - - # If an alert has successfully been sent, wait 'repeat_interval' to - # resend them. - repeat_interval: 12h - - # A default receiver - receiver: taler-warning-mails - - # All the above attributes are inherited by all child routes and can - # overwritten on each. - - # The child route trees. - routes: - - matchers: - - severity = critical - receiver: taler-admin-mails - - -# Inhibition rules allow to mute a set of alerts given that another alert is -# firing. -# We use this to mute any warning-level notifications if the same alert is -# already critical. -inhibit_rules: -- source_match: - severity: 'critical' - target_match: - severity: 'warning' - # Apply inhibition if the alertname is the same. - equal: ['alertname', 'cluster', 'service'] - - -receivers: -- name: 'taler-warning-mails' - email_configs: - - to: 'warnings@taler.net' -- name: 'taler-admin-mails' - email_configs: - - to: 'sysadmin@taler.net' - # - #- name: 'team-X-pager' - # email_configs: - # - to: 'team-X+alerts-critical@example.org' - # pagerduty_configs: - # - service_key: <team-X-key> - # - #- name: 'team-Y-mails' - # email_configs: - # - to: 'team-Y+alerts@example.org' - # - #- name: 'team-Y-pager' - # pagerduty_configs: - # - service_key: <team-Y-key> - # - #- name: 'team-DB-pager' - # pagerduty_configs: - # - service_key: <team-DB-key> diff --git a/roles/webserver/files/etc/nginx/conf.d/log-format-apm.conf b/roles/webserver/files/etc/nginx/conf.d/log-format-apm.conf @@ -8,7 +8,7 @@ log_format apm '"$time_local" client=$remote_addr ' 'upstream_status=$upstream_status ' 'request_time=$request_time'; -# This format is expected by alloy (see monitoring role) +# Compact format for GNU Taler request performance logs. log_format taler 'm=$request_method uri=$uri s=$status ' 'uct=$upstream_connect_time ' 'urt=$upstream_response_time '