commit 8cb4f255525f7ad45b72c6578715dae75e4c78b0
parent 9d13beb291ec28f6905362c0b7a73118fb93dec5
Author: Florian Dold <dold@taler.net>
Date: Tue, 8 Sep 2026 11:38:18 +0200
monitoring: send journal logs with Fluent Bit over mTLS
Consume JSON Lines monitoring bundles and replace rsyslog and RELP
with a pinned Fluent Bit collector. Keep persistent journal cursors and
filesystem buffers, exclude collector diagnostics, and expose collector
metrics through the existing authenticated proxy.
Update the Rusty public bundle while retaining its certificate pairs
and document delivery limits and the coordinated receiver cutover.
Diffstat:
13 files changed, 373 insertions(+), 96 deletions(-)
diff --git a/README b/README
@@ -183,29 +183,51 @@ $ ./deploy "$HOST"
The `monitoring` role installs Debian's node_exporter with a loopback-only
listener and systemd metrics, a dedicated `node-exporter-proxy` nginx
-service requiring Sentol's mTLS client identity, and durable rsyslog
-RELP/TLS forwarding. All connection settings and certificate identities
+service requiring Sentol's mTLS client identity, and buffered Fluent Bit
+JSON Lines forwarding over HTTPS/mTLS. All connection settings and certificate identities
come from the bundle. TLS material lives in `/etc/taler-monitoring/tls`;
private keys and the enclosing directory are accessible only to root.
-The role expects Debian's root-running rsyslog service to read those keys.
+The root-running Fluent Bit service reads the journal and client key. The role
+requires Debian 13 amd64/arm64 and installs pinned Fluent Bit 5.1.2 from its signed
+upstream APT repository. The bundle declares `logs.protocol: jsonline` and an HTTPS
+`logs.url` ending in `/jsonline`; old RELP bundles fail preflight.
Allow Sentol to reach the exported metrics proxy address and port
-(normally TCP 9100), and allow the node to reach the exported RELP server
+(normally TCP 9100), and allow the node to reach the exported HTTPS log server
(normally TCP 2514). Firewall and routing configuration is external to
this role. The dedicated proxy leaves exchange webserver sites intact.
Verify the local services with:
```
-$ systemctl is-active prometheus-node-exporter node-exporter-proxy rsyslog
+$ systemctl is-active prometheus-node-exporter node-exporter-proxy fluent-bit
$ curl --fail http://127.0.0.1:9101/metrics >/dev/null
$ logger --tag external-monitoring-enrollment 'Exchange monitoring test'
```
Use the exported backend address if it differs. Follow the external-client
runbook to perform an authenticated scrape from Sentol, confirm anonymous
-scrapes fail, and find the test message in the remote log archive and
-VictoriaLogs.
+scrapes fail, and find the test message in VictoriaLogs using the exported identity as
+`authenticated_host`. Fluent Bit metrics are available through the same mTLS proxy
+at `/fluent-bit/metrics`.
+
+Fluent Bit reads only new journal entries on first activation and resumes its saved
+cursor on later starts. Journald uses persistent storage without changing host-specific
+retention. `/var/lib/fluent-bit` retains the cursor and filesystem buffer, with a
+1 GiB output queue by default (`monitoring_fluent_bit_queue_limit`). Retryable failures
+are retried without a count limit; a full queue discards its oldest chunks. Permanent
+HTTP errors can discard records, and VictoriaLogs can accept a batch while rejecting
+individual malformed lines. Watch both collector drop counters and receiver errors.
+Lost responses can cause duplicates; this is not exactly-once or unconditional
+lossless delivery. Fluent Bit's own logs remain local to prevent feedback loops.
+
+Coordinate migration with Sentol's administrator: drain old RELP queues first, deploy
+the HTTPS receiver, then deploy this role with the new bundle. TCP 2514 is reused for
+HTTPS. First activation skips historical journal entries, so a cutover gap may be lost
+centrally. The role stops/removes rsyslog and rsyslog-relp without purging historical
+files, removes its forwarding configuration, and disables journal forwarding to syslog.
+Do not delete old spool files until any recovery is complete. Preserve the new cursor
+and buffer on rollback; restore old server and client configurations together.
For renewal, retrieve both refreshed exports after `make deploy-sentol`,
replace the local files, encrypt the complete new secrets file before
@@ -216,8 +238,7 @@ and reloads or restarts the affected services when their material changes.
`enable_monitoring` defaults to false. False skips management and leaves
installed monitoring running. To offboard, follow the Sentol runbook and,
on the exchange node, stop and disable `node-exporter-proxy` and
-`prometheus-node-exporter`, remove `/etc/rsyslog.d/60-sentol-forward.conf`,
-validate with `rsyslogd -N1` and restart rsyslog, then remove the proxy unit
+`prometheus-node-exporter` and `fluent-bit`, then remove the proxy unit
and `/etc/taler-monitoring` (including its private keys) and run
`systemctl daemon-reload`. Set the flag false and remove both inventory
exports. This does not revoke previously issued certificates; complete
@@ -485,4 +506,4 @@ devtesting revocation, preservation of unrelated configuration, failed deploymen
and recovery. All fixtures run in the disposable container, never an inventory
production host. Monitoring regressions also cover Vault-encrypted onboarding
bundles, preflight failures before application shutdown, authenticated metrics,
-RELP delivery, certificate renewal, idempotence, and check mode.
+JSON Lines delivery, certificate renewal, idempotence, and check mode.
diff --git a/contrib/tests/test_monitoring.py b/contrib/tests/test_monitoring.py
@@ -13,7 +13,7 @@ import time
REPO = Path(__file__).resolve().parents[2]
IDENTITY = 'enrolled-exchange.example'
-SERVICES = ['prometheus-node-exporter', 'node-exporter-proxy', 'rsyslog']
+SERVICES = ['prometheus-node-exporter', 'node-exporter-proxy', 'fluent-bit']
def run(*args, **kwargs):
@@ -73,8 +73,7 @@ def main():
'monitoring_ca_certificate': (pki / 'ca.cert').read_text(),
'client_certificate': (pki / 'client.cert').read_text(),
'server_certificate': (pki / 'server.cert').read_text(),
- 'relp': {'server_address': '127.0.0.1', 'server_port': 12514,
- 'server_name': 'receiver.example'},
+ 'logs': {'protocol': 'jsonline', 'url': 'https://receiver.example:12514/jsonline'},
'node_exporter': {'backend_listen_address': '127.0.0.1:9101',
'proxy_bind_address': '127.0.0.1', 'proxy_port': 9100,
'prometheus_client_identity': 'sentol'}}}
@@ -145,7 +144,7 @@ def main():
invoke(REPO / 'playbooks/setup.yml', failure=True)
vault_password.write_text('disposable-monitoring-test-password\n')
- for change in ['backend', 'wildcard-conflict', 'identity', 'purpose', 'key', 'missing', 'injection']:
+ for change in ['backend', 'wildcard-conflict', 'identity', 'purpose', 'key', 'missing', 'protocol', 'url', 'port', 'injection']:
bad_public, bad_secrets = copy.deepcopy(public), copy.deepcopy(secrets)
if change == 'backend':
bad_public['monitoring_client']['node_exporter']['backend_listen_address'] = '0.0.0.0:9101'
@@ -159,7 +158,14 @@ def main():
elif change == 'key':
bad_secrets['monitoring_client_secrets']['client_private_key'] = secrets['monitoring_client_secrets']['server_private_key']
elif change == 'missing':
- del bad_public['monitoring_client']['relp']
+ del bad_public['monitoring_client']['logs']
+ bad_public['monitoring_client']['relp'] = {}
+ elif change == 'protocol':
+ bad_public['monitoring_client']['logs']['protocol'] = 'otlp'
+ elif change == 'url':
+ bad_public['monitoring_client']['logs']['url'] = 'https://receiver.example/jsonline?debug=1'
+ elif change == 'port':
+ bad_public['monitoring_client']['logs']['url'] = 'https://receiver.example:65536/jsonline'
else:
bad_public['monitoring_client']['node_exporter']['prometheus_client_identity'] = 'sentol"; }'
install_bundle(bad_public, bad_secrets)
@@ -174,6 +180,7 @@ def main():
# move it to loopback before the dedicated TLS proxy takes that port.
container('apt-get', 'install', '-y', '--no-install-recommends', 'prometheus-node-exporter')
container('systemctl', 'start', 'prometheus-node-exporter')
+ container('sh', '-c', 'echo "127.0.0.1 receiver.example" >> /etc/hosts')
invoke(role_play)
remote = '/tmp/taler-monitoring-test'
@@ -225,19 +232,54 @@ def main():
assert container('stat', '-c', '%a:%U:%G', f'/etc/taler-monitoring/{path}').stdout.strip() == f'{expected}:root:root'
assert container('su', '-s', '/bin/sh', 'nobody', '-c',
'test -r /etc/taler-monitoring/tls/client.key.pem', check=False).returncode != 0
- receiver = f'''module(load="imrelp" tls.tlsLib="openssl")
-ruleset(name="received") {{ action(type="omfile" file="{remote}/received.log") }}
-input(type="imrelp" address="127.0.0.1" port="12514" ruleset="received"
- tls="on" tls.caCert="{remote}/pki/ca.cert"
- tls.myCert="{remote}/pki/receiver.cert" tls.myPrivKey="{remote}/pki/receiver.key"
- tls.authMode="name" tls.permittedPeer=["{IDENTITY}"])
+ # A real TLS HTTP receiver lets us exercise the output without another
+ # deployment repository. The tsys-infra suite also runs this role against
+ # its production nginx and VictoriaLogs configurations.
+ receiver = r'''import http.server
+import json
+import os
+from pathlib import Path
+import ssl
+
+root = Path(__file__).parent
+class Handler(http.server.BaseHTTPRequestHandler):
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers['Content-Length']))
+ subject = self.connection.getpeercert()['subject']
+ if subject != ((('commonName', 'enrolled-exchange.example'),),):
+ self.send_error(403)
+ return
+ if self.path != '/jsonline':
+ self.send_error(404)
+ return
+ status = root / 'response-status'
+ if status.exists():
+ self.send_error(int(status.read_text()))
+ return
+ rows = [json.loads(line) for line in body.splitlines()]
+ with (root / 'received.log').open('a') as output:
+ for row in rows:
+ output.write(json.dumps(row) + '\n')
+ output.flush()
+ os.fsync(output.fileno())
+ self.send_response(200)
+ self.send_header('Content-Length', '0')
+ self.end_headers()
+ def log_message(self, *args):
+ pass
+server = http.server.HTTPServer(('127.0.0.1', 12514), Handler)
+context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+context.load_cert_chain(root / 'pki/receiver.cert', root / 'pki/receiver.key')
+context.load_verify_locations(root / 'pki/ca.cert')
+context.verify_mode = ssl.CERT_REQUIRED
+server.socket = context.wrap_socket(server.socket, server_side=True)
+server.serve_forever()
'''
- container('tee', f'{remote}/receiver.conf', input=receiver)
- container('rsyslogd', '-N1', '-f', f'{remote}/receiver.conf')
+ container('tee', f'{remote}/receiver.py', input=receiver)
def receiver_start():
container('systemd-run', '--unit=monitoring-test-receiver', '--collect',
- 'rsyslogd', '-n', '-f', f'{remote}/receiver.conf', '-i', f'{remote}/receiver.pid')
+ 'python3', f'{remote}/receiver.py')
def log_delivery(message, emit=True):
if emit:
@@ -247,19 +289,36 @@ input(type="imrelp" address="127.0.0.1" port="12514" ruleset="received"
if message in received:
return
time.sleep(1)
- raise AssertionError('RELP test message was not delivered')
+ raise AssertionError('JSON Lines test message was not delivered')
receiver_start()
try:
log_delivery('initial-monitoring-enrollment')
- print('PASS: mTLS metrics, client allowlist, systemd metrics, permissions and RELP delivery', flush=True)
+ print('PASS: mTLS metrics, client allowlist, systemd metrics, permissions and JSON Lines delivery', flush=True)
container('systemctl', 'stop', 'monitoring-test-receiver')
container('logger', '--tag', 'monitoring-regression', 'queued-during-receiver-outage')
time.sleep(2)
- container('systemctl', 'restart', 'rsyslog')
+ container('systemctl', 'restart', 'fluent-bit')
receiver_start()
log_delivery('queued-during-receiver-outage', emit=False)
- print('PASS: RELP queue survives receiver outage and rsyslog restart', flush=True)
+ print('PASS: Fluent Bit queue survives receiver outage and collector restart', flush=True)
+ # Permanent HTTP errors discard the chunk even with unlimited
+ # retries; verify this is exposed by the output's drop counter.
+ def drops():
+ metrics = container('curl', '-fsS', 'http://127.0.0.1:2020/api/v1/metrics/prometheus').stdout
+ return sum(float(line.split()[1]) for line in metrics.splitlines()
+ if line.startswith('fluentbit_output_dropped_records_total{'))
+ container('tee', f'{remote}/response-status', input='400')
+ prior_drops = drops()
+ container('logger', '-t', 'monitoring-regression', 'permanently-rejected-jsonline')
+ for _ in range(20):
+ if drops() > prior_drops:
+ break
+ time.sleep(1)
+ assert drops() > prior_drops, 'HTTP 400 did not expose dropped records'
+ container('rm', f'{remote}/response-status')
+ log_delivery('recovered-after-permanent-error')
+ print('PASS: permanent HTTP errors expose dropped records and later requests recover', flush=True)
before = container('systemctl', 'show', '-p', 'MainPID', *SERVICES).stdout
output = invoke(role_play)
assert re.search(r'changed=0\s', output), 'Second monitoring deployment was not idempotent'
diff --git a/contrib/tests/test_monitoring_listeners.py b/contrib/tests/test_monitoring_listeners.py
@@ -59,6 +59,12 @@ class MonitoringListenersTest(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'backend must use'):
VALIDATOR.validate_listeners(self.node('*', backend))
+ def test_fluent_bit_listener_conflicts(self):
+ for node in [self.node('*', port=2020), self.node(port=2020),
+ self.node(backend='127.0.0.1:2020')]:
+ with self.assertRaisesRegex(ValueError, 'conflicts with Fluent Bit'):
+ VALIDATOR.validate_listeners(node)
+
def test_invalid_bind_addresses_remain_rejected(self):
for bind in ['0.0.0.0', '::', '[::]', '224.0.0.1', 'ff02::1',
'rusty.taler-ops.ch', '*; return 200;', 'fe80::1%eth0', 42]:
diff --git a/inventories/host_vars/rusty/monitoring-client.yml b/inventories/host_vars/rusty/monitoring-client.yml
@@ -45,10 +45,9 @@ monitoring_client:
eAIwPMv0cswAz+fGmqSZGabdyvDdvhiPdkR9E6FENTA+R191e1fyLibsShl10h4K
a74M
-----END CERTIFICATE-----
- relp:
- server_address: "46.225.94.164"
- server_port: 2514
- server_name: "monitoring.taler.net"
+ logs:
+ protocol: jsonline
+ url: "https://monitoring.taler.net:2514/jsonline"
node_exporter:
backend_listen_address: "127.0.0.1:9101"
proxy_bind_address: "*"
diff --git a/roles/monitoring/defaults/main.yml b/roles/monitoring/defaults/main.yml
@@ -0,0 +1,8 @@
+---
+monitoring_fluent_bit_version: "5.1.2"
+monitoring_fluent_bit_queue_limit: 1G
+monitoring_fluent_bit_tls_directory: /etc/taler-monitoring/tls
+monitoring_fluent_bit_excluded_units:
+ - fluent-bit.service
+monitoring_fluent_bit_url: "{{ monitoring_public_bundle.monitoring_client.logs.url }}"
+monitoring_fluent_bit_receiver_units: []
diff --git a/roles/monitoring/files/validate-bundle.py b/roles/monitoring/files/validate-bundle.py
@@ -7,6 +7,7 @@ from pathlib import Path
import subprocess
import sys
import tempfile
+from urllib.parse import urlsplit
def require(condition, message):
@@ -38,6 +39,7 @@ def validate_listeners(node):
match = re.fullmatch(r'(127\.0\.0\.1|\[::1\]):([0-9]+)', backend)
require(match and 0 < int(match[2]) < 65536,
'The node_exporter backend must use 127.0.0.1 or [::1] and a valid port.')
+ require(int(match[2]) != 2020, 'The node_exporter backend conflicts with Fluent Bit metrics.')
bind_value = node.get('proxy_bind_address', '127.0.0.1')
if bind_value == '*':
overlaps_backend = True
@@ -46,6 +48,8 @@ def validate_listeners(node):
require(not bind.is_unspecified and not bind.is_multicast,
'The monitoring proxy must bind a specific unicast address or use "*" for all interfaces.')
overlaps_backend = bind.is_loopback
+ require(not (overlaps_backend and node['proxy_port'] == 2020),
+ 'The monitoring proxy conflicts with Fluent Bit metrics.')
require(not (overlaps_backend and node['proxy_port'] == int(match[2])),
'Monitoring proxy and backend listeners must not conflict.')
@@ -60,14 +64,20 @@ def validate(bundle):
public = bundle['public']['monitoring_client']
secret = bundle['secrets']['monitoring_client_secrets']
node = public['node_exporter']
- relp = public['relp']
+ logs = public['logs']
require(identity(public['identity']) and identity(node['prometheus_client_identity']),
'Invalid monitoring certificate identity.')
- require(identity(relp['server_name']), 'Invalid RELP server identity.')
- target = relp['server_address']
- require(isinstance(target, str) and re.fullmatch(r'[A-Za-z0-9_.:-]+', target),
- 'Invalid RELP server address.')
- require(port(relp['server_port']) and port(node['proxy_port']), 'Invalid monitoring port.')
+ require(logs.get('protocol') == 'jsonline', 'Unsupported monitoring log protocol; export a JSON Lines bundle.')
+ url = logs.get('url')
+ require(isinstance(url, str) and re.fullmatch(
+ r'https://[A-Za-z0-9][A-Za-z0-9.-]*(?::[0-9]{1,5})?/jsonline', url),
+ 'Monitoring logs require an HTTPS URL ending in /jsonline without credentials, query or fragment.')
+ try:
+ target_port = urlsplit(url).port
+ except ValueError:
+ raise ValueError('Invalid monitoring log port.') from None
+ require(port(target_port if target_port is not None else 443) and port(node['proxy_port']),
+ 'Invalid monitoring port.')
validate_listeners(node)
# TemporaryDirectory is private (0700); private files are created as 0600.
diff --git a/roles/monitoring/handlers/main.yml b/roles/monitoring/handlers/main.yml
@@ -28,15 +28,21 @@
listen: Restart monitoring proxy
when: not ansible_check_mode
-- name: Validate monitoring rsyslog before restart
- ansible.builtin.command: /usr/sbin/rsyslogd -N1
- changed_when: false
- listen: Restart monitoring rsyslog
+- name: Restart monitoring journal
+ ansible.builtin.systemd_service:
+ name: systemd-journald
+ state: restarted
when: not ansible_check_mode
+ notify: Flush monitoring journal
-- name: Restart validated monitoring rsyslog
+- name: Flush monitoring journal
+ ansible.builtin.command: journalctl --flush
+ changed_when: true
+ when: not ansible_check_mode
+
+- name: Restart monitoring Fluent Bit
ansible.builtin.systemd_service:
- name: rsyslog
+ name: fluent-bit
+ daemon_reload: true
state: restarted
- listen: Restart monitoring rsyslog
when: not ansible_check_mode
diff --git a/roles/monitoring/tasks/fluent-bit.yml b/roles/monitoring/tasks/fluent-bit.yml
@@ -0,0 +1,153 @@
+---
+- name: Validate Fluent Bit platform and configuration
+ ansible.builtin.assert:
+ that:
+ - ansible_facts['distribution'] == 'Debian'
+ - ansible_facts['distribution_major_version'] == '13'
+ - ansible_facts['architecture'] in ['x86_64', 'aarch64']
+ - monitoring_fluent_bit_version is match('^[0-9]+\.[0-9]+\.[0-9]+$')
+ - monitoring_fluent_bit_queue_limit is match('^[1-9][0-9]*[KMG]$')
+ - monitoring_fluent_bit_url is match('^https://[A-Za-z0-9][A-Za-z0-9.-]*(?::[0-9]{1,5})?/jsonline$')
+ - >-
+ (monitoring_fluent_bit_url | urlsplit('port')) is none
+ or (monitoring_fluent_bit_url | urlsplit('port')) | int > 0
+ - ((monitoring_fluent_bit_url | urlsplit('port')) or 443) | int < 65536
+ - "'fluent-bit.service' in monitoring_fluent_bit_excluded_units"
+ - >-
+ (monitoring_fluent_bit_excluded_units + monitoring_fluent_bit_receiver_units)
+ | select('match', '^[A-Za-z0-9_.@-]+\.service$') | list | length
+ == (monitoring_fluent_bit_excluded_units + monitoring_fluent_bit_receiver_units) | length
+ fail_msg: Fluent Bit requires Debian 13 amd64/arm64 and valid HTTPS, buffer and exclusion settings.
+
+- name: Discover installed packages for check mode
+ ansible.builtin.package_facts:
+ manager: auto
+
+- name: Install Fluent Bit repository prerequisites
+ ansible.builtin.apt:
+ name: ca-certificates
+ state: present
+ update_cache: true
+ cache_valid_time: 3600
+ policy_rc_d: 101
+
+- name: Install the Fluent Bit repository signing key
+ ansible.builtin.get_url:
+ url: https://packages.fluentbit.io/fluentbit.key
+ dest: /usr/share/keyrings/fluentbit.asc
+ checksum: sha256:df248e2d7103ca62cb683c20a077198d0fb0a7f79dbf53a604af0317de3b4711
+ owner: root
+ group: root
+ mode: "0644"
+
+- name: Configure the signed Fluent Bit repository
+ ansible.builtin.copy:
+ content: >-
+ deb [signed-by=/usr/share/keyrings/fluentbit.asc]
+ https://packages.fluentbit.io/debian/trixie trixie main
+ dest: /etc/apt/sources.list.d/fluent-bit.list
+ owner: root
+ group: root
+ mode: "0644"
+ register: monitoring_fluent_bit_repository
+
+- name: Pin Fluent Bit to the tested release
+ ansible.builtin.copy:
+ content: |
+ Package: fluent-bit
+ Pin: version {{ monitoring_fluent_bit_version }}
+ Pin-Priority: 1001
+ dest: /etc/apt/preferences.d/taler-fluent-bit
+ owner: root
+ group: root
+ mode: "0644"
+
+- name: Install Fluent Bit without starting the package configuration
+ ansible.builtin.apt:
+ name: "fluent-bit={{ monitoring_fluent_bit_version }}"
+ state: present
+ allow_downgrade: true
+ update_cache: "{{ monitoring_fluent_bit_repository.changed }}"
+ install_recommends: false
+ policy_rc_d: 101
+ notify: Restart monitoring Fluent Bit
+ when: not ansible_check_mode or 'fluent-bit' in ansible_facts.packages
+
+- name: Create private Fluent Bit state and systemd configuration directories
+ ansible.builtin.file:
+ path: "{{ item.path }}"
+ state: directory
+ owner: root
+ group: root
+ mode: "{{ item.mode }}"
+ loop:
+ - { path: /var/lib/fluent-bit, mode: "0700" }
+ - { path: /var/lib/fluent-bit/buffer, mode: "0700" }
+ - { path: /etc/systemd/journald.conf.d, mode: "0755" }
+ - { path: /etc/systemd/system/fluent-bit.service.d, mode: "0755" }
+
+- name: Configure persistent journald storage without syslog forwarding
+ ansible.builtin.copy:
+ content: |
+ # Managed by Ansible. Keep host-specific journal retention settings.
+ [Journal]
+ Storage=persistent
+ ForwardToSyslog=no
+ dest: /etc/systemd/journald.conf.d/60-taler-monitoring.conf
+ owner: root
+ group: root
+ mode: "0644"
+ notify: Restart monitoring journal
+
+- name: Configure Fluent Bit service recovery and private state
+ ansible.builtin.copy:
+ content: |
+ [Service]
+ Restart=on-failure
+ RestartSec=5s
+ UMask=0077
+ TimeoutStopSec=45s
+ dest: /etc/systemd/system/fluent-bit.service.d/monitoring.conf
+ owner: root
+ group: root
+ mode: "0644"
+ notify: Restart monitoring Fluent Bit
+
+- name: Configure buffered JSON Lines forwarding
+ ansible.builtin.template:
+ src: fluent-bit.conf.j2
+ dest: /etc/fluent-bit/fluent-bit.conf
+ owner: root
+ group: root
+ mode: "0600"
+ validate: /opt/fluent-bit/bin/fluent-bit --dry-run -c %s
+ notify: Restart monitoring Fluent Bit
+
+- name: Discover legacy logging services
+ ansible.builtin.service_facts:
+
+- name: Stop and disable rsyslog before removing its packages
+ ansible.builtin.systemd_service:
+ name: rsyslog
+ state: stopped
+ enabled: false
+ when: "ansible_facts.services.get('rsyslog.service', {}).get('status', 'not-found') != 'not-found'"
+
+- name: Remove rsyslog and RELP packages while preserving historical files
+ ansible.builtin.apt:
+ name:
+ - rsyslog-relp
+ - rsyslog
+ state: absent
+ purge: false
+ autoremove: false
+
+- name: Remove obsolete managed RELP forwarding configuration and TLS copies
+ ansible.builtin.file:
+ path: "{{ item }}"
+ state: absent
+ loop:
+ - /etc/rsyslog.d/60-sentol-forward.conf
+ - /etc/rsyslog.d/tls/relp-ca.cert.pem
+ - /etc/rsyslog.d/tls/relp-client.cert.pem
+ - /etc/rsyslog.d/tls/relp-client.key.pem
diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml
@@ -11,8 +11,6 @@
name:
- nginx
- prometheus-node-exporter
- - rsyslog
- - rsyslog-relp
state: present
update_cache: true
cache_valid_time: 3600
@@ -57,7 +55,7 @@
diff: false
notify:
- Reload monitoring proxy
- - Restart monitoring rsyslog
+ - Restart monitoring Fluent Bit
- name: Create the node_exporter textfile directory
ansible.builtin.file:
@@ -97,28 +95,8 @@
mode: '0644'
notify: Restart monitoring proxy
-- name: Create the persistent RELP queue directory
- ansible.builtin.file:
- path: /var/spool/rsyslog
- state: directory
- owner: root
- group: root
- mode: '0700'
-
-- name: Configure authenticated RELP forwarding
- ansible.builtin.template:
- src: rsyslog-forward.conf.j2
- dest: /etc/rsyslog.d/60-sentol-forward.conf
- owner: root
- group: root
- mode: '0644'
- validate: /usr/sbin/rsyslogd -N1 -f %s
- notify: Restart monitoring rsyslog
-
-- name: Validate the complete rsyslog configuration before activation
- ansible.builtin.command: /usr/sbin/rsyslogd -N1
- changed_when: false
- when: not ansible_check_mode
+- name: Install and configure Fluent Bit
+ ansible.builtin.import_tasks: fluent-bit.yml
- name: Load the configured monitoring units
ansible.builtin.systemd_service:
@@ -138,5 +116,5 @@
loop:
- prometheus-node-exporter
- node-exporter-proxy
- - rsyslog
+ - fluent-bit
when: not ansible_check_mode or (item ~ '.service') in ansible_facts['services']
diff --git a/roles/monitoring/tasks/preflight.yml b/roles/monitoring/tasks/preflight.yml
@@ -62,7 +62,10 @@
Cannot load or validate the monitoring exports. Check the exported YAML,
Vault password, certificate identities, purposes, validity and key pairs.
-- name: Require a Debian-family monitoring client
+- name: Require a supported monitoring client platform
ansible.builtin.assert:
- that: ansible_facts['os_family'] == 'Debian'
+ that:
+ - ansible_facts['distribution'] == 'Debian'
+ - ansible_facts['distribution_major_version'] == '13'
+ - ansible_facts['architecture'] in ['x86_64', 'aarch64']
quiet: true
diff --git a/roles/monitoring/templates/fluent-bit.conf.j2 b/roles/monitoring/templates/fluent-bit.conf.j2
@@ -0,0 +1,51 @@
+# Managed by Ansible. Journal records are buffered before HTTPS delivery.
+[SERVICE]
+ Flush 1
+ Grace 30
+ Log_Level warn
+ HTTP_Server On
+ HTTP_Listen 127.0.0.1
+ HTTP_Port 2020
+ storage.path /var/lib/fluent-bit/buffer
+ storage.sync full
+ storage.checksum On
+ storage.max_chunks_up 32
+ storage.backlog.mem_limit 16M
+ storage.metrics On
+
+[INPUT]
+ Name systemd
+ Tag journal
+ DB /var/lib/fluent-bit/journal.db
+ DB.Sync Full
+ Read_From_Tail On
+ Strip_Underscores Off
+ Lowercase Off
+ storage.type filesystem
+
+# Exclude shipper and receiver diagnostics before buffering output records.
+[FILTER]
+ Name grep
+ Match journal
+ Exclude _SYSTEMD_UNIT ^({{ (monitoring_fluent_bit_excluded_units + monitoring_fluent_bit_receiver_units) | unique | map('regex_escape') | join('|') }})$
+
+[OUTPUT]
+ Name http
+ Alias victoria_logs
+ Match journal
+ Host {{ monitoring_fluent_bit_url | urlsplit('hostname') }}
+ Port {{ (monitoring_fluent_bit_url | urlsplit('port')) or 443 }}
+ URI /jsonline
+ Format json_lines
+ Json_Date_Key date
+ Json_Date_Format iso8601
+ tls On
+ tls.verify On
+ tls.verify_hostname On
+ tls.vhost {{ monitoring_fluent_bit_url | urlsplit('hostname') }}
+ tls.ca_file {{ monitoring_fluent_bit_tls_directory }}/ca.cert.pem
+ tls.crt_file {{ monitoring_fluent_bit_tls_directory }}/client.cert.pem
+ tls.key_file {{ monitoring_fluent_bit_tls_directory }}/client.key.pem
+ Retry_Limit False
+ storage.total_limit_size {{ monitoring_fluent_bit_queue_limit }}
+ Log_Response_Payload Off
diff --git a/roles/monitoring/templates/nginx.conf.j2 b/roles/monitoring/templates/nginx.conf.j2
@@ -45,6 +45,12 @@ http {
proxy_set_header Connection "";
proxy_buffering off;
}
+ location = /fluent-bit/metrics {
+ proxy_pass http://127.0.0.1:2020/api/v1/metrics/prometheus;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ proxy_buffering off;
+ }
location / {
return 404;
}
diff --git a/roles/monitoring/templates/rsyslog-forward.conf.j2 b/roles/monitoring/templates/rsyslog-forward.conf.j2
@@ -1,23 +0,0 @@
-# Managed by Ansible. Forward local messages with durable RELP/TLS.
-{% set relp = monitoring_public_bundle.monitoring_client.relp %}
-module(load="omrelp" tls.tlsLib="openssl")
-
-action(
- name="sentol_relp"
- type="omrelp"
- target={{ relp.server_address | to_json }}
- port="{{ relp.server_port }}"
- template="RSYSLOG_SyslogProtocol23Format"
- tls="on"
- tls.caCert="/etc/taler-monitoring/tls/ca.cert.pem"
- tls.myCert="/etc/taler-monitoring/tls/client.cert.pem"
- tls.myPrivKey="/etc/taler-monitoring/tls/client.key.pem"
- tls.authMode="name"
- tls.permittedPeer={{ relp.server_name | to_json }}
- action.resumeRetryCount="-1"
- queue.type="LinkedList"
- queue.filename="sentol_relp"
- queue.spoolDirectory="/var/spool/rsyslog"
- queue.maxDiskSpace="1g"
- queue.saveOnShutdown="on"
-)
