test_monitoring.py (18747B)
1 #!/usr/bin/env python3 2 """Exercise external monitoring in the disposable deployment test container.""" 3 import argparse 4 import copy 5 import json 6 import os 7 from pathlib import Path 8 import re 9 import shutil 10 import subprocess 11 import tempfile 12 import time 13 14 REPO = Path(__file__).resolve().parents[2] 15 IDENTITY = 'enrolled-exchange.example' 16 SERVICES = ['prometheus-node-exporter', 'node-exporter-proxy', 'fluent-bit'] 17 18 19 def run(*args, **kwargs): 20 return subprocess.run(list(map(str, args)), capture_output=True, text=True, 21 check=kwargs.pop('check', True), **kwargs) 22 23 24 def main(): 25 parser = argparse.ArgumentParser() 26 parser.add_argument('container') 27 parser.add_argument('private_key') 28 args = parser.parse_args() 29 30 def container(*cmd, **kwargs): 31 return run('podman', 'exec', '-i', args.container, *cmd, **kwargs) 32 33 with tempfile.TemporaryDirectory(prefix='taler-monitoring-test-') as directory: 34 work = Path(directory) 35 inventory = work / 'inventory' 36 host_vars = inventory / 'host_vars/podman-localhost' 37 shutil.copytree(REPO / 'inventories/host_vars/podman-localhost', host_vars) 38 shutil.copytree(REPO / 'inventories/group_vars', inventory / 'group_vars') 39 shutil.copyfile(REPO / 'inventories/default', inventory / 'hosts') 40 vault_password = work / 'vault-password' 41 vault_password.write_text('disposable-monitoring-test-password\n') 42 vault_password.chmod(0o600) 43 pki = work / 'pki' 44 pki.mkdir(mode=0o700) 45 46 def openssl(*cmd): 47 return run('openssl', *cmd) 48 49 openssl('req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', 50 '-nodes', '-keyout', pki / 'ca.key', '-out', pki / 'ca.cert', '-days', '2', 51 '-subj', '/CN=Disposable monitoring CA', '-addext', 'basicConstraints=critical,CA:TRUE') 52 53 def leaf(name, common_name, purpose): 54 openssl('req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', 55 '-nodes', '-keyout', pki / f'{name}.key', '-out', pki / f'{name}.csr', 56 '-subj', f'/CN={common_name}') 57 ext = pki / f'{name}.ext' 58 ext.write_text(f'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n' 59 f'extendedKeyUsage={purpose}\nsubjectAltName=DNS:{common_name}\n') 60 openssl('x509', '-req', '-in', pki / f'{name}.csr', '-CA', pki / 'ca.cert', 61 '-CAkey', pki / 'ca.key', '-CAcreateserial', '-days', '1', 62 '-extfile', ext, '-out', pki / f'{name}.cert') 63 64 leaf('client', IDENTITY, 'clientAuth') 65 leaf('server', IDENTITY, 'serverAuth') 66 leaf('sentol', 'sentol', 'clientAuth') 67 leaf('receiver', 'receiver.example', 'serverAuth') 68 secret_values = [] 69 70 def bundle(): 71 public = {'monitoring_client': { 72 'identity': IDENTITY, 73 'monitoring_ca_certificate': (pki / 'ca.cert').read_text(), 74 'client_certificate': (pki / 'client.cert').read_text(), 75 'server_certificate': (pki / 'server.cert').read_text(), 76 'logs': {'protocol': 'jsonline', 'url': 'https://receiver.example:12514/jsonline'}, 77 'node_exporter': {'backend_listen_address': '127.0.0.1:9101', 78 'proxy_bind_address': '127.0.0.1', 'proxy_port': 9100, 79 'prometheus_client_identity': 'sentol'}}} 80 secrets = {'monitoring_client_secrets': { 81 'client_private_key': (pki / 'client.key').read_text(), 82 'server_private_key': (pki / 'server.key').read_text()}} 83 secret_values.extend(secrets['monitoring_client_secrets'].values()) 84 return public, secrets 85 86 def install_bundle(public, secrets, encrypt=True): 87 (host_vars / 'monitoring-client.yml').write_text(json.dumps(public)) 88 path = host_vars / 'monitoring-client-secrets.yml' 89 path.write_text(json.dumps(secrets)) 90 path.chmod(0o600) 91 if encrypt: 92 run('ansible-vault', 'encrypt', '--vault-password-file', vault_password, path, 93 env=dict(os.environ, ANSIBLE_CONFIG=str(REPO / 'test-ansible.cfg'))) 94 95 env = dict(os.environ, ANSIBLE_CONFIG=str(REPO / 'test-ansible.cfg'), ANSIBLE_NOCOWS='1') 96 ansible = ['ansible-playbook', '-i', str(inventory / 'hosts'), '-l', 'podman-localhost', 97 '--user', 'root', '--private-key', args.private_key, '--vault-password-file', 98 str(vault_password), '-vvv', '--diff'] 99 100 def invoke(playbook, enabled=True, check=False, failure=False): 101 command = ansible + ['-e', json.dumps({'enable_monitoring': enabled})] 102 if check: 103 command += ['--check'] 104 result = run(*command, playbook, env=env, check=False) 105 output = result.stdout + result.stderr 106 assert 'BEGIN PRIVATE KEY' not in output, 'Private key appeared in Ansible output' 107 for secret in secret_values: 108 assert secret.splitlines()[1] not in output, 'Private key appeared in Ansible output' 109 if bool(result.returncode) != failure: 110 print(output) 111 raise AssertionError(f'Unexpected monitoring playbook status: {result.returncode}') 112 return output 113 114 role_play = work / 'monitoring.json' 115 role_play.write_text(json.dumps([{'hosts': 'all', 'gather_facts': False, 116 'pre_tasks': [{'ansible.builtin.setup': {}, 'no_log': True}], 117 'tasks': [{'ansible.builtin.include_role': {'name': 'monitoring'}, 118 'when': 'enable_monitoring | bool'}]}])) 119 120 def application_pid(): 121 return container('systemctl', 'show', '-p', 'MainPID', 'taler-exchange-httpd').stdout 122 123 # Install and activate our own unrelated site; another regression's 124 # configuration file need not have been loaded by the running master. 125 container('tee', '/etc/nginx/conf.d/monitoring-regression-unrelated.conf', 126 input='server { listen 127.0.0.1:8098; return 200 "unrelated"; }\n') 127 container('nginx', '-t') 128 container('systemctl', 'reload', 'nginx') 129 container('curl', '--fail', '--silent', '--show-error', 'http://127.0.0.1:8098/') 130 pid = application_pid() 131 invoke(role_play, enabled=False) 132 assert container('test', '-e', '/etc/taler-monitoring', check=False).returncode != 0 133 invoke(REPO / 'playbooks/setup.yml', failure=True) 134 assert application_pid() == pid, 'Missing bundle stopped the exchange' 135 public, secrets = bundle() 136 install_bundle(public, secrets, encrypt=False) 137 invoke(REPO / 'playbooks/setup.yml', failure=True) 138 assert application_pid() == pid, 'Plaintext secrets stopped the exchange' 139 install_bundle(public, secrets) 140 (host_vars / 'monitoring-client.yml').unlink() 141 invoke(REPO / 'playbooks/setup.yml', failure=True) 142 install_bundle(public, secrets) 143 vault_password.write_text('incorrect-password\n') 144 invoke(REPO / 'playbooks/setup.yml', failure=True) 145 vault_password.write_text('disposable-monitoring-test-password\n') 146 147 for change in ['backend', 'wildcard-conflict', 'identity', 'purpose', 'key', 'missing', 'protocol', 'url', 'port', 'injection']: 148 bad_public, bad_secrets = copy.deepcopy(public), copy.deepcopy(secrets) 149 if change == 'backend': 150 bad_public['monitoring_client']['node_exporter']['backend_listen_address'] = '0.0.0.0:9101' 151 elif change == 'wildcard-conflict': 152 bad_public['monitoring_client']['node_exporter']['proxy_bind_address'] = '*' 153 bad_public['monitoring_client']['node_exporter']['proxy_port'] = 9101 154 elif change == 'identity': 155 bad_public['monitoring_client']['identity'] = 'wrong-host' 156 elif change == 'purpose': 157 bad_public['monitoring_client']['client_certificate'] = public['monitoring_client']['server_certificate'] 158 elif change == 'key': 159 bad_secrets['monitoring_client_secrets']['client_private_key'] = secrets['monitoring_client_secrets']['server_private_key'] 160 elif change == 'missing': 161 del bad_public['monitoring_client']['logs'] 162 bad_public['monitoring_client']['relp'] = {} 163 elif change == 'protocol': 164 bad_public['monitoring_client']['logs']['protocol'] = 'otlp' 165 elif change == 'url': 166 bad_public['monitoring_client']['logs']['url'] = 'https://receiver.example/jsonline?debug=1' 167 elif change == 'port': 168 bad_public['monitoring_client']['logs']['url'] = 'https://receiver.example:65536/jsonline' 169 else: 170 bad_public['monitoring_client']['node_exporter']['prometheus_client_identity'] = 'sentol"; }' 171 install_bundle(bad_public, bad_secrets) 172 invoke(REPO / 'playbooks/setup.yml', failure=True) 173 assert application_pid() == pid, f'Invalid {change} stopped the exchange' 174 print('PASS: missing, plaintext, undecryptable and invalid bundles fail before shutdown', flush=True) 175 176 install_bundle(public, secrets) 177 invoke(role_play, check=True) 178 assert container('test', '-e', '/etc/taler-monitoring', check=False).returncode != 0 179 # A legacy exporter may already own port 9100. Reconfiguration must 180 # move it to loopback before the dedicated TLS proxy takes that port. 181 container('apt-get', 'install', '-y', '--no-install-recommends', 'prometheus-node-exporter') 182 container('systemctl', 'start', 'prometheus-node-exporter') 183 container('sh', '-c', 'echo "127.0.0.1 receiver.example" >> /etc/hosts') 184 invoke(role_play) 185 186 remote = '/tmp/taler-monitoring-test' 187 container('install', '-d', '-m', '0700', remote) 188 run('podman', 'cp', pki, f'{args.container}:{remote}/pki') 189 190 def proxy(kind='sentol', path='/metrics', status=False, address='127.0.0.1'): 191 command = ['curl', '--silent', '--show-error', '--max-time', '10', '--noproxy', '*', 192 '--cacert', f'{remote}/pki/ca.cert', '--resolve', f'{IDENTITY}:9100:{address}'] 193 if kind: 194 command += ['--cert', f'{remote}/pki/{kind}.cert', '--key', f'{remote}/pki/{kind}.key'] 195 if status: 196 command += ['--output', '/dev/null', '--write-out', '%{http_code}'] 197 else: 198 command += ['--fail'] 199 return container(*command, f'https://{IDENTITY}:9100{path}', check=False) 200 201 def verify_metrics(): 202 for _ in range(30): 203 result = proxy() 204 if result.returncode == 0: 205 break 206 time.sleep(1) 207 assert result.returncode == 0, result.stderr 208 assert 'node_systemd_unit_state{' in result.stdout 209 assert 'taler-exchange-httpd.service' in result.stdout 210 assert proxy('client', status=True).stdout == '403' 211 assert proxy(None, status=True).stdout in ['400', '403'] 212 assert proxy(path='/other', status=True).stdout == '404' 213 listeners = container('ss', '-lnt').stdout 214 assert '127.0.0.1:9101' in listeners 215 assert '0.0.0.0:9101' not in listeners and '[::]:9101' not in listeners 216 assert container('curl', '--fail', '--max-time', '5', 'http://127.0.0.1:9100/metrics', 217 check=False).returncode != 0 218 container('systemctl', 'is-active', '--quiet', *SERVICES) 219 220 verify_metrics() 221 public['monitoring_client']['node_exporter']['proxy_bind_address'] = '*' 222 install_bundle(public, secrets) 223 invoke(role_play) 224 verify_metrics() 225 result = proxy(address='[::1]') 226 assert result.returncode == 0, result.stderr 227 assert 'node_systemd_unit_state{' in result.stdout 228 assert proxy(None, status=True, address='[::1]').stdout in ['400', '403'] 229 assert proxy('client', status=True, address='[::1]').stdout == '403' 230 print('PASS: wildcard proxy serves mTLS metrics on IPv4 and IPv6', flush=True) 231 for path, expected in [('tls', '700'), ('tls/client.key.pem', '600'), ('tls/server.key.pem', '600')]: 232 assert container('stat', '-c', '%a:%U:%G', f'/etc/taler-monitoring/{path}').stdout.strip() == f'{expected}:root:root' 233 assert container('su', '-s', '/bin/sh', 'nobody', '-c', 234 'test -r /etc/taler-monitoring/tls/client.key.pem', check=False).returncode != 0 235 # A real TLS HTTP receiver lets us exercise the output without another 236 # deployment repository. The tsys-infra suite also runs this role against 237 # its production nginx and VictoriaLogs configurations. 238 receiver = r'''import http.server 239 import json 240 import os 241 from pathlib import Path 242 import ssl 243 244 root = Path(__file__).parent 245 class Handler(http.server.BaseHTTPRequestHandler): 246 def do_POST(self): 247 body = self.rfile.read(int(self.headers['Content-Length'])) 248 subject = self.connection.getpeercert()['subject'] 249 if subject != ((('commonName', 'enrolled-exchange.example'),),): 250 self.send_error(403) 251 return 252 if self.path != '/jsonline': 253 self.send_error(404) 254 return 255 status = root / 'response-status' 256 if status.exists(): 257 self.send_error(int(status.read_text())) 258 return 259 rows = [json.loads(line) for line in body.splitlines()] 260 with (root / 'received.log').open('a') as output: 261 for row in rows: 262 output.write(json.dumps(row) + '\n') 263 output.flush() 264 os.fsync(output.fileno()) 265 self.send_response(200) 266 self.send_header('Content-Length', '0') 267 self.end_headers() 268 def log_message(self, *args): 269 pass 270 server = http.server.HTTPServer(('127.0.0.1', 12514), Handler) 271 context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) 272 context.load_cert_chain(root / 'pki/receiver.cert', root / 'pki/receiver.key') 273 context.load_verify_locations(root / 'pki/ca.cert') 274 context.verify_mode = ssl.CERT_REQUIRED 275 server.socket = context.wrap_socket(server.socket, server_side=True) 276 server.serve_forever() 277 ''' 278 container('tee', f'{remote}/receiver.py', input=receiver) 279 280 def receiver_start(): 281 container('systemd-run', '--unit=monitoring-test-receiver', '--collect', 282 'python3', f'{remote}/receiver.py') 283 284 def log_delivery(message, emit=True): 285 if emit: 286 container('logger', '--tag', 'monitoring-regression', message) 287 for _ in range(45): 288 received = container('cat', f'{remote}/received.log', check=False).stdout 289 if message in received: 290 return 291 time.sleep(1) 292 raise AssertionError('JSON Lines test message was not delivered') 293 294 receiver_start() 295 try: 296 log_delivery('initial-monitoring-enrollment') 297 print('PASS: mTLS metrics, client allowlist, systemd metrics, permissions and JSON Lines delivery', flush=True) 298 container('systemctl', 'stop', 'monitoring-test-receiver') 299 container('logger', '--tag', 'monitoring-regression', 'queued-during-receiver-outage') 300 time.sleep(2) 301 container('systemctl', 'restart', 'fluent-bit') 302 receiver_start() 303 log_delivery('queued-during-receiver-outage', emit=False) 304 print('PASS: Fluent Bit queue survives receiver outage and collector restart', flush=True) 305 # Permanent HTTP errors discard the chunk even with unlimited 306 # retries; verify this is exposed by the output's drop counter. 307 def drops(): 308 metrics = container('curl', '-fsS', 'http://127.0.0.1:2020/api/v1/metrics/prometheus').stdout 309 return sum(float(line.split()[1]) for line in metrics.splitlines() 310 if line.startswith('fluentbit_output_dropped_records_total{')) 311 container('tee', f'{remote}/response-status', input='400') 312 prior_drops = drops() 313 container('logger', '-t', 'monitoring-regression', 'permanently-rejected-jsonline') 314 for _ in range(20): 315 if drops() > prior_drops: 316 break 317 time.sleep(1) 318 assert drops() > prior_drops, 'HTTP 400 did not expose dropped records' 319 container('rm', f'{remote}/response-status') 320 log_delivery('recovered-after-permanent-error') 321 print('PASS: permanent HTTP errors expose dropped records and later requests recover', flush=True) 322 before = container('systemctl', 'show', '-p', 'MainPID', *SERVICES).stdout 323 output = invoke(role_play) 324 assert re.search(r'changed=0\s', output), 'Second monitoring deployment was not idempotent' 325 invoke(role_play, check=True) 326 invoke(role_play, enabled=False) 327 assert container('systemctl', 'show', '-p', 'MainPID', *SERVICES).stdout == before 328 329 leaf('client', IDENTITY, 'clientAuth') 330 leaf('server', IDENTITY, 'serverAuth') 331 public, secrets = bundle() 332 install_bundle(public, secrets) 333 invoke(role_play) 334 verify_metrics() 335 log_delivery('renewed-monitoring-certificates') 336 assert container('cat', '/etc/taler-monitoring/tls/server.cert.pem').stdout == (pki / 'server.cert').read_text() 337 print('PASS: idempotence, check mode, disabled management and certificate renewal', flush=True) 338 339 # Exercise the real setup entrypoint with monitoring enabled. 340 invoke(REPO / 'playbooks/setup.yml') 341 verify_metrics() 342 log_delivery('monitoring-survives-deployment') 343 container('curl', '--fail', '--silent', '--show-error', '--unix-socket', 344 '/var/run/taler-exchange/httpd/exchange-http.sock', 'http://localhost/config') 345 container('curl', '--fail', '--silent', '--show-error', 'http://127.0.0.1:8098/') 346 print('PASS: normal deployment preserves monitoring and exchange endpoints', flush=True) 347 finally: 348 container('systemctl', 'stop', 'monitoring-test-receiver', check=False) 349 container('rm', '-rf', remote) 350 351 352 if __name__ == '__main__': 353 main()