test_deployment.py (16099B)
1 #!/usr/bin/env python3 2 """Integration regressions for the disposable container created by test.""" 3 import argparse 4 import json 5 import os 6 import re 7 from pathlib import Path 8 import subprocess 9 import tempfile 10 11 from test_upgrade_policy import verify_upgrade_policy_restoration 12 13 REPO = Path(__file__).resolve().parents[2] 14 UNITS = ['taler-exchange.target', 'taler-exchange-httpd.service', 15 'libeufin-nexus-httpd.service', 'taler-auditor.target', 16 'taler-auditor-httpd.service', 'sms-challenger-httpd.service', 17 'email-challenger-httpd.service', 'postal-challenger-httpd.service'] 18 19 20 def verify_nginx_configuration(container): 21 assert container('test', '-e', '/etc/nginx/conf.d/http2-http3.conf', check=False).returncode == 1 22 result = container('nginx', '-t') 23 assert 'duplicate' not in result.stderr, result.stderr 24 conffiles = container('dpkg-query', '-W', '-f=${Conffiles}', 'nginx-common').stdout 25 packaged_hash = re.search(r'^ /etc/nginx/nginx\.conf ([0-9a-f]+)$', conffiles, re.M)[1] 26 installed_hash = container('md5sum', '/etc/nginx/nginx.conf').stdout.split()[0] 27 assert installed_hash == packaged_hash, 'Deployment modified Debian\'s nginx.conf' 28 print('PASS: nginx accepts the managed sites with an unchanged Debian configuration', flush=True) 29 30 31 def main(): 32 parser = argparse.ArgumentParser() 33 parser.add_argument('container') 34 parser.add_argument('private_key') 35 parser.add_argument('setup_log') 36 args = parser.parse_args() 37 with tempfile.TemporaryDirectory(prefix='taler-deployment-regressions-') as directory: 38 work = Path(directory) 39 def container(*cmd, check=True, input=None): 40 return subprocess.run(['podman', 'exec', '-i', args.container, *cmd], 41 capture_output=True, text=True, check=check, input=input) 42 43 verify_nginx_configuration(container) 44 45 subprocess.run(['podman', 'cp', str(REPO / 'contrib/test-fact-helpers.sh'), 46 f'{args.container}:/tmp/test-fact-helpers.sh'], check=True) 47 helper_result = container('env', 'TALER_FACT_HELPERS_DIR=/bin', 'bash', '/tmp/test-fact-helpers.sh') 48 print(helper_result.stdout, end='', flush=True) 49 50 facts = json.loads(container('python3', '-c', '''import glob,json 51 print(json.dumps([json.load(open(p)) for p in glob.glob('/etc/ansible/facts.d/*secret.fact') + glob.glob('/etc/ansible/facts.d/*access-token.fact')])) 52 ''').stdout) 53 # These values come exclusively from the disposable testing inventory. 54 auditor_auth = container('cat', '/etc/nginx/auditor-auth.conf.inc').stdout 55 auditor_token = re.search(r'Bearer ([^"]+)', auditor_auth)[1] 56 secret_values = facts + ['SECRET2', auditor_token] 57 def assert_no_secrets(output): 58 for value in secret_values: 59 assert value not in output, 'A secret appeared in deployment output' 60 61 assert_no_secrets(Path(args.setup_log).read_text()) 62 ansible = ['ansible-playbook', '-i', str(REPO / 'inventories/default'), 63 '-l', 'podman-localhost', '--user', 'root', 64 '--private-key', args.private_key] 65 env = dict(os.environ, ANSIBLE_CONFIG=str(REPO / 'test-ansible.cfg'), ANSIBLE_NOCOWS='1') 66 def play(tasks, variables=None, check_mode=False, expect_failure=False): 67 p = work / 'regression.json' 68 p.write_text(json.dumps([{'hosts': 'all', 'gather_facts': False, 69 'pre_tasks': [{'ansible.builtin.setup': {}, 'no_log': True}], 70 'vars': variables or {}, 'tasks': tasks}])) 71 return invoke(p, check_mode, expect_failure) 72 73 def invoke(p, check_mode=False, expect_failure=False): 74 cmd = ansible + ['-vvv', '--diff'] 75 if check_mode: 76 cmd += ['--check'] 77 result = subprocess.run(cmd + [str(p)], capture_output=True, text=True, env=env) 78 output = result.stdout + result.stderr 79 assert_no_secrets(output) 80 if bool(result.returncode) != expect_failure: 81 print(output) 82 raise AssertionError(f'Unexpected Ansible exit status: {result.returncode}') 83 return output 84 85 def role(name, **variables): 86 return {'ansible.builtin.include_role': {'name': name}, 'vars': variables} 87 88 # Repair an old global drop-in that conflicts with Debian's nginx.conf. 89 container('tee', '/etc/nginx/conf.d/http2-http3.conf', input='ssl_prefer_server_ciphers on;\n') 90 assert container('nginx', '-t', check=False).returncode != 0 91 play([role('webserver')]) 92 verify_nginx_configuration(container) 93 94 def states(): 95 return [container('systemctl', 'is-active', unit, check=False).stdout.strip() for unit in UNITS] 96 97 def assert_active(): 98 assert states() == ['active'] * len(UNITS), states() 99 100 assert_active() 101 # Check mode must not restart active application processes. 102 before = container('systemctl', 'show', '--property=MainPID', *UNITS).stdout 103 invoke(REPO / 'playbooks/setup.yml', check_mode=True) 104 after = container('systemctl', 'show', '--property=MainPID', *UNITS).stdout 105 assert before == after, 'Check mode restarted application processes' 106 print('PASS: check mode preserves running applications', flush=True) 107 108 # All three KYC consumers can read the unchanged key; unrelated users cannot. 109 tasks = [] 110 for user in ['taler-exchange-httpd', 'taler-exchange-aggregator', 'taler-exchange-sanctionscheck']: 111 tasks += [{'ansible.builtin.command': {'argv': ['taler-exchange-config', '-c', 112 '/etc/taler-exchange/taler-exchange.conf', '-s', 'exchange', '-o', 'ATTRIBUTE_ENCRYPTION_KEY']}, 113 'become': True, 'become_user': user, 'register': 'key_read', 'no_log': True, 'changed_when': False}, 114 {'ansible.builtin.assert': {'that': 'key_read.stdout == exchange_attribute_encryption_key'}, 'no_log': True}] 115 for path in ['/etc/taler-exchange/secrets/exchange-attributes.secret.conf', '/etc/nginx/auditor-auth.conf.inc']: 116 tasks += [{'ansible.builtin.command': {'argv': ['test', '-r', path]}, 117 'become': True, 'become_user': 'nobody', 'register': 'read_test', 118 'changed_when': False, 'failed_when': 'read_test.rc != 1'}] 119 tasks += [{'ansible.builtin.command': {'argv': ['taler-exchange-config', '-c', 120 '/etc/taler-exchange/taler-exchange.conf', '-s', 'exchange', '-o', 'CURRENCY']}, 121 'become': True, 'become_user': 'nobody', 'changed_when': False}] 122 play(tasks) 123 auditor_site = container('cat', '/etc/nginx/sites-available/auditor-nginx.conf').stdout 124 assert auditor_token not in auditor_site 125 assert container('grep', '-q', 'ATTRIBUTE_ENCRYPTION_KEY =', 126 '/etc/taler-exchange/conf.d/exchange-business.conf', check=False).returncode == 1 127 auditor_host = re.search(r'server_name ([^;]+);', auditor_site)[1] 128 def http_status(path, token=None): 129 cmd = ['curl', '--noproxy', '*', '--silent', '--show-error', '--insecure', 130 '--resolve', f'{auditor_host}:443:127.0.0.1', '-o', '/dev/null', '-w', '%{http_code}'] 131 if token: 132 cmd += ['-H', 'Authorization: Bearer ' + token] 133 return container(*cmd, 'https://' + auditor_host + path).stdout 134 assert http_status('/config') == '200' 135 assert http_status('/regression-protected') == '401' 136 assert http_status('/regression-protected', auditor_token) != '401' 137 print('PASS: secrets are restricted and public configuration remains readable', flush=True) 138 139 # Actual services/targets and timers must stop, with enablement preserved. 140 container('systemd-run', '--unit=taler-exchange-regression', '--on-active=1h', '/bin/true') 141 enabled_before = container('systemctl', 'is-enabled', *UNITS, check=False).stdout 142 output = play([role('stop_services', stop_services_include_merchant=False), 143 {'ansible.builtin.assert': {'that': [ 144 "'taler-exchange.target' in stop_services_active_units", 145 "'taler-auditor.target' in stop_services_active_units", 146 "'sms-challenger-httpd.service' in stop_services_active_units", 147 "'taler-exchange-regression.timer' in stop_services_active_units"]}}]) 148 assert 'active' not in states(), states() 149 assert enabled_before == container('systemctl', 'is-enabled', *UNITS, check=False).stdout 150 print('PASS: shutdown includes targets, services, and timers without disabling them', flush=True) 151 152 # Run the real dependency bootstrap on the image's pre-2.20 controller. 153 version = container('ansible-playbook', '--version').stdout.splitlines()[0] 154 assert 'core 2.19.' in version or 'core 2.18.' in version, version 155 container('mkdir', '-p', '/tmp/taler-compat/roles') 156 subprocess.run(['podman', 'cp', str(REPO / 'roles/common_packages'), 157 f'{args.container}:/tmp/taler-compat/roles/common_packages'], check=True) 158 compat = [{'hosts': 'localhost', 'connection': 'local', 'vars': { 159 'taler_repo_suites': 'trixie', 'use_pregenerated_dhparam': True}, 'roles': ['common_packages']}] 160 container('python3', '-c', 'import sys; open("/tmp/taler-compat/play.json","w").write(sys.stdin.read())', 161 input=json.dumps(compat)) 162 container('dpkg', '--remove', '--force-depends', 'python3-debian') 163 compat_result = container('ansible-playbook', '-i', 'localhost,', '/tmp/taler-compat/play.json', check=False) 164 assert compat_result.returncode == 0, compat_result.stdout + compat_result.stderr 165 print('PASS: dependency bootstrap on ' + version, flush=True) 166 167 verify_upgrade_policy_restoration(container, play) 168 169 # Exercise package scripts attempting to start a service and restoration 170 # of a pre-existing policy-rc.d. Everything here is disposable test data. 171 container('sh', '-eu', '-c', ''' 172 mkdir -p /tmp/taler-policy-probe/DEBIAN /tmp/taler-policy-probe/etc/init.d 173 cat > /tmp/taler-policy-probe/DEBIAN/control <<'EOF' 174 Package: taler-policy-probe 175 Version: 1.0 176 Architecture: all 177 Maintainer: Test <test@example.invalid> 178 Description: Disposable service-start policy probe 179 EOF 180 cat > /tmp/taler-policy-probe/DEBIAN/postinst <<'EOF' 181 #!/bin/sh 182 set -e 183 invoke-rc.d taler-policy-probe start 184 EOF 185 cat > /tmp/taler-policy-probe/etc/init.d/taler-policy-probe <<'EOF' 186 #!/bin/sh 187 touch /tmp/taler-policy-started 188 EOF 189 chmod 755 /tmp/taler-policy-probe/DEBIAN/postinst /tmp/taler-policy-probe/etc/init.d/taler-policy-probe 190 printf '#!/bin/sh\\nexit 0\\n' > /usr/sbin/policy-rc.d 191 chmod 755 /usr/sbin/policy-rc.d 192 dpkg-deb --build /tmp/taler-policy-probe /tmp/taler-policy-probe.deb 193 ''') 194 original_policy = container('cat', '/usr/sbin/policy-rc.d').stdout 195 play([{'ansible.builtin.apt': {'deb': '/tmp/taler-policy-probe.deb', 'policy_rc_d': 101}}]) 196 assert container('test', '-e', '/tmp/taler-policy-started', check=False).returncode == 1 197 assert container('cat', '/usr/sbin/policy-rc.d').stdout == original_policy 198 container('rm', '/usr/sbin/policy-rc.d') 199 container('dpkg', '--remove', 'taler-policy-probe') 200 assert 'active' not in states(), states() 201 print('PASS: package starts are suppressed and existing policy is restored', flush=True) 202 203 # Restore skipped optional services from a captured list without changing flags. 204 play([role('start_services')], {'deployment_previously_active_units': UNITS, 205 'deploy_auditor': False, 'deploy_challenger': False}) 206 assert_active() 207 print('PASS: previously active optional services resume', flush=True) 208 209 # Revocation must affect existing sessions and retain the account's data. 210 dev_vars = {'dangerously_enable_devtesting': True, 'devtesting_ssh_keys': []} 211 play([role('devtesting')], dev_vars) 212 container('touch', '/home/devtesting/keep-data') 213 container('systemd-run', '--unit=devtesting-regression', '--uid=devtesting', '/bin/sleep', '600') 214 play([role('devtesting')], {'dangerously_enable_devtesting': False}) 215 assert container('pgrep', '-u', 'devtesting', check=False).returncode == 1 216 assert container('test', '-f', '/home/devtesting/keep-data', check=False).returncode == 0 217 assert container('test', '-e', '/etc/sudoers.d/devtesting', check=False).returncode == 1 218 assert '/usr/sbin/nologin' in container('getent', 'passwd', 'devtesting').stdout 219 result = container('su', '-s', '/bin/sh', 'devtesting', '-c', 220 'sudo -n -u libeufin-nexus /usr/bin/libeufin-nexus testing --help', check=False) 221 assert result.returncode != 0, 'Revoked user retained sudo access' 222 play([role('devtesting')], dev_vars) 223 assert '/bin/bash' in container('getent', 'passwd', 'devtesting').stdout 224 assert container('test', '-f', '/etc/sudoers.d/devtesting', check=False).returncode == 0 225 play([role('devtesting')], {'dangerously_enable_devtesting': False}) 226 print('PASS: devtesting revocation and re-enablement preserve home data', flush=True) 227 228 # Backup provisioning must not overwrite another application's SSH settings. 229 container('sh', '-eu', '-c', ''' 230 printf 'Host unrelated\\n HostName example.invalid\\n' > /root/.ssh/config 231 ssh-keygen -q -t ed25519 -N '' -f /root/.ssh/borg 232 ssh-keyscan localhost > /root/.ssh/known_hosts 233 ''') 234 ssh_before = container('cat', '/root/.ssh/config').stdout 235 play([role('borg-start')], {'borg_host': 'localhost', 'borg_repo': '/tmp/test-backups', 236 'borg_passphrase': 'synthetic-backup-passphrase'}) 237 assert container('cat', '/root/.ssh/config').stdout == ssh_before 238 assert 'IdentityFile ~/.ssh/borg' in container('ssh', '-G', '-F', '/root/.ssh/borg-config', 'localhost').stdout or \ 239 'identityfile ~/.ssh/borg' in container('ssh', '-G', '-F', '/root/.ssh/borg-config', 'localhost').stdout 240 container('rm', '-f', '/var/spool/cron/crontabs/root') 241 for override in [None, '/tmp/independent-repo']: 242 variables = {} if override is None else {'borg_repo': override} 243 play([{'ansible.builtin.include_vars': str(REPO / 'roles/database_restore/defaults/main.yml')}, 244 {'ansible.builtin.assert': {'that': 'database_restore_borg_repository == borg_repo'}}], variables) 245 print('PASS: SSH configuration survives and backup/restore repository settings agree', flush=True) 246 247 # An invalid global config must fail a real redeploy, preserve unrelated 248 # site links, and leave apps stopped. A subsequent corrected redeploy recovers. 249 container('sh', '-eu', '-c', ''' 250 printf 'server { listen 127.0.0.1:8099; return 200 "unrelated"; }\\n' > /etc/nginx/sites-available/unrelated 251 ln -s /etc/nginx/sites-available/unrelated /etc/nginx/sites-enabled/unrelated 252 printf 'invalid_regression_directive;\\n' > /etc/nginx/conf.d/regression-invalid.conf 253 ''') 254 invoke(REPO / 'playbooks/setup.yml', expect_failure=True) 255 assert 'active' not in states(), states() 256 assert container('test', '-L', '/etc/nginx/sites-enabled/unrelated', check=False).returncode == 0 257 container('rm', '/etc/nginx/conf.d/regression-invalid.conf') 258 invoke(REPO / 'playbooks/setup.yml') 259 assert_active() 260 assert container('test', '-L', '/etc/nginx/sites-enabled/unrelated', check=False).returncode == 0 261 assert container('cat', '/root/.ssh/config').stdout == ssh_before 262 print('PASS: failed deployment preserves sites and stops applications; redeploy recovers', flush=True) 263 264 265 if __name__ == '__main__': 266 main()