commit 323ef8f4a478bab0da5bc08f29d62338ab4ee621
parent 4f133f3a273daa863c3f638a99ad2851e4734bde
Author: Florian Dold <dold@taler.net>
Date: Sat, 5 Sep 2026 20:13:00 +0200
Regression checks: exercise deployment failures and recovery in CI
Use the shared disposable-container runner in CI and cover application
shutdown, package-start suppression, secret access, devtesting
revocation, configuration preservation and recovery. Run fact helpers
inside the container and retain fixtures only when explicitly requested.
Diffstat:
7 files changed, 283 insertions(+), 52 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,6 @@ vault_pass.txt
# Text editor files
*~
.vscode
+
+# Python regression test bytecode
+__pycache__/
diff --git a/README b/README
@@ -376,3 +376,13 @@ encounters another backup, it exits nonzero (script status 75) without touching
that run's snapshot. Retry after the existing backup finishes; a reboot playbook
aborts before rebooting on this failure. Pruning and compaction are skipped after
an unsuccessful or warning-producing archive creation.
+
+## Regression checks
+
+Run python3 contrib/tests/test_backup.py for isolated concurrency, failure,
+cleanup and archive round-trip checks (requires Borg and Python Jinja2).
+The extended ./test.sh also exercises real systemd shutdown, package-start
+suppression, a pre-2.20 Ansible dependency bootstrap, secret permissions,
+devtesting revocation, preservation of unrelated configuration, failed deployment
+and recovery. All fixtures run in the disposable container, never an inventory
+production host.
diff --git a/contrib/ci/Containerfile b/contrib/ci/Containerfile
@@ -2,6 +2,4 @@
FROM quay.io/podman/stable:v5.2.3
RUN dnf update -yq && \
- dnf install -yq \
- ansible #\
- #systemd
+ dnf install -yq ansible borgbackup ShellCheck
diff --git a/contrib/ci/jobs/001-build/build.sh b/contrib/ci/jobs/001-build/build.sh
@@ -1,44 +1,7 @@
#!/bin/bash
-set -exuo pipefail
+set -euo pipefail
-#### WARNING: THIS SCRIPT IS INTENED TO BE RUN INSIDE OF A CONTAINER
-
-
-# Print some debug info
-id ; cat /proc/self/uid_map ; mount | grep cgroup || true
-
-# Check the fact helpers before spending time on the container
-contrib/test-fact-helpers.sh
-
-# Hack to make podman adapt to being nested
+# This job runs inside the disposable CI container. The shared runner provisions
+# its nested container over the published SSH port and executes all regressions.
rm -f /etc/containers/storage.conf
-
-# Build our image
-podman build -f Containerfile -t ansible-taler-test
-
-# Run in background (-d) with systemd init
-podman run \
- --privileged \
- --tmpfs /sys \
- --rm \
- --name ansible-taler-test \
- -d localhost/ansible-taler-test sh -c "id ; cat /proc/self/uid_map ; mount | grep cgroup; exec /usr/sbin/init --show-status"
-
-# Print to log that container is running
-podman ps
-
-# TOFU SSH host keys (so we don't get user prompt)
-echo "StrictHostKeyChecking=accept-new" > ~/.ssh/config
-
-# Run our playbook(s)
-# NOTE: Trailing comma is correct (and required) in agument for -i flag
-ansible-playbook --verbose -i 127.0.0.1:22, --user root playbooks/setup.yml
-
-echo -e '
- #############################
- #############################
- #############################
- ###### Setup finished. ######
- #############################
- #############################
- #############################'
+exec ./test.sh
diff --git a/contrib/test-fact-helpers.sh b/contrib/test-fact-helpers.sh
@@ -7,7 +7,7 @@
set -u
-helpers="$(dirname "$0")/../roles/common_packages/files"
+helpers="${TALER_FACT_HELPERS_DIR:-$(dirname "$0")/../roles/common_packages/files}"
failures=0
check() {
diff --git a/contrib/tests/test_deployment.py b/contrib/tests/test_deployment.py
@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Integration regressions for the disposable container created by test.sh."""
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+import subprocess
+import tempfile
+
+from test_upgrade_policy import verify_upgrade_policy_restoration
+
+REPO = Path(__file__).resolve().parents[2]
+UNITS = ['taler-exchange.target', 'taler-exchange-httpd.service',
+ 'libeufin-nexus-httpd.service', 'taler-auditor.target',
+ 'taler-auditor-httpd.service', 'sms-challenger-httpd.service',
+ 'email-challenger-httpd.service', 'postal-challenger-httpd.service']
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('container')
+ parser.add_argument('private_key')
+ parser.add_argument('setup_log')
+ args = parser.parse_args()
+ with tempfile.TemporaryDirectory(prefix='taler-deployment-regressions-') as directory:
+ work = Path(directory)
+ def container(*cmd, check=True, input=None):
+ return subprocess.run(['podman', 'exec', '-i', args.container, *cmd],
+ capture_output=True, text=True, check=check, input=input)
+
+ subprocess.run(['podman', 'cp', str(REPO / 'contrib/test-fact-helpers.sh'),
+ f'{args.container}:/tmp/test-fact-helpers.sh'], check=True)
+ helper_result = container('env', 'TALER_FACT_HELPERS_DIR=/bin', 'bash', '/tmp/test-fact-helpers.sh')
+ print(helper_result.stdout, end='', flush=True)
+
+ facts = json.loads(container('python3', '-c', '''import glob,json
+print(json.dumps([json.load(open(p)) for p in glob.glob('/etc/ansible/facts.d/*secret.fact') + glob.glob('/etc/ansible/facts.d/*access-token.fact')]))
+''').stdout)
+ # These values come exclusively from the disposable testing inventory.
+ auditor_auth = container('cat', '/etc/nginx/auditor-auth.conf.inc').stdout
+ auditor_token = re.search(r'Bearer ([^"]+)', auditor_auth)[1]
+ secret_values = facts + ['SECRET2', auditor_token]
+ def assert_no_secrets(output):
+ for value in secret_values:
+ assert value not in output, 'A secret appeared in deployment output'
+
+ assert_no_secrets(Path(args.setup_log).read_text())
+ ansible = ['ansible-playbook', '-i', str(REPO / 'inventories/default'),
+ '-l', 'podman-localhost', '--user', 'root',
+ '--private-key', args.private_key]
+ env = dict(os.environ, ANSIBLE_CONFIG=str(REPO / 'test-ansible.cfg'), ANSIBLE_NOCOWS='1')
+ def play(tasks, variables=None, check_mode=False, expect_failure=False):
+ p = work / 'regression.json'
+ p.write_text(json.dumps([{'hosts': 'all', 'gather_facts': False,
+ 'pre_tasks': [{'ansible.builtin.setup': {}, 'no_log': True}],
+ 'vars': variables or {}, 'tasks': tasks}]))
+ return invoke(p, check_mode, expect_failure)
+
+ def invoke(p, check_mode=False, expect_failure=False):
+ cmd = ansible + ['-vvv', '--diff']
+ if check_mode:
+ cmd += ['--check']
+ result = subprocess.run(cmd + [str(p)], capture_output=True, text=True, env=env)
+ output = result.stdout + result.stderr
+ assert_no_secrets(output)
+ if bool(result.returncode) != expect_failure:
+ print(output)
+ raise AssertionError(f'Unexpected Ansible exit status: {result.returncode}')
+ return output
+
+ def role(name, **variables):
+ return {'ansible.builtin.include_role': {'name': name}, 'vars': variables}
+
+ def states():
+ return [container('systemctl', 'is-active', unit, check=False).stdout.strip() for unit in UNITS]
+
+ def assert_active():
+ assert states() == ['active'] * len(UNITS), states()
+
+ assert_active()
+ # Check mode must not restart active application processes.
+ before = container('systemctl', 'show', '--property=MainPID', *UNITS).stdout
+ invoke(REPO / 'playbooks/setup.yml', check_mode=True)
+ after = container('systemctl', 'show', '--property=MainPID', *UNITS).stdout
+ assert before == after, 'Check mode restarted application processes'
+ print('PASS: check mode preserves running applications', flush=True)
+
+ # All three KYC consumers can read the unchanged key; unrelated users cannot.
+ tasks = []
+ for user in ['taler-exchange-httpd', 'taler-exchange-aggregator', 'taler-exchange-sanctionscheck']:
+ tasks += [{'ansible.builtin.command': {'argv': ['taler-exchange-config', '-c',
+ '/etc/taler-exchange/taler-exchange.conf', '-s', 'exchange', '-o', 'ATTRIBUTE_ENCRYPTION_KEY']},
+ 'become': True, 'become_user': user, 'register': 'key_read', 'no_log': True, 'changed_when': False},
+ {'ansible.builtin.assert': {'that': 'key_read.stdout == exchange_attribute_encryption_key'}, 'no_log': True}]
+ for path in ['/etc/taler-exchange/secrets/exchange-attributes.secret.conf', '/etc/nginx/auditor-auth.conf.inc']:
+ tasks += [{'ansible.builtin.command': {'argv': ['test', '-r', path]},
+ 'become': True, 'become_user': 'nobody', 'register': 'read_test',
+ 'changed_when': False, 'failed_when': 'read_test.rc != 1'}]
+ tasks += [{'ansible.builtin.command': {'argv': ['taler-exchange-config', '-c',
+ '/etc/taler-exchange/taler-exchange.conf', '-s', 'exchange', '-o', 'CURRENCY']},
+ 'become': True, 'become_user': 'nobody', 'changed_when': False}]
+ play(tasks)
+ auditor_site = container('cat', '/etc/nginx/sites-available/auditor-nginx.conf').stdout
+ assert auditor_token not in auditor_site
+ assert container('grep', '-q', 'ATTRIBUTE_ENCRYPTION_KEY =',
+ '/etc/taler-exchange/conf.d/exchange-business.conf', check=False).returncode == 1
+ auditor_host = re.search(r'server_name ([^;]+);', auditor_site)[1]
+ def http_status(path, token=None):
+ cmd = ['curl', '--noproxy', '*', '--silent', '--show-error', '--insecure',
+ '--resolve', f'{auditor_host}:443:127.0.0.1', '-o', '/dev/null', '-w', '%{http_code}']
+ if token:
+ cmd += ['-H', 'Authorization: Bearer ' + token]
+ return container(*cmd, 'https://' + auditor_host + path).stdout
+ assert http_status('/config') == '200'
+ assert http_status('/regression-protected') == '401'
+ assert http_status('/regression-protected', auditor_token) != '401'
+ print('PASS: secrets are restricted and public configuration remains readable', flush=True)
+
+ # Actual services/targets and timers must stop, with enablement preserved.
+ container('systemd-run', '--unit=taler-exchange-regression', '--on-active=1h', '/bin/true')
+ enabled_before = container('systemctl', 'is-enabled', *UNITS, check=False).stdout
+ output = play([role('stop_services', stop_services_include_merchant=False),
+ {'ansible.builtin.assert': {'that': [
+ "'taler-exchange.target' in stop_services_active_units",
+ "'taler-auditor.target' in stop_services_active_units",
+ "'sms-challenger-httpd.service' in stop_services_active_units",
+ "'taler-exchange-regression.timer' in stop_services_active_units"]}}])
+ assert 'active' not in states(), states()
+ assert enabled_before == container('systemctl', 'is-enabled', *UNITS, check=False).stdout
+ print('PASS: shutdown includes targets, services, and timers without disabling them', flush=True)
+
+ # Run the real dependency bootstrap on the image's pre-2.20 controller.
+ version = container('ansible-playbook', '--version').stdout.splitlines()[0]
+ assert 'core 2.19.' in version or 'core 2.18.' in version, version
+ container('mkdir', '-p', '/tmp/taler-compat/roles')
+ subprocess.run(['podman', 'cp', str(REPO / 'roles/common_packages'),
+ f'{args.container}:/tmp/taler-compat/roles/common_packages'], check=True)
+ compat = [{'hosts': 'localhost', 'connection': 'local', 'vars': {
+ 'taler_repo_suites': 'trixie', 'use_pregenerated_dhparam': True}, 'roles': ['common_packages']}]
+ container('python3', '-c', 'import sys; open("/tmp/taler-compat/play.json","w").write(sys.stdin.read())',
+ input=json.dumps(compat))
+ container('dpkg', '--remove', '--force-depends', 'python3-debian')
+ compat_result = container('ansible-playbook', '-i', 'localhost,', '/tmp/taler-compat/play.json', check=False)
+ assert compat_result.returncode == 0, compat_result.stdout + compat_result.stderr
+ print('PASS: dependency bootstrap on ' + version, flush=True)
+
+ verify_upgrade_policy_restoration(container, play)
+
+ # Exercise package scripts attempting to start a service and restoration
+ # of a pre-existing policy-rc.d. Everything here is disposable test data.
+ container('sh', '-eu', '-c', '''
+mkdir -p /tmp/taler-policy-probe/DEBIAN /tmp/taler-policy-probe/etc/init.d
+cat > /tmp/taler-policy-probe/DEBIAN/control <<'EOF'
+Package: taler-policy-probe
+Version: 1.0
+Architecture: all
+Maintainer: Test <test@example.invalid>
+Description: Disposable service-start policy probe
+EOF
+cat > /tmp/taler-policy-probe/DEBIAN/postinst <<'EOF'
+#!/bin/sh
+set -e
+invoke-rc.d taler-policy-probe start
+EOF
+cat > /tmp/taler-policy-probe/etc/init.d/taler-policy-probe <<'EOF'
+#!/bin/sh
+touch /tmp/taler-policy-started
+EOF
+chmod 755 /tmp/taler-policy-probe/DEBIAN/postinst /tmp/taler-policy-probe/etc/init.d/taler-policy-probe
+printf '#!/bin/sh\\nexit 0\\n' > /usr/sbin/policy-rc.d
+chmod 755 /usr/sbin/policy-rc.d
+dpkg-deb --build /tmp/taler-policy-probe /tmp/taler-policy-probe.deb
+''')
+ original_policy = container('cat', '/usr/sbin/policy-rc.d').stdout
+ play([{'ansible.builtin.apt': {'deb': '/tmp/taler-policy-probe.deb', 'policy_rc_d': 101}}])
+ assert container('test', '-e', '/tmp/taler-policy-started', check=False).returncode == 1
+ assert container('cat', '/usr/sbin/policy-rc.d').stdout == original_policy
+ container('rm', '/usr/sbin/policy-rc.d')
+ container('dpkg', '--remove', 'taler-policy-probe')
+ assert 'active' not in states(), states()
+ print('PASS: package starts are suppressed and existing policy is restored', flush=True)
+
+ # Restore skipped optional services from a captured list without changing flags.
+ play([role('start_services')], {'deployment_previously_active_units': UNITS,
+ 'deploy_auditor': False, 'deploy_challenger': False})
+ assert_active()
+ print('PASS: previously active optional services resume', flush=True)
+
+ # Revocation must affect existing sessions and retain the account's data.
+ dev_vars = {'dangerously_enable_devtesting': True, 'devtesting_ssh_keys': []}
+ play([role('devtesting')], dev_vars)
+ container('touch', '/home/devtesting/keep-data')
+ container('systemd-run', '--unit=devtesting-regression', '--uid=devtesting', '/bin/sleep', '600')
+ play([role('devtesting')], {'dangerously_enable_devtesting': False})
+ assert container('pgrep', '-u', 'devtesting', check=False).returncode == 1
+ assert container('test', '-f', '/home/devtesting/keep-data', check=False).returncode == 0
+ assert container('test', '-e', '/etc/sudoers.d/devtesting', check=False).returncode == 1
+ assert '/usr/sbin/nologin' in container('getent', 'passwd', 'devtesting').stdout
+ result = container('su', '-s', '/bin/sh', 'devtesting', '-c',
+ 'sudo -n -u libeufin-nexus /usr/bin/libeufin-nexus testing --help', check=False)
+ assert result.returncode != 0, 'Revoked user retained sudo access'
+ play([role('devtesting')], dev_vars)
+ assert '/bin/bash' in container('getent', 'passwd', 'devtesting').stdout
+ assert container('test', '-f', '/etc/sudoers.d/devtesting', check=False).returncode == 0
+ play([role('devtesting')], {'dangerously_enable_devtesting': False})
+ print('PASS: devtesting revocation and re-enablement preserve home data', flush=True)
+
+ # Backup provisioning must not overwrite another application's SSH settings.
+ container('sh', '-eu', '-c', '''
+printf 'Host unrelated\\n HostName example.invalid\\n' > /root/.ssh/config
+ssh-keygen -q -t ed25519 -N '' -f /root/.ssh/borg
+ssh-keyscan localhost > /root/.ssh/known_hosts
+''')
+ ssh_before = container('cat', '/root/.ssh/config').stdout
+ play([role('borg-start')], {'borg_host': 'localhost', 'borg_repo': '/tmp/test-backups',
+ 'borg_passphrase': 'synthetic-backup-passphrase'})
+ assert container('cat', '/root/.ssh/config').stdout == ssh_before
+ assert 'IdentityFile ~/.ssh/borg' in container('ssh', '-G', '-F', '/root/.ssh/borg-config', 'localhost').stdout or \
+ 'identityfile ~/.ssh/borg' in container('ssh', '-G', '-F', '/root/.ssh/borg-config', 'localhost').stdout
+ container('rm', '-f', '/var/spool/cron/crontabs/root')
+ for override in [None, '/tmp/independent-repo']:
+ variables = {} if override is None else {'borg_repo': override}
+ play([{'ansible.builtin.include_vars': str(REPO / 'roles/database_restore/defaults/main.yml')},
+ {'ansible.builtin.assert': {'that': 'database_restore_borg_repository == borg_repo'}}], variables)
+ print('PASS: SSH configuration survives and backup/restore repository settings agree', flush=True)
+
+ # An invalid global config must fail a real redeploy, preserve unrelated
+ # site links, and leave apps stopped. A subsequent corrected redeploy recovers.
+ container('sh', '-eu', '-c', '''
+printf 'server { listen 127.0.0.1:8099; return 200 "unrelated"; }\\n' > /etc/nginx/sites-available/unrelated
+ln -s /etc/nginx/sites-available/unrelated /etc/nginx/sites-enabled/unrelated
+printf 'invalid_regression_directive;\\n' > /etc/nginx/conf.d/regression-invalid.conf
+''')
+ invoke(REPO / 'playbooks/setup.yml', expect_failure=True)
+ assert 'active' not in states(), states()
+ assert container('test', '-L', '/etc/nginx/sites-enabled/unrelated', check=False).returncode == 0
+ container('rm', '/etc/nginx/conf.d/regression-invalid.conf')
+ invoke(REPO / 'playbooks/setup.yml')
+ assert_active()
+ assert container('test', '-L', '/etc/nginx/sites-enabled/unrelated', check=False).returncode == 0
+ assert container('cat', '/root/.ssh/config').stdout == ssh_before
+ print('PASS: failed deployment preserves sites and stops applications; redeploy recovers', flush=True)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test.sh b/test.sh
@@ -1,23 +1,32 @@
#!/bin/bash
-set -exuo pipefail
+set -euo pipefail
+export ANSIBLE_NOCOWS=1
repo_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
cd "$repo_dir"
-test_container=ansible-taler-test
+test_image=ansible-taler-test
+test_container=ansible-taler-test-$$
test_state_dir=$(mktemp -d)
cleanup() {
+ if [[ ${TALER_TEST_KEEP_CONTAINER:-0} == 1 ]]; then
+ echo "Kept test container $test_container and test files $test_state_dir"
+ return
+ fi
podman rm --force "$test_container" >/dev/null 2>&1 || true
rm -rf -- "${test_state_dir:?}"
}
trap cleanup EXIT
ssh-keygen -q -t ed25519 -N "" -f "$test_state_dir/id_ed25519"
-podman rm --force "$test_container" >/dev/null 2>&1 || true
+
+# Fast isolated regressions run before the complete deployment.
+python3 contrib/tests/test_backup.py
+python3 contrib/tests/test_upgrade_policy.py
# Build our image
-podman build -f Containerfile -t "$test_container"
+podman build -f Containerfile -t "$test_image"
# Run in background (-d) with systemd init. Taler's hardened systemd units
# require capabilities that Podman otherwise removes from the container.
@@ -27,7 +36,7 @@ podman run \
-p 127.0.0.1:8022:22 \
--systemd=always \
--privileged \
- -d "localhost/$test_container" sh -c "exec /usr/sbin/init --show-status"
+ -d "localhost/$test_image" sh -c "exec /usr/sbin/init --show-status"
# Use a disposable key because the deployment correctly disables SSH password
# authentication, and check mode runs in a separate SSH session afterwards.
@@ -66,8 +75,9 @@ ansible_args=(
)
# Provision, then prove that a separate check-mode run can inspect the result.
-ansible-playbook --verbose "${ansible_args[@]}"
-ansible-playbook --check --diff "${ansible_args[@]}"
+ansible-playbook --verbose --diff "${ansible_args[@]}" | tee "$test_state_dir/setup.log"
+python3 contrib/tests/test_deployment.py \
+ "$test_container" "$test_state_dir/id_ed25519" "$test_state_dir/setup.log"
# Basic smoke checks independent of Ansible's post-deployment checks.
podman exec "$test_container" systemctl is-active --quiet \
