test_backup.py (7785B)
1 #!/usr/bin/env python3 2 """Run the rendered backup script against disposable data and command fixtures.""" 3 import json 4 import os 5 from pathlib import Path 6 import shlex 7 import shutil 8 import signal 9 import subprocess 10 import tempfile 11 import time 12 import unittest 13 14 from jinja2 import Environment 15 16 REPO = Path(__file__).resolve().parents[2] 17 SOURCE = REPO / 'roles/borg-start/templates/root/bin/borg-backup.sh' 18 19 20 class BackupTest(unittest.TestCase): 21 def setUp(self): 22 self.temp = tempfile.TemporaryDirectory(prefix='taler-backup-test-') 23 self.addCleanup(self.temp.cleanup) 24 self.base = Path(self.temp.name) 25 self.root = self.base / 'root' 26 self.root.mkdir() 27 self.bin = self.base / 'bin' 28 self.bin.mkdir() 29 self.env = dict(os.environ, PATH=f'{self.bin}:{os.environ["PATH"]}', 30 TEST_BASE=str(self.base), TEST_ROOT=str(self.root)) 31 self.executable('sudo', '''#!/bin/sh 32 printf '%s\\n' '-- synthetic database snapshot' 33 exit "${DUMP_EXIT:-0}" 34 ''') 35 self.executable('borg', '''#!/usr/bin/env python3 36 import json, os, pathlib, subprocess, sys, time 37 base = pathlib.Path(os.environ['TEST_BASE']) 38 command = sys.argv[1] 39 with (base / 'calls').open('a') as out: 40 out.write(command + '\\n') 41 if command == 'create': 42 (base / 'entered').touch() 43 while os.environ.get('BLOCK_CREATE') and not (base / 'release').exists(): 44 time.sleep(.02) 45 assert (base / 'root/postgres-backup.sql.gz').exists() 46 (base / 'passphrase').write_text(os.environ['BORG_PASSPHRASE']) 47 if os.environ.get('REAL_BORG'): 48 args = [a.lstrip('/') if a in ['/root', '/etc', '/var/lib/libeufin-nexus', '/var/lib/taler-exchange'] else a for a in sys.argv[1:]] 49 sys.exit(subprocess.call([os.environ['REAL_BORG'], *args], cwd=base)) 50 sys.exit(int(os.environ.get(command.upper() + '_EXIT', '0'))) 51 ''') 52 env = Environment() 53 env.filters['quote'] = shlex.quote 54 self.passphrase = "test passphrase with ' and $() characters" 55 text = env.from_string(SOURCE.read_text()).render( 56 borg_repo=str(self.base / 'repository'), borg_passphrase=self.passphrase) 57 # Only filesystem locations are redirected; the locking, command ordering, 58 # error handling, compression, and cleanup are the production script. 59 text = text.replace('/run/taler-borg-backup.lock', str(self.base / 'lock')) 60 text = text.replace('cd /root', 'cd "$TEST_ROOT"') 61 self.script = self.base / 'backup.sh' 62 self.script.write_text(text) 63 64 def executable(self, name, contents): 65 path = self.bin / name 66 path.write_text(contents) 67 path.chmod(0o700) 68 69 def run_backup(self, **env): 70 return subprocess.run(['bash', str(self.script)], env=dict(self.env, **env), 71 capture_output=True, timeout=20) 72 73 def calls(self): 74 p = self.base / 'calls' 75 return p.read_text().splitlines() if p.exists() else [] 76 77 def assert_clean(self): 78 self.assertFalse((self.root / 'postgres-backup.sql').exists()) 79 self.assertFalse((self.root / 'postgres-backup.sql.gz').exists()) 80 81 def blocked_backup(self): 82 p = subprocess.Popen(['bash', str(self.script)], env=dict(self.env, BLOCK_CREATE='1'), 83 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, 84 start_new_session=True) 85 self.addCleanup(lambda: self.stop_process(p)) 86 deadline = time.monotonic() + 10 87 while not (self.base / 'entered').exists(): 88 if p.poll() is not None or time.monotonic() > deadline: 89 self.fail('backup never reached archive creation') 90 time.sleep(.02) 91 return p 92 93 @staticmethod 94 def stop_process(p): 95 if p.poll() is None: 96 os.killpg(p.pid, signal.SIGTERM) 97 p.wait(timeout=5) 98 99 @unittest.skipUnless(shutil.which('shellcheck'), 'ShellCheck is not installed') 100 def test_rendered_script_shellcheck(self): 101 result = subprocess.run(['shellcheck', '-S', 'warning', str(self.script)], 102 capture_output=True, text=True) 103 self.assertEqual(result.returncode, 0, result.stdout + result.stderr) 104 105 def test_success_and_shell_quoting(self): 106 result = self.run_backup() 107 self.assertEqual(result.returncode, 0, result.stderr) 108 self.assertEqual(self.calls(), ['create', 'prune', 'compact']) 109 self.assertEqual((self.base / 'passphrase').read_text(), self.passphrase) 110 self.assertNotIn(self.passphrase.encode(), result.stdout + result.stderr) 111 self.assert_clean() 112 113 def test_competing_run_cannot_remove_snapshot(self): 114 first = self.blocked_backup() 115 before = (self.root / 'postgres-backup.sql.gz').read_bytes() 116 second = self.run_backup() 117 self.assertEqual(second.returncode, 75) 118 self.assertEqual((self.root / 'postgres-backup.sql.gz').read_bytes(), before) 119 self.assertEqual(self.calls(), ['create']) 120 (self.base / 'release').touch() 121 self.assertEqual(first.wait(timeout=10), 0) 122 self.assert_clean() 123 124 def test_dump_failure(self): 125 self.assertNotEqual(self.run_backup(DUMP_EXIT='1').returncode, 0) 126 self.assertEqual(self.calls(), []) 127 self.assert_clean() 128 129 def test_compression_failure(self): 130 self.executable('gzip', '#!/bin/sh\nexit 1\n') 131 self.assertNotEqual(self.run_backup().returncode, 0) 132 self.assertEqual(self.calls(), []) 133 self.assert_clean() 134 135 def test_create_warning_and_failure_skip_retention(self): 136 for code in [1, 2]: 137 with self.subTest(code=code): 138 (self.base / 'calls').unlink(missing_ok=True) 139 self.assertEqual(self.run_backup(CREATE_EXIT=str(code)).returncode, code) 140 self.assertEqual(self.calls(), ['create']) 141 self.assert_clean() 142 143 def test_prune_failure_skips_compaction(self): 144 self.assertEqual(self.run_backup(PRUNE_EXIT='2').returncode, 2) 145 self.assertEqual(self.calls(), ['create', 'prune']) 146 self.assert_clean() 147 148 def test_compaction_failure_is_reported(self): 149 self.assertEqual(self.run_backup(COMPACT_EXIT='2').returncode, 2) 150 self.assert_clean() 151 152 def test_interrupt_cleans_snapshot_and_releases_lock(self): 153 first = self.blocked_backup() 154 os.killpg(first.pid, signal.SIGTERM) 155 self.assertEqual(first.wait(timeout=10), 2) 156 self.assert_clean() 157 self.assertEqual(self.run_backup().returncode, 0) 158 159 @unittest.skipUnless(shutil.which('borg'), 'Borg required for archive round trip') 160 def test_real_archive_contains_restore_member(self): 161 borg = shutil.which('borg') 162 for directory in ['etc', 'var/lib/libeufin-nexus', 'var/lib/taler-exchange']: 163 (self.base / directory).mkdir(parents=True, exist_ok=True) 164 repo = str(self.base / 'repository') 165 env = dict(self.env, BORG_REPO=repo, BORG_PASSPHRASE=self.passphrase) 166 subprocess.run([borg, 'init', '--encryption=repokey'], env=env, check=True, 167 capture_output=True) 168 result = self.run_backup(REAL_BORG=borg) 169 self.assertEqual(result.returncode, 0, result.stderr) 170 archives = json.loads(subprocess.check_output([borg, 'list', '--json'], env=env)) 171 archive = archives['archives'][0]['name'] 172 dump = subprocess.check_output([borg, 'extract', '--stdout', f'::{archive}', 173 'root/postgres-backup.sql.gz'], env=env) 174 import gzip 175 self.assertEqual(gzip.decompress(dump), b'-- synthetic database snapshot\n') 176 self.assert_clean() 177 178 179 if __name__ == '__main__': 180 unittest.main(verbosity=2)