commit c416ef11cce47fb225d1f0de3490310748123111
parent 35e55f0716866aefb0e72de35f5db9bc38f9dcad
Author: Florian Dold <dold@taler.net>
Date: Sat, 5 Sep 2026 20:12:02 +0200
Backups: serialize snapshots and preserve repository settings
Share backup and restore repository defaults and use a dedicated SSH
configuration without replacing administrator settings. Lock the full
backup operation, clean up failed dumps, and skip retention after
archive warnings or failures.
Diffstat:
9 files changed, 270 insertions(+), 35 deletions(-)
diff --git a/README b/README
@@ -130,8 +130,8 @@ $ ./extract-borg-key.sh $DEPLOYMENT
```
The resulting SSH public key should be added to the borg-account
-of the host storing the backup. The playbook contains the target
-hostname!
+of the host storing the backup. The borg_host inventory setting selects the target
+hostname.
Once the SSH key is deployed and the backup has been initialized
server-side (see admin-logs/pixel/03-borg.txt), start the daily
@@ -329,3 +329,19 @@ $ ./contrib/test-fact-helpers.sh # the /bin helpers that generate local facts
```
Also run by the CI job in "contrib/ci/jobs/001-build".
+
+## Deployment safety and recovery
+
+Backup and restore share borg_host and borg_repo inventory defaults, retaining
+ssh://borg@pixel.taler-systems.com/~/spec-backup for existing installations. Set
+borg_repo in host variables to use another already provisioned repository.
+Existing database_restore_borg_repository and database_restore_borg_host overrides
+remain supported. No archives are moved automatically.
+
+The backup script uses /root/.ssh/borg-config and leaves /root/.ssh/config intact.
+A lock protects the entire backup operation. Install the updated backup script
+between backup runs: an already running older script does not acquire this lock. If cron, backup.sh or reboot.sh
+encounters another backup, it exits nonzero (script status 75) without touching
+that run's snapshot. Retry after the existing backup finishes; a reboot playbook
+aborts before rebooting on this failure. Pruning and compaction are skipped after
+an unsuccessful or warning-producing archive creation.
diff --git a/contrib/tests/test_backup.py b/contrib/tests/test_backup.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Run the rendered backup script against disposable data and command fixtures."""
+import json
+import os
+from pathlib import Path
+import shlex
+import shutil
+import signal
+import subprocess
+import tempfile
+import time
+import unittest
+
+from jinja2 import Environment
+
+REPO = Path(__file__).resolve().parents[2]
+SOURCE = REPO / 'roles/borg-start/templates/root/bin/borg-backup.sh'
+
+
+class BackupTest(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory(prefix='taler-backup-test-')
+ self.addCleanup(self.temp.cleanup)
+ self.base = Path(self.temp.name)
+ self.root = self.base / 'root'
+ self.root.mkdir()
+ self.bin = self.base / 'bin'
+ self.bin.mkdir()
+ self.env = dict(os.environ, PATH=f'{self.bin}:{os.environ["PATH"]}',
+ TEST_BASE=str(self.base), TEST_ROOT=str(self.root))
+ self.executable('sudo', '''#!/bin/sh
+printf '%s\\n' '-- synthetic database snapshot'
+exit "${DUMP_EXIT:-0}"
+''')
+ self.executable('borg', '''#!/usr/bin/env python3
+import json, os, pathlib, subprocess, sys, time
+base = pathlib.Path(os.environ['TEST_BASE'])
+command = sys.argv[1]
+with (base / 'calls').open('a') as out:
+ out.write(command + '\\n')
+if command == 'create':
+ (base / 'entered').touch()
+ while os.environ.get('BLOCK_CREATE') and not (base / 'release').exists():
+ time.sleep(.02)
+ assert (base / 'root/postgres-backup.sql.gz').exists()
+ (base / 'passphrase').write_text(os.environ['BORG_PASSPHRASE'])
+if os.environ.get('REAL_BORG'):
+ args = [a.lstrip('/') if a in ['/root', '/etc', '/var/lib/libeufin-nexus', '/var/lib/taler-exchange'] else a for a in sys.argv[1:]]
+ sys.exit(subprocess.call([os.environ['REAL_BORG'], *args], cwd=base))
+sys.exit(int(os.environ.get(command.upper() + '_EXIT', '0')))
+''')
+ env = Environment()
+ env.filters['quote'] = shlex.quote
+ self.passphrase = "test passphrase with ' and $() characters"
+ text = env.from_string(SOURCE.read_text()).render(
+ borg_repo=str(self.base / 'repository'), borg_passphrase=self.passphrase)
+ # Only filesystem locations are redirected; the locking, command ordering,
+ # error handling, compression, and cleanup are the production script.
+ text = text.replace('/run/taler-borg-backup.lock', str(self.base / 'lock'))
+ text = text.replace('cd /root', 'cd "$TEST_ROOT"')
+ self.script = self.base / 'backup.sh'
+ self.script.write_text(text)
+
+ def executable(self, name, contents):
+ path = self.bin / name
+ path.write_text(contents)
+ path.chmod(0o700)
+
+ def run_backup(self, **env):
+ return subprocess.run(['bash', str(self.script)], env=dict(self.env, **env),
+ capture_output=True, timeout=20)
+
+ def calls(self):
+ p = self.base / 'calls'
+ return p.read_text().splitlines() if p.exists() else []
+
+ def assert_clean(self):
+ self.assertFalse((self.root / 'postgres-backup.sql').exists())
+ self.assertFalse((self.root / 'postgres-backup.sql.gz').exists())
+
+ def blocked_backup(self):
+ p = subprocess.Popen(['bash', str(self.script)], env=dict(self.env, BLOCK_CREATE='1'),
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+ start_new_session=True)
+ self.addCleanup(lambda: self.stop_process(p))
+ deadline = time.monotonic() + 10
+ while not (self.base / 'entered').exists():
+ if p.poll() is not None or time.monotonic() > deadline:
+ self.fail('backup never reached archive creation')
+ time.sleep(.02)
+ return p
+
+ @staticmethod
+ def stop_process(p):
+ if p.poll() is None:
+ os.killpg(p.pid, signal.SIGTERM)
+ p.wait(timeout=5)
+
+ @unittest.skipUnless(shutil.which('shellcheck'), 'ShellCheck is not installed')
+ def test_rendered_script_shellcheck(self):
+ result = subprocess.run(['shellcheck', '-S', 'warning', str(self.script)],
+ capture_output=True, text=True)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+ def test_success_and_shell_quoting(self):
+ result = self.run_backup()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(self.calls(), ['create', 'prune', 'compact'])
+ self.assertEqual((self.base / 'passphrase').read_text(), self.passphrase)
+ self.assertNotIn(self.passphrase.encode(), result.stdout + result.stderr)
+ self.assert_clean()
+
+ def test_competing_run_cannot_remove_snapshot(self):
+ first = self.blocked_backup()
+ before = (self.root / 'postgres-backup.sql.gz').read_bytes()
+ second = self.run_backup()
+ self.assertEqual(second.returncode, 75)
+ self.assertEqual((self.root / 'postgres-backup.sql.gz').read_bytes(), before)
+ self.assertEqual(self.calls(), ['create'])
+ (self.base / 'release').touch()
+ self.assertEqual(first.wait(timeout=10), 0)
+ self.assert_clean()
+
+ def test_dump_failure(self):
+ self.assertNotEqual(self.run_backup(DUMP_EXIT='1').returncode, 0)
+ self.assertEqual(self.calls(), [])
+ self.assert_clean()
+
+ def test_compression_failure(self):
+ self.executable('gzip', '#!/bin/sh\nexit 1\n')
+ self.assertNotEqual(self.run_backup().returncode, 0)
+ self.assertEqual(self.calls(), [])
+ self.assert_clean()
+
+ def test_create_warning_and_failure_skip_retention(self):
+ for code in [1, 2]:
+ with self.subTest(code=code):
+ (self.base / 'calls').unlink(missing_ok=True)
+ self.assertEqual(self.run_backup(CREATE_EXIT=str(code)).returncode, code)
+ self.assertEqual(self.calls(), ['create'])
+ self.assert_clean()
+
+ def test_prune_failure_skips_compaction(self):
+ self.assertEqual(self.run_backup(PRUNE_EXIT='2').returncode, 2)
+ self.assertEqual(self.calls(), ['create', 'prune'])
+ self.assert_clean()
+
+ def test_compaction_failure_is_reported(self):
+ self.assertEqual(self.run_backup(COMPACT_EXIT='2').returncode, 2)
+ self.assert_clean()
+
+ def test_interrupt_cleans_snapshot_and_releases_lock(self):
+ first = self.blocked_backup()
+ os.killpg(first.pid, signal.SIGTERM)
+ self.assertEqual(first.wait(timeout=10), 2)
+ self.assert_clean()
+ self.assertEqual(self.run_backup().returncode, 0)
+
+ @unittest.skipUnless(shutil.which('borg'), 'Borg required for archive round trip')
+ def test_real_archive_contains_restore_member(self):
+ borg = shutil.which('borg')
+ for directory in ['etc', 'var/lib/libeufin-nexus', 'var/lib/taler-exchange']:
+ (self.base / directory).mkdir(parents=True, exist_ok=True)
+ repo = str(self.base / 'repository')
+ env = dict(self.env, BORG_REPO=repo, BORG_PASSPHRASE=self.passphrase)
+ subprocess.run([borg, 'init', '--encryption=repokey'], env=env, check=True,
+ capture_output=True)
+ result = self.run_backup(REAL_BORG=borg)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ archives = json.loads(subprocess.check_output([borg, 'list', '--json'], env=env))
+ archive = archives['archives'][0]['name']
+ dump = subprocess.check_output([borg, 'extract', '--stdout', f'::{archive}',
+ 'root/postgres-backup.sql.gz'], env=env)
+ import gzip
+ self.assertEqual(gzip.decompress(dump), b'-- synthetic database snapshot\n')
+ self.assert_clean()
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)
diff --git a/inventories/group_vars/all/defaults.yml b/inventories/group_vars/all/defaults.yml
@@ -49,3 +49,8 @@ taler_repo_suites: "{{ ansible_facts['distribution_release'] }}"
# Use letsencrypt by default
exchange_use_letsencrypt: true
nexus_use_letsencrypt: true
+
+# Shared backup/restore location. Preserve the repository used by existing
+# backup scripts; override borg_repo per host for independently provisioned repos.
+borg_host: pixel.taler-systems.com
+borg_repo: "ssh://borg@{{ borg_host }}/~/spec-backup"
diff --git a/playbooks/borg-ssh-export.yml b/playbooks/borg-ssh-export.yml
@@ -3,6 +3,3 @@
hosts: all
roles:
- borg-ssh-export
- vars:
- # Hostname where we will store backups
- borg_host: pixel.taler-systems.com
diff --git a/playbooks/borg-start.yml b/playbooks/borg-start.yml
@@ -4,8 +4,3 @@
any_errors_fatal: true
roles:
- borg-start
- vars:
- # Hostname where we will store backups
- borg_host: pixel.taler-systems.com
- # Target for the backup (repo must exist and we must have SSH access).
- borg_repo: "ssh://borg@{{ borg_host }}/~/spec-backup"
diff --git a/roles/borg-ssh-export/tasks/main.yml b/roles/borg-ssh-export/tasks/main.yml
@@ -15,7 +15,7 @@
state: directory
owner: root
group: root
- mode: "0744"
+ mode: "0700"
- name: Create SSH key pair for use for backups by root
ansible.builtin.command:
diff --git a/roles/borg-start/tasks/main.yml b/roles/borg-start/tasks/main.yml
@@ -1,4 +1,29 @@
---
+- name: Install backup prerequisites
+ ansible.builtin.apt:
+ name: [borgbackup, gzip, cron, util-linux, sudo]
+ state: present
+ update_cache: true
+ policy_rc_d: 101
+
+- name: Ensure the private SSH directory exists
+ ansible.builtin.file:
+ path: /root/.ssh
+ state: directory
+ owner: root
+ group: root
+ mode: "0700"
+
+- name: Check SSH key for backups exists
+ stat:
+ path: "/root/.ssh/borg"
+ register: have_ssh_key
+
+- name: Fail if we do not have an SSH key for the backup server
+ fail:
+ msg: "You need to first run extract-borg-key.sh"
+ when: not have_ssh_key.stat.exists
+
- name: Ensure /root/bin/ directory exists
file:
path: "/root/bin/"
@@ -14,16 +39,13 @@
owner: root
group: root
mode: "0700"
-
-- name: Check SSH key for backups exists
- stat:
- path: "/root/.ssh/borg"
- register: have_ssh_key
+ no_log: true
+ diff: false
- name: Place ssh configuration
ansible.builtin.template:
src: templates/root/.ssh/config
- dest: /root/.ssh/config
+ dest: /root/.ssh/borg-config
owner: root
group: root
mode: "0600"
@@ -40,14 +62,15 @@
cmd: ssh-keyscan {{ borg_host }} >> /root/.ssh/known_hosts
when: known_host.rc != 0
-- name: Fail if we do not have an SSH key for the backup server
- fail:
- msg: "You need to first run extract-borg-key.sh"
- when: not have_ssh_key.stat.exists
-
- name: Create cron job to run daily backups
ansible.builtin.cron:
name: "perform backup"
minute: "43"
hour: "2"
job: "/root/bin/borg-backup.sh"
+
+- name: Enable the cron service
+ ansible.builtin.systemd:
+ name: cron
+ state: started
+ enabled: true
diff --git a/roles/borg-start/templates/root/bin/borg-backup.sh b/roles/borg-start/templates/root/bin/borg-backup.sh
@@ -1,16 +1,30 @@
#!/bin/bash
-export BORG_REPO='{{ borg_repo }}'
-export BORG_PASSPHRASE='{{ borg_passphrase }}'
-
-# some helpers and error handling:
-info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; }
-trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM
-
-cd /root
+export BORG_REPO={{ borg_repo | quote }}
+export BORG_PASSPHRASE={{ borg_passphrase | quote }}
+export BORG_RSH='ssh -F /root/.ssh/borg-config'
+
+umask 077
+info() { printf "\n%s %s\n\n" "$(date)" "$*" >&2; }
+
+# Acquire the lock before changing any dump files or installing cleanup traps.
+# A competing cron/manual/reboot run must not touch the owner's snapshot.
+exec 9>/run/taler-borg-backup.lock || exit 2
+if ! flock --exclusive --nonblock 9; then
+ info "Another backup is running; this run made no changes"
+ exit 75
+fi
+cd /root || exit 2
+cleanup() {
+ rm -f -- postgres-backup.sql postgres-backup.sql.gz
+}
+trap cleanup EXIT
+trap 'exit 2' HUP INT TERM
info "Dumping database"
+# The root-owned output is intentionally opened by this shell.
+# shellcheck disable=SC2024
sudo -u postgres pg_dumpall > postgres-backup.sql
db_exit=$?
@@ -56,10 +70,10 @@ backup_exit=$?
info "Removing database dump"
-rm postgres-backup.sql.gz
+rm -f -- postgres-backup.sql.gz || exit 2
-if [[ $backup_exit -gt 1 ]]; then
- info "Backup failed, exit status $backup_exit; not pruning"
+if [[ $backup_exit -ne 0 ]]; then
+ info "Backup did not complete cleanly, exit status $backup_exit; not pruning"
exit $backup_exit
fi
@@ -79,6 +93,10 @@ borg prune \
--keep-monthly 6
prune_exit=$?
+if [[ $prune_exit -ne 0 ]]; then
+ info "Prune did not complete cleanly, exit status $prune_exit; not compacting"
+ exit "$prune_exit"
+fi
# actually free repo disk space by compacting segments
diff --git a/roles/database_restore/defaults/main.yml b/roles/database_restore/defaults/main.yml
@@ -1,5 +1,6 @@
---
-database_restore_borg_host: pixel.taler-systems.com
+database_restore_borg_host: "{{ borg_host }}"
database_restore_borg_repository: >-
- ssh://borg@{{ database_restore_borg_host }}/~/{{ inventory_hostname }}-backup
+ {{ borg_repo | replace('ssh://borg@' ~ borg_host ~ '/',
+ 'ssh://borg@' ~ database_restore_borg_host ~ '/') }}
database_restore_archive_member: root/postgres-backup.sql.gz
