commit df524c55ecc8d4cbae333069b86b97b8eb15e6c7
parent 0dedd91f6c4c3ff1e862c762c11f180497396355
Author: Florian Dold <dold@taler.net>
Date: Fri, 4 Sep 2026 19:13:59 +0200
database restore: require explicit recovery on fresh hosts
Move database restore into a dedicated playbook that requires one host
and an exact Borg archive. Validate postgres freshness, dump
compatibility, SQL errors, and restored exchange schemas before
reporting success. Keep normal deployments restore-free and document the
recovery workflow.
Diffstat:
10 files changed, 468 insertions(+), 136 deletions(-)
diff --git a/README b/README
@@ -24,11 +24,28 @@ directory.
### Main setup (restore.sh, deploy.sh)
-The "restore.sh" script extracts the latest database backup from the
-backup server. It should be run before the deploy.sh script to obtain
-the latest version of the database to be restored, unless you are
-literally setting up a service from scratch (which should be ultra-rare
-in production).
+Database restore is deliberately separate from normal deployment. It is a
+fresh-host recovery operation, not an idempotent part of `deploy.sh`, and it
+never chooses an archive implicitly. First list the Borg repository and
+select the exact archive to restore:
+
+```
+$ BORG_PASSPHRASE=... borg list ssh://borg@pixel.taler-systems.com/~/spec-backup
+$ BORG_PASSPHRASE=... ./restore.sh spec 'archive-name-copied-from-list'
+```
+
+The first argument must be one exact inventory hostname (not a group or host
+pattern). The destination PostgreSQL cluster must contain only the standard
+`postgres` database and role. The restore validates the compressed
+`root/postgres-backup.sql.gz` artifact, rejects dumps made by a newer
+PostgreSQL major version, restores all databases and roles, and verifies the
+exchange database before reporting success. It does not deploy or start the
+application; inspect the recovered host and then run `deploy.sh` separately.
+
+If SQL restore or verification fails, the partially restored cluster is
+considered tainted and must be reinitialized before another attempt. The
+temporary database dump is removed, while a private error log is retained in
+`/var/tmp/taler-postgres-restore-*.log` on the target for diagnosis.
The "deploy.sh" script deploys the latest version of a system on a host.
If you are root@rusty.taler-ops.ch, you may be able to:
@@ -51,6 +68,14 @@ Instead, if you had to edit them, re-encrypt them to all admins:
$ ./contrib/encrypt inventories/host_vars/spec/prod-secrets.yml
```
+### Staging offline master key
+
+The `test-master.priv` offline master key is intentionally committed to this
+repository and referenced by `stage-offline.conf`. It is a disposable test key
+for the staging exchange, where allowing repository users to sign staging
+configuration is intentional. It must never be used for production, for an
+exchange holding funds of value, or as the basis for any security assumption.
+
### sanction-check.sh
This command imports and checks the latest sanction lists:
@@ -229,9 +254,14 @@ Sets up Taler package repo and installs Taler packages.
### database
-Installs the Postgresql database and, if ENABLE_RESTORE_BACKUP is set and
-the target has no database yet, restores the snapshot fetched by
-restore.sh.
+Installs PostgreSQL and the dependencies needed to administer it through
+Ansible, then ensures the database service is running. Normal deployment
+never restores a backup.
+
+### database_restore
+
+Implements the guarded fresh-cluster restore used only by `playbooks/restore.yml`
+and `restore.sh`.
### devtesting
diff --git a/deploy.sh b/deploy.sh
@@ -12,9 +12,4 @@ ansible-playbook -v \
--limit "$1" \
playbooks/setup.yml
-if [ -f root/postgres-backup.sql.gz ]
-then
- echo "Remember to delete root/postgres-backup.sql.gz"
-fi
-
exit 0
diff --git a/inventories/group_vars/all/defaults.yml b/inventories/group_vars/all/defaults.yml
@@ -49,12 +49,3 @@ taler_repo_suites: "{{ ansible_facts['distribution_release'] }}"
# Use letsencrypt by default
exchange_use_letsencrypt: true
nexus_use_letsencrypt: true
-
-# Enable restore from backup? MUST be set to "false" in production,
-# unless restoring from backup.
-# Note that we do not restore backups if a database already exists at
-# the target server.
-# If no database exists on the target system and this option is 'true',
-# then a backup must have been provided at the originating host
-# (you get get it using the 'restore.sh' script).
-enable_restore_backup: false
diff --git a/playbooks/restore.yml b/playbooks/restore.yml
@@ -0,0 +1,43 @@
+---
+- name: Restore a database backup onto a fresh PostgreSQL host
+ hosts: all
+ any_errors_fatal: true
+ vars:
+ database_restore_archive: >-
+ {{ lookup('ansible.builtin.env', 'TALER_RESTORE_ARCHIVE') }}
+ database_restore_controller_directory: >-
+ {{ lookup('ansible.builtin.env', 'TALER_RESTORE_CONTROLLER_DIRECTORY') }}
+ database_restore_requested_target: >-
+ {{ lookup('ansible.builtin.env', 'TALER_RESTORE_TARGET') }}
+ pre_tasks:
+ - name: Reject check mode for a restore
+ ansible.builtin.assert:
+ that: not ansible_check_mode
+ fail_msg: Database restore cannot run in check mode.
+ quiet: true
+
+ - name: Require one exact inventory host
+ ansible.builtin.assert:
+ that:
+ - ansible_play_hosts_all | length == 1
+ - inventory_hostname == database_restore_requested_target
+ fail_msg: >-
+ Restore requires one exact inventory hostname, not a host group or
+ pattern.
+ quiet: true
+
+ - name: Validate controller restore inputs
+ ansible.builtin.assert:
+ that:
+ - database_restore_archive | length > 0
+ - database_restore_archive != 'latest'
+ - database_restore_archive != 'LATEST'
+ - database_restore_archive is match('^[A-Za-z0-9][A-Za-z0-9._:+-]*$')
+ - database_restore_controller_directory | length > 0
+ - lookup('ansible.builtin.env', 'BORG_PASSPHRASE') | length > 0
+ fail_msg: Run restore.sh with a passphrase, host, and exact archive.
+ quiet: true
+
+ roles:
+ - role: database
+ - role: database_restore
diff --git a/playbooks/setup.yml b/playbooks/setup.yml
@@ -3,6 +3,14 @@
hosts: all
any_errors_fatal: true
pre_tasks:
+ - name: Reject the removed in-deployment restore switch
+ ansible.builtin.assert:
+ that: not (enable_restore_backup | default(false) | bool)
+ fail_msg: >-
+ enable_restore_backup is no longer supported. Restore a fresh host
+ with restore.sh before running the normal deployment.
+ quiet: true
+
- name: "Fail if the deployment kind is not defined"
ansible.builtin.fail:
msg: "deployment_kind is not set; it selects the exchange_$KIND role"
diff --git a/restore.sh b/restore.sh
@@ -2,35 +2,75 @@
set -eu
+usage()
+{
+ echo "Usage: BORG_PASSPHRASE=... $0 <inventory-host> <archive-name>" >&2
+}
+
if [ -z "${BORG_PASSPHRASE:-}" ]
then
- echo "You must set the BORG_PASSPHRASE environment variable first!"
- echo "You can find it encrypted in admin-log.git, under the target host"
+ echo "BORG_PASSPHRASE must be set." >&2
+ echo "It can be found encrypted in admin-log.git under the target host." >&2
exit 1
fi
-if [ -z "${1:-}" ]
+if [ "$#" -ne 2 ]
then
- echo "Call with 'spec' or another host/group to select target"
+ usage
exit 1
fi
-TARGET="$1"
-HOSTNAME="pixel.taler-systems.com"
-echo "Restoring backup for $TARGET from $HOSTNAME"
+restore_target=$1
+restore_archive=$2
-REPO="ssh://borg@$HOSTNAME/~/$TARGET-backup"
+case "$restore_target" in
+ ''|*[!A-Za-z0-9._-]*)
+ echo "The target must be one exact inventory hostname." >&2
+ exit 1
+ ;;
+esac
-LATEST=$(borg list --last 1 --format '{archive}' "${REPO}")
+case "$restore_archive" in
+ latest|LATEST)
+ echo "Select an exact Borg archive; 'latest' is not accepted." >&2
+ exit 1
+ ;;
+ ''|*[!A-Za-z0-9._:+-]*)
+ echo "Invalid Borg archive name: $restore_archive" >&2
+ exit 1
+ ;;
+esac
-echo "Latest backup is $LATEST"
-if [ -z "${LATEST}" ]
-then
- echo "No backups found?"
- exit 1
-fi
+for restore_command in ansible-playbook borg gzip zgrep
+do
+ if ! command -v "$restore_command" >/dev/null 2>&1
+ then
+ echo "Required command not found: $restore_command" >&2
+ exit 1
+ fi
+done
+
+restore_workdir=$(mktemp -d "${TMPDIR:-/tmp}/taler-database-restore.XXXXXX")
+chmod 700 "$restore_workdir"
+
+cleanup()
+{
+ rm -f -- "$restore_workdir/postgres-backup.sql.gz"
+ rmdir -- "$restore_workdir" 2>/dev/null || true
+}
+trap cleanup EXIT
+trap 'exit 1' HUP INT TERM
+
+export TALER_RESTORE_TARGET="$restore_target"
+export TALER_RESTORE_ARCHIVE="$restore_archive"
+export TALER_RESTORE_CONTROLLER_DIRECTORY="$restore_workdir"
+
+echo "Restoring $restore_target from exact archive $restore_archive"
+
+ansible-playbook -v \
+ --inventory inventories/default \
+ --limit "$restore_target" \
+ playbooks/restore.yml
-borg extract \
- --list \
- "${REPO}::${LATEST}" \
- root/postgres-backup.sql.gz
+echo "Database restore and verification completed successfully."
+echo "Inspect the host, then deploy the application with: ./deploy.sh $restore_target"
diff --git a/roles/database/handlers/main.yml b/roles/database/handlers/main.yml
@@ -1,4 +0,0 @@
-- name: Restart postgresql
- service:
- name: postgresql
- state: restarted
diff --git a/roles/database/tasks/main.yml b/roles/database/tasks/main.yml
@@ -2,102 +2,17 @@
# Database role
- name: Install PostgreSQL on Debian/Ubuntu
- apt:
- name: postgresql
+ ansible.builtin.apt:
+ name:
+ - postgresql
+ - python3-psycopg2
+ - sudo
state: present
update_cache: true
- notify:
- - Restart postgresql
when: ansible_facts["os_family"] == 'Debian'
- name: Ensure PostgreSQL is started and enabled
- systemd:
+ ansible.builtin.systemd:
name: postgresql
state: started
enabled: true
-
-- name: Collect database information
- become: true
- become_user: postgres
- community.postgresql.postgresql_info:
- filter:
- - "databases*"
- register: database_info
-
-- name: Check if exchange database already exists
- become: true
- become_user: postgres
- ansible.builtin.set_fact:
- exchange_db_exists: "{{ 'taler-exchange' in database_info.databases.keys() }}"
-
-- name: Check if versioning schema exists
- become: true
- become_user: postgres
- community.postgresql.postgresql_query:
- login_user: postgres
- db: taler-exchange
- query:
- SELECT schema_name FROM information_schema.schemata WHERE schema_name = '_v';
- register: schema_check
- when: exchange_db_exists | bool
-
-- name: Set versioning schema existence fact
- ansible.builtin.set_fact:
- versioning_schema_exists: "{{ schema_check.rowcount | default(0) > 0 }}"
- when: exchange_db_exists | bool
-
-- name: Check if postgres backup file exists locally
- ansible.builtin.stat:
- path: "{{ role_path }}/files/postgres-backup.sql.gz"
- follow: yes
- delegate_to: localhost
- register: backup_file_status
-
-- name: Set local backup existence fact
- ansible.builtin.set_fact:
- local_backup_exists: "{{ backup_file_status.stat.exists | default(false) }}"
-
-- name: Fail if trying to import backup and versioning schema exists
- fail: msg="Backup for import provided, but _v schema exists on target host"
- when:
- - enable_restore_backup
- - versioning_schema_exists | default(false) | bool
- - local_backup_exists | bool
-
-# Without this the deploy would silently continue onto an empty database.
-- name: Fail if a restore was requested but no backup is available
- fail:
- msg: >-
- enable_restore_backup is set but
- {{ role_path }}/files/postgres-backup.sql.gz does not resolve to a
- file. Fetch the backup with restore.sh first.
- when:
- - enable_restore_backup
- - not (local_backup_exists | bool)
- - not (exchange_db_exists | bool)
-
-# Note: the postgres-backup.sql.gz is a symbolic link in Git.
-# The target of that symbolic link is created via the 'restore.sh' script.
-- name: Upload database backup file to server if restoring from backup
- copy:
- src: postgres-backup.sql.gz
- dest: /tmp/postgres-backup.sql.gz
- owner: postgres
- group: postgres
- mode: "0400"
- when:
- - enable_restore_backup
- - local_backup_exists | bool
-
-- name: Restore PostgreSQL database from backup
- become: true
- become_user: postgres
- shell: "gunzip -c /tmp/postgres-backup.sql.gz | psql -X -d postgres"
- when:
- - enable_restore_backup
- - local_backup_exists | bool
-
-- name: Remove backup from server (delete file)
- ansible.builtin.file:
- path: /tmp/postgres-backup.sql.gz
- state: absent
diff --git a/roles/database_restore/defaults/main.yml b/roles/database_restore/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+database_restore_borg_host: pixel.taler-systems.com
+database_restore_borg_repository: >-
+ ssh://borg@{{ database_restore_borg_host }}/~/{{ inventory_hostname }}-backup
+database_restore_archive_member: root/postgres-backup.sql.gz
diff --git a/roles/database_restore/tasks/main.yml b/roles/database_restore/tasks/main.yml
@@ -0,0 +1,309 @@
+---
+- name: Find non-system databases on the destination
+ become: true
+ become_user: postgres
+ community.postgresql.postgresql_query:
+ login_db: postgres
+ query: >-
+ SELECT datname
+ FROM pg_database
+ WHERE NOT datistemplate AND datname <> 'postgres'
+ ORDER BY datname
+ register: database_restore_existing_databases
+ changed_when: false
+
+- name: Find non-system roles on the destination
+ become: true
+ become_user: postgres
+ community.postgresql.postgresql_query:
+ login_db: postgres
+ query: >-
+ SELECT rolname
+ FROM pg_roles
+ WHERE rolname <> 'postgres' AND rolname !~ '^pg_'
+ ORDER BY rolname
+ register: database_restore_existing_roles
+ changed_when: false
+
+- name: Require a fresh PostgreSQL cluster
+ ansible.builtin.assert:
+ that:
+ - database_restore_existing_databases.rowcount == 0
+ - database_restore_existing_roles.rowcount == 0
+ fail_msg: >-
+ Refusing to restore into a non-fresh cluster. Non-system databases:
+ {{ database_restore_existing_databases.query_result | map(attribute='datname') | list }};
+ non-system roles:
+ {{ database_restore_existing_roles.query_result | map(attribute='rolname') | list }}.
+ Reinitialize the cluster before restoring.
+ quiet: true
+
+- name: Inspect the controller staging directory
+ ansible.builtin.stat:
+ path: "{{ database_restore_controller_directory }}"
+ delegate_to: localhost
+ register: database_restore_controller_directory_stat
+
+- name: Require a private controller staging directory
+ ansible.builtin.assert:
+ that:
+ - database_restore_controller_directory_stat.stat.exists
+ - database_restore_controller_directory_stat.stat.isdir
+ - database_restore_controller_directory_stat.stat.mode == '0700'
+ fail_msg: The controller restore staging directory is absent or not private.
+ quiet: true
+
+- name: Set restore working paths
+ ansible.builtin.set_fact:
+ database_restore_controller_dump_path: >-
+ {{ database_restore_controller_directory }}/postgres-backup.sql.gz
+ database_restore_succeeded: false
+
+- name: Extract, restore, and verify the database snapshot
+ block:
+ - name: Extract the selected database snapshot on the controller
+ ansible.builtin.shell:
+ cmd: >-
+ set -o pipefail &&
+ borg extract --stdout
+ ::{{ database_restore_archive | quote }}
+ {{ database_restore_archive_member | quote }}
+ > {{ database_restore_controller_dump_path | quote }}
+ executable: /bin/bash
+ environment:
+ BORG_REPO: "{{ database_restore_borg_repository }}"
+ BORG_PASSPHRASE: >-
+ {{ lookup('ansible.builtin.env', 'BORG_PASSPHRASE') }}
+ delegate_to: localhost
+ register: database_restore_borg_extract
+ changed_when: false
+ no_log: true
+
+ - name: Validate the compressed database snapshot
+ ansible.builtin.command:
+ argv:
+ - gzip
+ - --test
+ - "{{ database_restore_controller_dump_path }}"
+ delegate_to: localhost
+ changed_when: false
+
+ - name: Read the source PostgreSQL version from the dump header
+ ansible.builtin.command:
+ argv:
+ - zgrep
+ - -m
+ - '1'
+ - --
+ - ^-- Dumped from database version
+ - "{{ database_restore_controller_dump_path }}"
+ delegate_to: localhost
+ register: database_restore_dump_header
+ changed_when: false
+ failed_when: database_restore_dump_header.rc > 1
+
+ - name: Validate the dump header
+ ansible.builtin.assert:
+ that:
+ - database_restore_dump_header.rc == 0
+ - >-
+ database_restore_dump_header.stdout is
+ match('^-- Dumped from database version [0-9]+([.][0-9]+)*')
+ fail_msg: The snapshot does not contain a recognizable pg_dumpall header.
+ quiet: true
+
+ - name: Record the source PostgreSQL major version
+ ansible.builtin.set_fact:
+ database_restore_source_major: >-
+ {{ database_restore_dump_header.stdout | regex_search('[0-9]+') | int }}
+
+ - name: Read the destination PostgreSQL version
+ become: true
+ become_user: postgres
+ community.postgresql.postgresql_query:
+ login_db: postgres
+ query: SHOW server_version_num
+ register: database_restore_destination_version
+ changed_when: false
+
+ - name: Record the destination PostgreSQL major version
+ ansible.builtin.set_fact:
+ database_restore_destination_major: >-
+ {{ database_restore_destination_version.query_result[0].server_version_num | int // 10000 }}
+
+ - name: Reject a dump from a newer PostgreSQL major version
+ ansible.builtin.assert:
+ that:
+ - database_restore_source_major | int <= database_restore_destination_major | int
+ fail_msg: >-
+ PostgreSQL {{ database_restore_source_major }} dump cannot be safely
+ restored by PostgreSQL {{ database_restore_destination_major }}.
+ quiet: true
+
+ - name: Allocate the remote database snapshot file
+ ansible.builtin.tempfile:
+ state: file
+ path: /var/tmp
+ prefix: taler-postgres-restore-
+ suffix: .sql.gz
+ register: database_restore_remote_dump
+
+ - name: Upload the database snapshot
+ ansible.builtin.copy:
+ src: "{{ database_restore_controller_dump_path }}"
+ dest: "{{ database_restore_remote_dump.path }}"
+ owner: postgres
+ group: postgres
+ mode: '0400'
+ diff: false
+ no_log: true
+
+ - name: Allocate the remote restore error log
+ ansible.builtin.tempfile:
+ state: file
+ path: /var/tmp
+ prefix: taler-postgres-restore-
+ suffix: .log
+ register: database_restore_error_log
+
+ - name: Protect the remote restore error log
+ ansible.builtin.file:
+ path: "{{ database_restore_error_log.path }}"
+ owner: postgres
+ group: postgres
+ mode: '0600'
+
+ - name: Restore the PostgreSQL cluster snapshot
+ become: true
+ become_user: postgres
+ ansible.builtin.shell:
+ cmd: >-
+ set -o pipefail &&
+ gzip --decompress --stdout {{ database_restore_remote_dump.path | quote }} |
+ psql -X --dbname postgres
+ > /dev/null
+ 2> {{ database_restore_error_log.path | quote }}
+ executable: /bin/bash
+ environment:
+ LC_ALL: C
+ register: database_restore_psql
+ changed_when: true
+ failed_when: false
+ no_log: true
+
+ - name: Check the restore log for unexpected SQL errors
+ ansible.builtin.command:
+ argv:
+ - awk
+ - |-
+ /(ERROR|FATAL|PANIC):/ {
+ if ($0 ~ /ERROR:[[:space:]]+role "postgres" already exists[[:space:]]*$/)
+ next
+ unexpected = 1
+ }
+ END { exit unexpected }
+ - "{{ database_restore_error_log.path }}"
+ register: database_restore_error_check
+ changed_when: false
+ failed_when: false
+ no_log: true
+
+ - name: Require a clean restore result
+ ansible.builtin.assert:
+ that:
+ - database_restore_psql.rc == 0
+ - database_restore_error_check.rc == 0
+ fail_msg: >-
+ Database restore failed or emitted an unexpected SQL error. The
+ private log is preserved on the target at
+ {{ database_restore_error_log.path }}. The partially restored cluster
+ must be reinitialized before retrying.
+ quiet: true
+
+ - name: Find restored databases
+ become: true
+ become_user: postgres
+ community.postgresql.postgresql_query:
+ login_db: postgres
+ query: >-
+ SELECT datname
+ FROM pg_database
+ WHERE NOT datistemplate
+ ORDER BY datname
+ register: database_restore_restored_databases
+ changed_when: false
+
+ - name: Require the exchange database
+ ansible.builtin.assert:
+ that:
+ - >-
+ 'taler-exchange' in
+ (database_restore_restored_databases.query_result | map(attribute='datname') | list)
+ fail_msg: The restored snapshot does not contain the taler-exchange database.
+ quiet: true
+
+ - name: Find restored exchange schemas
+ become: true
+ become_user: postgres
+ community.postgresql.postgresql_query:
+ login_db: taler-exchange
+ query: >-
+ SELECT schema_name
+ FROM information_schema.schemata
+ WHERE schema_name IN ('exchange', '_v')
+ ORDER BY schema_name
+ register: database_restore_exchange_schemas
+ changed_when: false
+
+ - name: Require the exchange schemas
+ ansible.builtin.assert:
+ that:
+ - >-
+ 'exchange' in
+ (database_restore_exchange_schemas.query_result | map(attribute='schema_name') | list)
+ - >-
+ '_v' in
+ (database_restore_exchange_schemas.query_result | map(attribute='schema_name') | list)
+ fail_msg: The restored exchange database is missing required schemas.
+ quiet: true
+
+ - name: Verify connections to all restored databases
+ become: true
+ become_user: postgres
+ community.postgresql.postgresql_query:
+ login_db: "{{ item.datname }}"
+ query: SELECT 1
+ loop: "{{ database_restore_restored_databases.query_result }}"
+ loop_control:
+ label: "{{ item.datname }}"
+ changed_when: false
+
+ - name: Mark the restore as successful
+ ansible.builtin.set_fact:
+ database_restore_succeeded: true
+
+ always:
+ - name: Remove the remote database snapshot
+ ansible.builtin.file:
+ path: "{{ database_restore_remote_dump.path }}"
+ state: absent
+ when:
+ - database_restore_remote_dump is defined
+ - database_restore_remote_dump.path is defined
+ no_log: true
+
+ - name: Remove the controller database snapshot
+ ansible.builtin.file:
+ path: "{{ database_restore_controller_dump_path }}"
+ state: absent
+ delegate_to: localhost
+
+ - name: Remove the successful restore log
+ ansible.builtin.file:
+ path: "{{ database_restore_error_log.path }}"
+ state: absent
+ when:
+ - database_restore_succeeded | bool
+ - database_restore_error_log is defined
+ - database_restore_error_log.path is defined
+ no_log: true
