diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 36d35f8..0be0162 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -7,3 +7,10 @@ root_domain: contrapeso.xyz # playbooks are kept deliberately — the check logic is meant to be rewired to # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. uptime_kuma_enabled: false + +# age recipient for all backup artefacts +age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" + +# Public key small-backups-box pulls with +# Authorised on each source host for an unprivileged, dedicated user only +backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml index 2ff6994..1072b51 100644 --- a/ansible/group_vars/nodito_vms.yml +++ b/ansible/group_vars/nodito_vms.yml @@ -2,16 +2,23 @@ # Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. # # ansible_host is a MagicDNS name. If tailscaled is not running on the control -# node that name does not resolve, so `nc %h %p` fails fast and the second nc -# takes over on the LAN address recorded as lan_ip in inventory.ini. +# node that name does not resolve, the probe fails, and the LAN address recorded +# as lan_ip in inventory.ini takes over. # -# This is safe against the LAN addresses drifting again (which is how -# fulcrum/mempool came to be transposed): known_hosts is keyed to the MagicDNS -# NAME, so if lan_ip ever points at a different machine the host key will not -# match and ssh aborts. Verified 2026-09-12 by pointing fulcrum-box at -# mempool-box's address: "Host key verification failed." +# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`: +# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a +# single `nc -w5` silently tears down the SSH session after five quiet seconds. +# That produced intermittent "Data could not be sent to remote host" failures on +# exactly the long, quiet operations (apt) where a dropped connection costs most. +# `nc -z` probes, then `exec nc` carries the session with no timeout at all. +# +# Safe against the LAN addresses drifting again (which is how fulcrum/mempool +# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if +# lan_ip ever points at a different machine the host key will not match and ssh +# aborts. Verified by pointing fulcrum-box at mempool-box's address: +# "Host key verification failed." # # lan_ip is a convenience, not an identity. If it goes stale the fallback stops # working; it will never connect you to the wrong box. ansible_ssh_common_args: >- - -o ProxyCommand="sh -c 'nc -w2 %h %p 2>/dev/null || nc -w4 {{ lan_ip }} %p'" + -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'" diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 842942e..0686ab6 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -14,3 +14,15 @@ - name: headscale source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" retention_days: 90 + - name: memos + source: "backup-pull@memos-box:/opt/backups/memos/" + retention_days: 90 + - name: vaultwarden + source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/" + retention_days: 90 + - name: lnbits + source: "backup-pull@prd-vipy:/opt/backups/lnbits/" + retention_days: 90 + - name: forgejo + source: "backup-pull@prd-vipy:/opt/backups/forgejo/" + retention_days: 14 diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md index 065e1fc..a7794ab 100644 --- a/ansible/roles/backup_source/README.md +++ b/ansible/roles/backup_source/README.md @@ -39,6 +39,56 @@ The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` with relative paths in `tar` rather than absolute ones: it avoids tar's "removing leading /" and makes the restore target explicit. +## Services that are not systemd + +`backup_source_stop_service` runs `systemctl stop/start`. For anything else, +give the pair explicitly — vaultwarden is a docker compose stack, so +`systemctl stop vaultwarden` silently does nothing: + +```yaml +backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" +backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" +``` + +The same EXIT trap wraps both forms. The assert refuses a stop command without a +matching start command, because that combination fails in the one way you would +not notice: the service stops and never comes back. + +## More than one thing to back up + +`tar` takes several paths, so multiple files or directories are normally **one** +artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together, +lnbits captures its data directory and its `.env`. + +Prefer one artefact. A backup should be a consistent snapshot, and two artefacts +written by two runs can drift — you can end up restoring an `.env` that does not +match the database it configures. Pulling a single file back out needs no +unpacking: + +```bash +age -d -i | tar -xzO opt/lnbits/lnbits/.env +``` + +If you genuinely need separate artefacts, call the role twice with different +`backup_source_name`s rather than extending it — but only one call may set +`backup_source_stop_service`, or the service is stopped twice per night. + +The case this shape cannot express is a **database dump plus a file tree** +(`pg_dump` and a media directory, say): you cannot merge those into one stream +without staging plaintext on disk, which is exactly what this design avoids. +None of the current services need it — all are file trees, all stopped for the +dump. A future one that does should use two role calls. + +## Everything here is sqlite, so everything stops + +All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files +present). A live copy of a WAL-mode database can be torn or stale, so each is +stopped for the duration. Measured downtime: under a second for headscale and +memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos +and database. That last one is the real cost of a consistent snapshot; if it +becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy, +not skipping the stop. + ## The trap is the reason this role exists When `backup_source_stop_service` is set, the script stops the unit and installs diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml index 77d17e0..7de6006 100644 --- a/ansible/roles/backup_source/defaults/main.yml +++ b/ansible/roles/backup_source/defaults/main.yml @@ -17,8 +17,12 @@ backup_source_recipient: "{{ age_backup_recipient }}" backup_source_pull_user: backup-pull backup_source_pull_key: "{{ backup_pull_public_key }}" -# Safety -backup_source_stop_service: "" # local unit stopped for the dump, restored by a trap +# Safety. Give either a systemd unit, or an explicit pair of commands for +# services that are not systemd-managed (vaultwarden is a docker compose stack). +# Whichever is used, a trap guarantees the restart. +backup_source_stop_service: "" # systemd unit stopped for the dump +backup_source_stop_command: "" # overrides stop_service when set +backup_source_start_command: "" # required alongside stop_command # Retention here is LOCAL and short; small-backups-box keeps the long tail. backup_source_retention_days: 7 diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index 54d6ac1..4283b84 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -7,9 +7,11 @@ - backup_source_dump_command | length > 0 - backup_source_recipient | length > 0 - backup_source_recipient is match('^age1[0-9a-z]{58}$') + - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0) fail_msg: >- backup_source: '{{ backup_source_name | default("") }}' needs a name, description, dump command and a valid age recipient (age1... 62 chars). + backup_source_stop_command must be paired with backup_source_start_command. quiet: true # Declared here rather than assumed. Stage 1 installed it by hand; this is what diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index 35e4398..18fabd6 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -11,8 +11,10 @@ RETENTION_DAYS={{ backup_source_retention_days }} RECIPIENT="{{ backup_source_recipient }}" SUFFIX="{{ backup_source_artifact_suffix }}" NAME="{{ backup_source_name }}" -{% if backup_source_stop_service %} -SERVICE="{{ backup_source_stop_service }}" +{% if backup_source_stop_service or backup_source_stop_command %} +STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }} +START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }} +SERVICE="{{ backup_source_stop_service or backup_source_description }}" # label for the log only {% endif %} TIMESTAMP=$(date +%Y%m%d_%H%M%S) @@ -39,13 +41,13 @@ chmod 700 "$BACKUP_DIR" # here or they accumulate forever. rm -f "${BACKUP_DIR}/${NAME}_"*.partial -{% if backup_source_stop_service %} +{% if backup_source_stop_service or backup_source_stop_command %} # --- Stop the service, and guarantee it comes back --- # The trap is the point: without it a failed dump leaves the service down until # the next timer fires. Every hand-written script this replaced had that bug. log "Stopping ${SERVICE}..." -systemctl stop "$SERVICE" -trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT +eval "$STOP_CMD" +trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT {% endif %} # --- Dump straight into age; plaintext never touches the disk --- diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml new file mode 100644 index 0000000..d6aafef --- /dev/null +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -0,0 +1,25 @@ +--- +# Forgejo backup: dumps locally on vipy, encrypted with age. +# +# The biggest artefact in the estate (~2.7 G) and the reason retention here is +# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14. +# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the +# old job did the same. +- name: Configure the Forgejo backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./forgejo_vars.yml + + tasks: + - name: Ensure Forgejo dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: forgejo + backup_source_description: "Forgejo" + backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo" + backup_source_stop_service: forgejo + backup_source_retention_days: 2 + backup_source_on_calendar: "*-*-* 02:30:00" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml new file mode 100644 index 0000000..46abf31 --- /dev/null +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -0,0 +1,25 @@ +--- +# LNBits backup: dumps locally on vipy, encrypted with age. +# The old job produced TWO gpg artefacts (data, then .env separately). They are +# folded into one tar here so the wallet database and the .env that configures +# it are always the same point in time; two artefacts written by two runs can +# drift. Pulling one file back out needs no unpacking: +# age -d -i | tar -xzO opt/lnbits/lnbits/.env +- name: Configure the LNBits backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./lnbits_vars.yml + + tasks: + - name: Ensure LNBits dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: lnbits + backup_source_description: "LNBits" + backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env" + backup_source_stop_service: lnbits + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:20:00" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml new file mode 100644 index 0000000..2131924 --- /dev/null +++ b/ansible/services/memos/setup_backup_memos.yml @@ -0,0 +1,26 @@ +--- +# Memos backup: dumps locally on memos-box, encrypted with age. +# Replaces the lapy pull, which had been writing EMPTY directories since +# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned +# to a different machine that has no rsync. +- name: Configure the Memos backup on its own host + hosts: memos + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./memos_vars.yml + + tasks: + - name: Ensure Memos dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: memos + backup_source_description: "Memos" + backup_source_dump_command: "tar -czf - -C / var/lib/memos" + # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a + # consistent database rather than a torn mid-write copy. The old rsync + # job did not stop it. + backup_source_stop_service: memos + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml new file mode 100644 index 0000000..8430475 --- /dev/null +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -0,0 +1,25 @@ +--- +# Vaultwarden backup: dumps locally on vipy, encrypted with age. +# Previously rsynced to lapy in the CLEAR; the artefact now never exists +# unencrypted, on disk or on the wire. +- name: Configure the Vaultwarden backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./vaultwarden_vars.yml + + tasks: + - name: Ensure Vaultwarden dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: vaultwarden + backup_source_description: "Vaultwarden" + backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data" + # Not systemd — a docker compose stack — so stop/start explicitly. + # sqlite in WAL mode, hence stopping at all. + backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" + backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:10:00"