age backups everywhere

ansible/group_vars/all/main.yml

@@ -7,3 +7,10 @@ root_domain: contrapeso.xyz
 # playbooks are kept deliberately — the check logic is meant to be rewired to
 # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
 uptime_kuma_enabled: false
+
+# age recipient for all backup artefacts
+age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza"
+
+# Public key small-backups-box pulls with
+# Authorised on each source host for an unprivileged, dedicated user only
+backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull"

ansible/group_vars/nodito_vms.yml

@@ -2,16 +2,23 @@
 # Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down.
 #
 # ansible_host is a MagicDNS name. If tailscaled is not running on the control
-# node that name does not resolve, so `nc %h %p` fails fast and the second nc
-# takes over on the LAN address recorded as lan_ip in inventory.ini.
+# node that name does not resolve, the probe fails, and the LAN address recorded
+# as lan_ip in inventory.ini takes over.
 #
-# This is safe against the LAN addresses drifting again (which is how
-# fulcrum/mempool came to be transposed): known_hosts is keyed to the MagicDNS
-# NAME, so if lan_ip ever points at a different machine the host key will not
-# match and ssh aborts. Verified 2026-09-12 by pointing fulcrum-box at
-# mempool-box's address: "Host key verification failed."
+# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`:
+# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a
+# single `nc -w5` silently tears down the SSH session after five quiet seconds.
+# That produced intermittent "Data could not be sent to remote host" failures on
+# exactly the long, quiet operations (apt) where a dropped connection costs most.
+# `nc -z` probes, then `exec nc` carries the session with no timeout at all.
+#
+# Safe against the LAN addresses drifting again (which is how fulcrum/mempool
+# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if
+# lan_ip ever points at a different machine the host key will not match and ssh
+# aborts. Verified by pointing fulcrum-box at mempool-box's address:
+# "Host key verification failed."
 #
 # lan_ip is a convenience, not an identity. If it goes stale the fallback stops
 # working; it will never connect you to the wrong box.
 ansible_ssh_common_args: >-
-  -o ProxyCommand="sh -c 'nc -w2 %h %p 2>/dev/null || nc -w4 {{ lan_ip }} %p'"
+  -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'"

ansible/playbooks/backups.yml

@@ -14,3 +14,15 @@
           - name: headscale
             source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
             retention_days: 90
+          - name: memos
+            source: "backup-pull@memos-box:/opt/backups/memos/"
+            retention_days: 90
+          - name: vaultwarden
+            source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/"
+            retention_days: 90
+          - name: lnbits
+            source: "backup-pull@prd-vipy:/opt/backups/lnbits/"
+            retention_days: 90
+          - name: forgejo
+            source: "backup-pull@prd-vipy:/opt/backups/forgejo/"
+            retention_days: 14

ansible/roles/backup_source/README.md

@@ -39,6 +39,56 @@ The role pipes it into `age`, so plaintext never touches the disk. Use `-C /`
 with relative paths in `tar` rather than absolute ones: it avoids tar's "removing
 leading /" and makes the restore target explicit.
 
+## Services that are not systemd
+
+`backup_source_stop_service` runs `systemctl stop/start`. For anything else,
+give the pair explicitly — vaultwarden is a docker compose stack, so
+`systemctl stop vaultwarden` silently does nothing:
+
+```yaml
+backup_source_stop_command:  "docker compose -f /opt/vaultwarden/docker-compose.yml stop"
+backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start"
+```
+
+The same EXIT trap wraps both forms. The assert refuses a stop command without a
+matching start command, because that combination fails in the one way you would
+not notice: the service stops and never comes back.
+
+## More than one thing to back up
+
+`tar` takes several paths, so multiple files or directories are normally **one**
+artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together,
+lnbits captures its data directory and its `.env`.
+
+Prefer one artefact. A backup should be a consistent snapshot, and two artefacts
+written by two runs can drift — you can end up restoring an `.env` that does not
+match the database it configures. Pulling a single file back out needs no
+unpacking:
+
+```bash
+age -d -i <identity> <artefact> | tar -xzO opt/lnbits/lnbits/.env
+```
+
+If you genuinely need separate artefacts, call the role twice with different
+`backup_source_name`s rather than extending it — but only one call may set
+`backup_source_stop_service`, or the service is stopped twice per night.
+
+The case this shape cannot express is a **database dump plus a file tree**
+(`pg_dump` and a media directory, say): you cannot merge those into one stream
+without staging plaintext on disk, which is exactly what this design avoids.
+None of the current services need it — all are file trees, all stopped for the
+dump. A future one that does should use two role calls.
+
+## Everything here is sqlite, so everything stops
+
+All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files
+present). A live copy of a WAL-mode database can be torn or stale, so each is
+stopped for the duration. Measured downtime: under a second for headscale and
+memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos
+and database. That last one is the real cost of a consistent snapshot; if it
+becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy,
+not skipping the stop.
+
 ## The trap is the reason this role exists
 
 When `backup_source_stop_service` is set, the script stops the unit and installs

ansible/roles/backup_source/defaults/main.yml

@@ -17,8 +17,12 @@ backup_source_recipient: "{{ age_backup_recipient }}"
 backup_source_pull_user: backup-pull
 backup_source_pull_key: "{{ backup_pull_public_key }}"
 
-# Safety
-backup_source_stop_service: ""          # local unit stopped for the dump, restored by a trap
+# Safety. Give either a systemd unit, or an explicit pair of commands for
+# services that are not systemd-managed (vaultwarden is a docker compose stack).
+# Whichever is used, a trap guarantees the restart.
+backup_source_stop_service: ""          # systemd unit stopped for the dump
+backup_source_stop_command: ""          # overrides stop_service when set
+backup_source_start_command: ""         # required alongside stop_command
 
 # Retention here is LOCAL and short; small-backups-box keeps the long tail.
 backup_source_retention_days: 7

ansible/roles/backup_source/tasks/main.yml

@@ -7,9 +7,11 @@
       - backup_source_dump_command | length > 0
       - backup_source_recipient | length > 0
       - backup_source_recipient is match('^age1[0-9a-z]{58}$')
+      - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0)
     fail_msg: >-
       backup_source: '{{ backup_source_name | default("<unnamed>") }}' needs a name,
       description, dump command and a valid age recipient (age1... 62 chars).
+      backup_source_stop_command must be paired with backup_source_start_command.
     quiet: true
 
 # Declared here rather than assumed. Stage 1 installed it by hand; this is what

ansible/roles/backup_source/templates/backup.sh.j2

@@ -11,8 +11,10 @@ RETENTION_DAYS={{ backup_source_retention_days }}
 RECIPIENT="{{ backup_source_recipient }}"
 SUFFIX="{{ backup_source_artifact_suffix }}"
 NAME="{{ backup_source_name }}"
-{% if backup_source_stop_service %}
-SERVICE="{{ backup_source_stop_service }}"
+{% if backup_source_stop_service or backup_source_stop_command %}
+STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }}
+START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }}
+SERVICE="{{ backup_source_stop_service or backup_source_description }}"   # label for the log only
 {% endif %}
 
 TIMESTAMP=$(date +%Y%m%d_%H%M%S)
@@ -39,13 +41,13 @@ chmod 700 "$BACKUP_DIR"
 # here or they accumulate forever.
 rm -f "${BACKUP_DIR}/${NAME}_"*.partial
 
-{% if backup_source_stop_service %}
+{% if backup_source_stop_service or backup_source_stop_command %}
 # --- Stop the service, and guarantee it comes back ---
 # The trap is the point: without it a failed dump leaves the service down until
 # the next timer fires. Every hand-written script this replaced had that bug.
 log "Stopping ${SERVICE}..."
-systemctl stop "$SERVICE"
-trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT
+eval "$STOP_CMD"
+trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT
 {% endif %}
 
 # --- Dump straight into age; plaintext never touches the disk ---

ansible/services/forgejo/setup_backup_forgejo.yml

@@ -0,0 +1,25 @@
+---
+# Forgejo backup: dumps locally on vipy, encrypted with age.
+#
+# The biggest artefact in the estate (~2.7 G) and the reason retention here is
+# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14.
+# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the
+# old job did the same.
+- name: Configure the Forgejo backup on the edge host
+  hosts: edge
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./forgejo_vars.yml
+
+  tasks:
+    - name: Ensure Forgejo dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: forgejo
+        backup_source_description: "Forgejo"
+        backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo"
+        backup_source_stop_service: forgejo
+        backup_source_retention_days: 2
+        backup_source_on_calendar: "*-*-* 02:30:00"

ansible/services/lnbits/setup_backup_lnbits.yml

@@ -0,0 +1,25 @@
+---
+# LNBits backup: dumps locally on vipy, encrypted with age.
+# The old job produced TWO gpg artefacts (data, then .env separately). They are
+# folded into one tar here so the wallet database and the .env that configures
+# it are always the same point in time; two artefacts written by two runs can
+# drift. Pulling one file back out needs no unpacking:
+#   age -d -i <identity> <artefact> | tar -xzO opt/lnbits/lnbits/.env
+- name: Configure the LNBits backup on the edge host
+  hosts: edge
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./lnbits_vars.yml
+
+  tasks:
+    - name: Ensure LNBits dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: lnbits
+        backup_source_description: "LNBits"
+        backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env"
+        backup_source_stop_service: lnbits
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:20:00"

ansible/services/memos/setup_backup_memos.yml

@@ -0,0 +1,26 @@
+---
+# Memos backup: dumps locally on memos-box, encrypted with age.
+# Replaces the lapy pull, which had been writing EMPTY directories since
+# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned
+# to a different machine that has no rsync.
+- name: Configure the Memos backup on its own host
+  hosts: memos
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./memos_vars.yml
+
+  tasks:
+    - name: Ensure Memos dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: memos
+        backup_source_description: "Memos"
+        backup_source_dump_command: "tar -czf - -C / var/lib/memos"
+        # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a
+        # consistent database rather than a torn mid-write copy. The old rsync
+        # job did not stop it.
+        backup_source_stop_service: memos
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:00:00"

ansible/services/vaultwarden/setup_backup_vaultwarden.yml

@@ -0,0 +1,25 @@
+---
+# Vaultwarden backup: dumps locally on vipy, encrypted with age.
+# Previously rsynced to lapy in the CLEAR; the artefact now never exists
+# unencrypted, on disk or on the wire.
+- name: Configure the Vaultwarden backup on the edge host
+  hosts: edge
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./vaultwarden_vars.yml
+
+  tasks:
+    - name: Ensure Vaultwarden dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: vaultwarden
+        backup_source_description: "Vaultwarden"
+        backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data"
+        # Not systemd — a docker compose stack — so stop/start explicitly.
+        # sqlite in WAL mode, hence stopping at all.
+        backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop"
+        backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start"
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:10:00"