ansible/group_vars/all/main.yml

@@ -7,10 +7,3 @@ root_domain: contrapeso.xyz
 # playbooks are kept deliberately — the check logic is meant to be rewired to
 # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
 uptime_kuma_enabled: false
-
-# age recipient for all backup artefacts
-age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza"
-
-# Public key small-backups-box pulls with
-# Authorised on each source host for an unprivileged, dedicated user only
-backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull"

ansible/group_vars/nodito_vms.yml

@@ -1,24 +0,0 @@
----
-# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down.
-#
-# ansible_host is a MagicDNS name. If tailscaled is not running on the control
-# node that name does not resolve, the probe fails, and the LAN address recorded
-# as lan_ip in inventory.ini takes over.
-#
-# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`:
-# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a
-# single `nc -w5` silently tears down the SSH session after five quiet seconds.
-# That produced intermittent "Data could not be sent to remote host" failures on
-# exactly the long, quiet operations (apt) where a dropped connection costs most.
-# `nc -z` probes, then `exec nc` carries the session with no timeout at all.
-#
-# Safe against the LAN addresses drifting again (which is how fulcrum/mempool
-# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if
-# lan_ip ever points at a different machine the host key will not match and ssh
-# aborts. Verified by pointing fulcrum-box at mempool-box's address:
-# "Host key verification failed."
-#
-# lan_ip is a convenience, not an identity. If it goes stale the fallback stops
-# working; it will never connect you to the wrong box.
-ansible_ssh_common_args: >-
-  -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'"

ansible/playbooks/backups.yml

@@ -1,28 +0,0 @@
-- name: Configure the offsite backup pull
-  hosts: backup_store
-  gather_facts: yes
-
-  tasks:
-    - name: Ensure the box pulls every source on a timer
-      ansible.builtin.include_role:
-        name: backup_store
-      vars:
-        backup_store_sources:
-          - name: arbret
-            source: "arbret@prd-arbret:/opt/arbret/backups/"
-            retention_days: 90
-          - name: headscale
-            source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
-            retention_days: 90
-          - name: memos
-            source: "backup-pull@memos-box:/opt/backups/memos/"
-            retention_days: 90
-          - name: vaultwarden
-            source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/"
-            retention_days: 90
-          - name: lnbits
-            source: "backup-pull@prd-vipy:/opt/backups/lnbits/"
-            retention_days: 90
-          - name: forgejo
-            source: "backup-pull@prd-vipy:/opt/backups/forgejo/"
-            retention_days: 14

ansible/roles/backup_source/README.md

@@ -1,126 +0,0 @@
-# `backup_source`
-
-Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a
-local directory, prune, on a systemd timer. `small-backups-box` pulls the
-directory later (see `backup_store`).
-
-Modelled on `prd-arbret`, which has been doing exactly this correctly since
-before the rest of the estate was migrated.
-
-## Usage
-
-```yaml
-- ansible.builtin.include_role:
-    name: backup_source
-  vars:
-    backup_source_name: headscale
-    backup_source_description: "Headscale"
-    backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale"
-    backup_source_stop_service: headscale
-    backup_source_retention_days: 7
-```
-
-Produces `/opt/backups/headscale/headscale_<YYYYmmdd_HHMMSS>.tar.gz.age`,
-`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`.
-
-## Why the source encrypts, not the destination
-
-`age -r <recipient>` is asymmetric and the host holds only the **public** key, so
-a compromised host cannot read its own backups — or anyone else's. The scripts
-this replaces encrypted with GPG *on the laptop, after the data had already
-crossed the network*, which protects the artefact at rest but not in transit.
-
-The matching identity lives only on lapy and is escrowed. **Lose it and every
-artefact everywhere becomes noise**, including arbret's.
-
-## `backup_source_dump_command` writes to STDOUT
-
-The role pipes it into `age`, so plaintext never touches the disk. Use `-C /`
-with relative paths in `tar` rather than absolute ones: it avoids tar's "removing
-leading /" and makes the restore target explicit.
-
-## Services that are not systemd
-
-`backup_source_stop_service` runs `systemctl stop/start`. For anything else,
-give the pair explicitly — vaultwarden is a docker compose stack, so
-`systemctl stop vaultwarden` silently does nothing:
-
-```yaml
-backup_source_stop_command:  "docker compose -f /opt/vaultwarden/docker-compose.yml stop"
-backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start"
-```
-
-The same EXIT trap wraps both forms. The assert refuses a stop command without a
-matching start command, because that combination fails in the one way you would
-not notice: the service stops and never comes back.
-
-## More than one thing to back up
-
-`tar` takes several paths, so multiple files or directories are normally **one**
-artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together,
-lnbits captures its data directory and its `.env`.
-
-Prefer one artefact. A backup should be a consistent snapshot, and two artefacts
-written by two runs can drift — you can end up restoring an `.env` that does not
-match the database it configures. Pulling a single file back out needs no
-unpacking:
-
-```bash
-age -d -i <identity> <artefact> | tar -xzO opt/lnbits/lnbits/.env
-```
-
-If you genuinely need separate artefacts, call the role twice with different
-`backup_source_name`s rather than extending it — but only one call may set
-`backup_source_stop_service`, or the service is stopped twice per night.
-
-The case this shape cannot express is a **database dump plus a file tree**
-(`pg_dump` and a media directory, say): you cannot merge those into one stream
-without staging plaintext on disk, which is exactly what this design avoids.
-None of the current services need it — all are file trees, all stopped for the
-dump. A future one that does should use two role calls.
-
-## Everything here is sqlite, so everything stops
-
-All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files
-present). A live copy of a WAL-mode database can be torn or stale, so each is
-stopped for the duration. Measured downtime: under a second for headscale and
-memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos
-and database. That last one is the real cost of a consistent snapshot; if it
-becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy,
-not skipping the stop.
-
-## The trap is the reason this role exists
-
-When `backup_source_stop_service` is set, the script stops the unit and installs
-an EXIT trap that starts it again. Without it, a failed dump leaves the service
-down until the next timer fires — **every hand-written script this replaced had
-that bug**, and it was only ever masked because their `systemctl stop` failed
-first, before anything was stopped.
-
-Verified on spacey: with the dump forced to fail, the log shows
-`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit
-failed rather than hiding it), and headscale is `active` afterwards.
-
-If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed —
-which is correct, because nothing was stopped.
-
-## `.partial`
-
-The dump writes `<artifact>.partial` and only `mv`s it into place on success, so
-a truncated file is never mistaken for a backup. A failure inside the pipeline
-does leave one behind, and the prune glob cannot match it (it ends `.partial`,
-not `.tar.gz.age`), so the script clears stale partials at the **start** of each
-run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run.
-
-## `backup_source_stop_service` may be a bare name
-
-`headscale` and `headscale.service` both work. The unit template normalises it,
-because systemd rejects a bare name in `After=` with
-`Failed to add dependency ... Invalid argument` — which it logs and then ignores,
-so the unit appears to work while carrying no ordering at all.
-
-## Retention is two-tier
-
-`backup_source_retention_days` is **local** and short — these hosts are
-disk-constrained. The long tail lives on `small-backups-box`, which decides its
-own retention per source. Losing the local copy is expected and fine.

ansible/roles/backup_source/defaults/main.yml

@@ -1,31 +0,0 @@
----
-# Required
-backup_source_name: ""                  # "headscale" -> headscale_<ts>.tar.gz.age
-backup_source_description: ""           # "Headscale"
-backup_source_dump_command: ""          # must write the payload to STDOUT
-
-# Placement
-backup_source_dir: "/opt/backups/{{ backup_source_name }}"
-backup_source_artifact_suffix: "tar.gz.age"
-
-# Encryption. Asymmetric: the host holds only the public key and cannot decrypt
-# what it produces.
-backup_source_recipient: "{{ age_backup_recipient }}"
-
-# The unprivileged account small-backups-box pulls as. It owns the dump
-# directory and nothing else; it deliberately has no sudo.
-backup_source_pull_user: backup-pull
-backup_source_pull_key: "{{ backup_pull_public_key }}"
-
-# Safety. Give either a systemd unit, or an explicit pair of commands for
-# services that are not systemd-managed (vaultwarden is a docker compose stack).
-# Whichever is used, a trap guarantees the restart.
-backup_source_stop_service: ""          # systemd unit stopped for the dump
-backup_source_stop_command: ""          # overrides stop_service when set
-backup_source_start_command: ""         # required alongside stop_command
-
-# Retention here is LOCAL and short; small-backups-box keeps the long tail.
-backup_source_retention_days: 7
-
-# Schedule. The box pulls at 04:00, so dumps must land before that.
-backup_source_on_calendar: "*-*-* 02:00:00"

ansible/roles/backup_source/handlers/main.yml

@@ -1,4 +0,0 @@
----
-- name: Reload systemd for backup units
-  ansible.builtin.systemd:
-    daemon_reload: yes

ansible/roles/backup_source/tasks/main.yml

@@ -1,91 +0,0 @@
----
-- name: Assert backup_source parameters are sane
-  ansible.builtin.assert:
-    that:
-      - backup_source_name | length > 0
-      - backup_source_description | length > 0
-      - backup_source_dump_command | length > 0
-      - backup_source_recipient | length > 0
-      - backup_source_recipient is match('^age1[0-9a-z]{58}$')
-      - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0)
-    fail_msg: >-
-      backup_source: '{{ backup_source_name | default("<unnamed>") }}' needs a name,
-      description, dump command and a valid age recipient (age1... 62 chars).
-      backup_source_stop_command must be paired with backup_source_start_command.
-    quiet: true
-
-# Declared here rather than assumed. Stage 1 installed it by hand; this is what
-# makes a rebuilt host get it too.
-- name: Ensure age is installed
-  ansible.builtin.apt:
-    name: age
-    state: present
-    update_cache: yes
-    cache_valid_time: 3600
-
-# The pull account: unprivileged, no sudo, exists only so small-backups-box can
-# read the dump directory. Trust points one way — the box can read backups, and
-# can do nothing else on this host.
-- name: "Ensure the {{ backup_source_pull_user }} account exists"
-  ansible.builtin.user:
-    name: "{{ backup_source_pull_user }}"
-    system: yes
-    shell: /bin/sh          # rsync-over-ssh needs a shell; nologin breaks it
-    home: "/var/lib/{{ backup_source_pull_user }}"
-    create_home: yes
-    password: '!'           # no password login, ever
-  when: backup_source_pull_user | length > 0
-
-- name: "Authorise the backup box's key for {{ backup_source_pull_user }}"
-  ansible.posix.authorized_key:
-    user: "{{ backup_source_pull_user }}"
-    key: "{{ backup_source_pull_key }}"
-    key_options: "restrict"   # no pty, no forwarding, no user rc
-    exclusive: yes
-    state: present
-  when: backup_source_pull_user | length > 0
-
-# The shared container above the per-service directories. It must be traversable
-# or the pull account cannot reach its own directory. The script's `mkdir -p`
-# runs under `umask 077` and would otherwise create this 0700.
-- name: "Ensure {{ backup_source_dir | dirname }} is traversable"
-  ansible.builtin.file:
-    path: "{{ backup_source_dir | dirname }}"
-    state: directory
-    owner: root
-    group: root
-    mode: '0755'
-
-- name: "Ensure {{ backup_source_dir }} exists"
-  ansible.builtin.file:
-    path: "{{ backup_source_dir }}"
-    state: directory
-    owner: root
-    group: "{{ backup_source_pull_user | default('root', true) }}"
-    mode: '0750'
-
-- name: "Install the {{ backup_source_name }} backup script"
-  ansible.builtin.template:
-    src: backup.sh.j2
-    dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh"
-    owner: root
-    group: root
-    mode: '0750'
-    validate: "bash -n %s"
-
-- name: "Install the {{ backup_source_name }}-backup systemd units"
-  ansible.builtin.template:
-    src: "backup.{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}"
-    owner: root
-    group: root
-    mode: '0644'
-  loop: [service, timer]
-  notify: Reload systemd for backup units
-
-- name: "Enable the {{ backup_source_name }}-backup timer"
-  ansible.builtin.systemd:
-    name: "{{ backup_source_name }}-backup.timer"
-    enabled: yes
-    state: started
-    daemon_reload: yes

ansible/roles/backup_source/templates/backup.service.j2

@@ -1,14 +0,0 @@
-[Unit]
-Description={{ backup_source_description }} backup
-{% if backup_source_stop_service %}
-{# systemd rejects a bare name here ("Failed to add dependency ... Invalid
-   argument"), so normalise to a full unit name. #}
-After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }}
-{% endif %}
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier={{ backup_source_name }}-backup

ansible/roles/backup_source/templates/backup.sh.j2

@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source)
-#
-# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}.
-# The host holds only the age PUBLIC key, so it cannot read its own backups.
-set -euo pipefail
-umask 077
-
-BACKUP_DIR="{{ backup_source_dir }}"
-RETENTION_DAYS={{ backup_source_retention_days }}
-RECIPIENT="{{ backup_source_recipient }}"
-SUFFIX="{{ backup_source_artifact_suffix }}"
-NAME="{{ backup_source_name }}"
-{% if backup_source_stop_service or backup_source_stop_command %}
-STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }}
-START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }}
-SERVICE="{{ backup_source_stop_service or backup_source_description }}"   # label for the log only
-{% endif %}
-
-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
-ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}"
-
-die() { echo "FATAL: $*" >&2; exit 1; }
-log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
-
-# --- Pre-flight ---
-[[ -n "$RECIPIENT" ]] || die "no age recipient configured"
-command -v age >/dev/null || die "age is not installed"
-
-# Mode must agree with what the role sets, or each undoes the other every run.
-mkdir -p "$BACKUP_DIR"
-{% if backup_source_pull_user %}
-chown root:{{ backup_source_pull_user }} "$BACKUP_DIR"
-chmod 750 "$BACKUP_DIR"
-{% else %}
-chmod 700 "$BACKUP_DIR"
-{% endif %}
-
-# A run that died mid-dump leaves a .partial. It is not a backup, and the prune
-# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them
-# here or they accumulate forever.
-rm -f "${BACKUP_DIR}/${NAME}_"*.partial
-
-{% if backup_source_stop_service or backup_source_stop_command %}
-# --- Stop the service, and guarantee it comes back ---
-# The trap is the point: without it a failed dump leaves the service down until
-# the next timer fires. Every hand-written script this replaced had that bug.
-log "Stopping ${SERVICE}..."
-eval "$STOP_CMD"
-trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT
-{% endif %}
-
-# --- Dump straight into age; plaintext never touches the disk ---
-log "Writing ${ARTIFACT}..."
-{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial"
-mv "${ARTIFACT}.partial" "$ARTIFACT"
-{% if backup_source_pull_user %}
-# Readable by the pull account and nobody else. The contents are age-encrypted
-# regardless, so this is depth rather than the actual protection.
-chown root:{{ backup_source_pull_user }} "$ARTIFACT"
-chmod 640 "$ARTIFACT"
-{% else %}
-chmod 600 "$ARTIFACT"
-{% endif %}
-log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))"
-
-# --- Prune ---
-log "Pruning local artefacts older than ${RETENTION_DAYS} days..."
-find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete
-
-log "Done."

ansible/roles/backup_source/templates/backup.timer.j2

@@ -1,11 +0,0 @@
-[Unit]
-Description={{ backup_source_description }} backup
-
-[Timer]
-OnCalendar={{ backup_source_on_calendar }}
-# Persistent: a window missed while the host was down runs on next boot. cron on
-# a laptop had no equivalent, which is how two backups went unnoticed for months.
-Persistent=true
-
-[Install]
-WantedBy=timers.target

ansible/roles/backup_store/README.md

@@ -1,55 +0,0 @@
-# `backup_store`
-
-Pulls already-encrypted backup artefacts from every source host onto
-`small-backups-box`, on a timer, and expires them per source.
-
-Generalises the hand-written `pull-backups.sh` that had one hardcoded source
-(`arbret`). That job's behaviour is preserved exactly: same source path, same
-90 days, same destination directory.
-
-## This host holds no key
-
-Everything pulled here is ciphertext produced by `backup_source` on the source
-host. The box cannot read any of it — the age identity lives only on lapy. That
-is deliberate: the machine holding every backup should not also be able to open
-them.
-
-## One failing source must not stop the others
-
-The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own
-function, failures are counted, and the script exits non-zero at the end so
-systemd marks the unit failed. A dead host costs you that one source, not the
-whole run.
-
-This is the specific failure the whole plan exists to prevent: the laptop jobs
-aborted on first error and then silently produced empty directories for nine
-months.
-
-## Trust points one way
-
-The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated
-account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised
-with `restrict`. That account can read one directory and do nothing else — no
-sudo, no pty, no forwarding. A compromised backup box cannot reach into
-production.
-
-## Addressing: names, never IPs
-
-Sources are addressed by name. The job this replaced hardcoded spacey's IP; the
-droplet was later rebuilt, the address was recycled to a stranger, and the
-backup failed silently from 2025-12-01 while the directory listing still looked
-healthy.
-
-Two kinds of name are in play:
-
-- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a
-  headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without
-  it the box cannot even resolve the peer, let alone reach it.
-- **spacey** is *not* a tailnet member — it is the headscale control server — so
-  its backup is pulled over the public internet via `headscale.contrapeso.xyz`,
-  which follows the host if the droplet is rebuilt.
-
-## Retention here is the long tail
-
-Sources keep a few days locally; this box keeps 90 (or whatever the source entry
-says). Losing the source's local copy is expected.

ansible/roles/backup_store/defaults/main.yml

@@ -1,11 +0,0 @@
----
-backup_store_dir: "{{ ansible_env.HOME }}/backups"
-backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull"
-backup_store_on_calendar: "*-*-* 04:00:00"
-
-# One entry per source. `retention_days` is the LONG tail; the source keeps its
-# own short local retention.
-#   - name: headscale
-#     source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
-#     retention_days: 90
-backup_store_sources: []

ansible/roles/backup_store/handlers/main.yml

@@ -1,5 +0,0 @@
----
-- name: Reload systemd for pull-backups
-  ansible.builtin.systemd:
-    daemon_reload: yes
-  become: yes

ansible/roles/backup_store/tasks/main.yml

@@ -1,53 +0,0 @@
----
-- name: Assert backup_store sources are sane
-  ansible.builtin.assert:
-    that:
-      - backup_store_sources | length > 0
-      - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length
-      - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length
-      - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length
-    fail_msg: "backup_store: every source needs name, source and retention_days"
-    quiet: true
-
-- name: Ensure rsync is installed
-  ansible.builtin.apt:
-    name: rsync
-    state: present
-    update_cache: yes
-    cache_valid_time: 3600
-  become: yes
-
-- name: Ensure the backup store directory exists
-  ansible.builtin.file:
-    path: "{{ backup_store_dir }}"
-    state: directory
-    mode: '0700'
-
-- name: Install the pull-backups script
-  ansible.builtin.template:
-    src: pull-backups.sh.j2
-    dest: /usr/local/bin/pull-backups.sh
-    owner: root
-    group: root
-    mode: '0755'
-    validate: "bash -n %s"
-  become: yes
-
-- name: Install the pull-backups systemd units
-  ansible.builtin.template:
-    src: "pull-backups.{{ item }}.j2"
-    dest: "/etc/systemd/system/pull-backups.{{ item }}"
-    owner: root
-    group: root
-    mode: '0644'
-  loop: [service, timer]
-  become: yes
-  notify: Reload systemd for pull-backups
-
-- name: Enable the pull-backups timer
-  ansible.builtin.systemd:
-    name: pull-backups.timer
-    enabled: yes
-    state: started
-    daemon_reload: yes
-  become: yes

ansible/roles/backup_store/templates/pull-backups.service.j2

@@ -1,10 +0,0 @@
-[Unit]
-Description=Pull encrypted backups from production
-
-[Service]
-Type=oneshot
-User={{ ansible_user_id }}
-ExecStart=/usr/local/bin/pull-backups.sh
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier=pull-backups

ansible/roles/backup_store/templates/pull-backups.sh.j2

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Pull encrypted backups from production — managed by Ansible (roles/backup_store)
-#
-# Everything here is already ciphertext: this host only moves and expires files,
-# and holds no key that can read them.
-set -uo pipefail          # deliberately NOT -e; see the loop below
-
-SSH_KEY="{{ backup_store_ssh_key }}"
-STORE="{{ backup_store_dir }}"
-
-log()  { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
-fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); }
-
-failures=0
-
-# One source failing must not stop the others. The whole point of this box is
-# that a single dead host cannot silently take the rest of the backups with it —
-# which is exactly how the laptop-based jobs failed unnoticed for nine months.
-{% for src in backup_store_sources %}
-# --- {{ src.name }} ---
-pull_{{ src.name | replace('-', '_') }}() {
-  local dir="${STORE}/{{ src.name }}"
-  mkdir -p "$dir"
-  log "Pulling {{ src.name }} from {{ src.source }}..."
-  if rsync -az --timeout=120 \
-       -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
-       "{{ src.source }}" "$dir/"; then
-    log "  {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"
-  else
-    fail "{{ src.name }}: rsync failed"
-    return 1
-  fi
-  log "  {{ src.name }}: pruning older than {{ src.retention_days }} days"
-  find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete
-}
-pull_{{ src.name | replace('-', '_') }} || true
-
-{% endfor %}
-if [ "$failures" -gt 0 ]; then
-  log "FAILED: $failures source(s) did not pull"
-  exit 1
-fi
-log "All sources pulled."

ansible/roles/backup_store/templates/pull-backups.timer.j2

@@ -1,9 +0,0 @@
-[Unit]
-Description=Daily offsite backup pull
-
-[Timer]
-OnCalendar={{ backup_store_on_calendar }}
-Persistent=true
-
-[Install]
-WantedBy=timers.target

ansible/services/forgejo/setup_backup_forgejo.yml

@@ -1,25 +0,0 @@
----
-# Forgejo backup: dumps locally on vipy, encrypted with age.
-#
-# The biggest artefact in the estate (~2.7 G) and the reason retention here is
-# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14.
-# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the
-# old job did the same.
-- name: Configure the Forgejo backup on the edge host
-  hosts: edge
-  become: yes
-  vars_files:
-    - ../../group_vars/all/main.yml
-    - ./forgejo_vars.yml
-
-  tasks:
-    - name: Ensure Forgejo dumps itself, encrypted, on a timer
-      ansible.builtin.include_role:
-        name: backup_source
-      vars:
-        backup_source_name: forgejo
-        backup_source_description: "Forgejo"
-        backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo"
-        backup_source_stop_service: forgejo
-        backup_source_retention_days: 2
-        backup_source_on_calendar: "*-*-* 02:30:00"

ansible/services/headscale/setup_backup_headscale.yml

@@ -1,21 +0,0 @@
----
-- name: Configure the Headscale backup on the vpn_control host
-  hosts: vpn_control
-  become: yes
-  vars_files:
-    - ../../group_vars/all/main.yml
-    - ./headscale_vars.yml
-
-  tasks:
-    - name: Ensure Headscale dumps itself, encrypted, on a timer
-      ansible.builtin.include_role:
-        name: backup_source
-      vars:
-        backup_source_name: headscale
-        backup_source_description: "Headscale"
-        # -C / with relative paths: avoids tar's "removing leading /" and makes
-        # the restore target explicit.
-        backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale"
-        backup_source_stop_service: headscale
-        backup_source_retention_days: 7
-        backup_source_on_calendar: "*-*-* 02:00:00"

ansible/services/lnbits/setup_backup_lnbits.yml

@@ -1,25 +0,0 @@
----
-# LNBits backup: dumps locally on vipy, encrypted with age.
-# The old job produced TWO gpg artefacts (data, then .env separately). They are
-# folded into one tar here so the wallet database and the .env that configures
-# it are always the same point in time; two artefacts written by two runs can
-# drift. Pulling one file back out needs no unpacking:
-#   age -d -i <identity> <artefact> | tar -xzO opt/lnbits/lnbits/.env
-- name: Configure the LNBits backup on the edge host
-  hosts: edge
-  become: yes
-  vars_files:
-    - ../../group_vars/all/main.yml
-    - ./lnbits_vars.yml
-
-  tasks:
-    - name: Ensure LNBits dumps itself, encrypted, on a timer
-      ansible.builtin.include_role:
-        name: backup_source
-      vars:
-        backup_source_name: lnbits
-        backup_source_description: "LNBits"
-        backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env"
-        backup_source_stop_service: lnbits
-        backup_source_retention_days: 7
-        backup_source_on_calendar: "*-*-* 02:20:00"

ansible/services/memos/setup_backup_memos.yml

@@ -1,26 +0,0 @@
----
-# Memos backup: dumps locally on memos-box, encrypted with age.
-# Replaces the lapy pull, which had been writing EMPTY directories since
-# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned
-# to a different machine that has no rsync.
-- name: Configure the Memos backup on its own host
-  hosts: memos
-  become: yes
-  vars_files:
-    - ../../group_vars/all/main.yml
-    - ./memos_vars.yml
-
-  tasks:
-    - name: Ensure Memos dumps itself, encrypted, on a timer
-      ansible.builtin.include_role:
-        name: backup_source
-      vars:
-        backup_source_name: memos
-        backup_source_description: "Memos"
-        backup_source_dump_command: "tar -czf - -C / var/lib/memos"
-        # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a
-        # consistent database rather than a torn mid-write copy. The old rsync
-        # job did not stop it.
-        backup_source_stop_service: memos
-        backup_source_retention_days: 7
-        backup_source_on_calendar: "*-*-* 02:00:00"

ansible/services/vaultwarden/setup_backup_vaultwarden.yml

@@ -1,25 +0,0 @@
----
-# Vaultwarden backup: dumps locally on vipy, encrypted with age.
-# Previously rsynced to lapy in the CLEAR; the artefact now never exists
-# unencrypted, on disk or on the wire.
-- name: Configure the Vaultwarden backup on the edge host
-  hosts: edge
-  become: yes
-  vars_files:
-    - ../../group_vars/all/main.yml
-    - ./vaultwarden_vars.yml
-
-  tasks:
-    - name: Ensure Vaultwarden dumps itself, encrypted, on a timer
-      ansible.builtin.include_role:
-        name: backup_source
-      vars:
-        backup_source_name: vaultwarden
-        backup_source_description: "Vaultwarden"
-        backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data"
-        # Not systemd — a docker compose stack — so stop/start explicitly.
-        # sqlite in WAL mode, hence stopping at all.
-        backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop"
-        backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start"
-        backup_source_retention_days: 7
-        backup_source_on_calendar: "*-*-* 02:10:00"