From 01b83a80ec80b44289a0637fa4b1f1aaaf9b43d8 Mon Sep 17 00:00:00 2001
From: counterweight <counterweightoperator@protonmail.com>
Date: Sat, 12 Sep 2026 15:32:17 +0200
Subject: [PATCH 1/3] ip tricks

---
 ansible/group_vars/nodito_vms.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 ansible/group_vars/nodito_vms.yml

diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml
new file mode 100644
index 0000000..2ff6994
--- /dev/null
+++ b/ansible/group_vars/nodito_vms.yml
@@ -0,0 +1,17 @@
+---
+# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down.
+#
+# ansible_host is a MagicDNS name. If tailscaled is not running on the control
+# node that name does not resolve, so `nc %h %p` fails fast and the second nc
+# takes over on the LAN address recorded as lan_ip in inventory.ini.
+#
+# This is safe against the LAN addresses drifting again (which is how
+# fulcrum/mempool came to be transposed): known_hosts is keyed to the MagicDNS
+# NAME, so if lan_ip ever points at a different machine the host key will not
+# match and ssh aborts. Verified 2026-09-12 by pointing fulcrum-box at
+# mempool-box's address: "Host key verification failed."
+#
+# lan_ip is a convenience, not an identity. If it goes stale the fallback stops
+# working; it will never connect you to the wrong box.
+ansible_ssh_common_args: >-
+  -o ProxyCommand="sh -c 'nc -w2 %h %p 2>/dev/null || nc -w4 {{ lan_ip }} %p'"

From 27e036eccd1d5fa88f57ad22ed521cb1fe46275e Mon Sep 17 00:00:00 2001
From: counterweight <counterweightoperator@protonmail.com>
Date: Sat, 12 Sep 2026 16:02:00 +0200
Subject: [PATCH 2/3] backup stuff

---
 ansible/playbooks/backups.yml                 | 16 ++++
 ansible/roles/backup_source/README.md         | 76 ++++++++++++++++
 ansible/roles/backup_source/defaults/main.yml | 27 ++++++
 ansible/roles/backup_source/handlers/main.yml |  4 +
 ansible/roles/backup_source/tasks/main.yml    | 89 +++++++++++++++++++
 .../backup_source/templates/backup.service.j2 | 14 +++
 .../backup_source/templates/backup.sh.j2      | 69 ++++++++++++++
 .../backup_source/templates/backup.timer.j2   | 11 +++
 ansible/roles/backup_store/README.md          | 55 ++++++++++++
 ansible/roles/backup_store/defaults/main.yml  | 11 +++
 ansible/roles/backup_store/handlers/main.yml  |  5 ++
 ansible/roles/backup_store/tasks/main.yml     | 53 +++++++++++
 .../templates/pull-backups.service.j2         | 10 +++
 .../backup_store/templates/pull-backups.sh.j2 | 43 +++++++++
 .../templates/pull-backups.timer.j2           |  9 ++
 .../headscale/setup_backup_headscale.yml      | 21 +++++
 16 files changed, 513 insertions(+)
 create mode 100644 ansible/playbooks/backups.yml
 create mode 100644 ansible/roles/backup_source/README.md
 create mode 100644 ansible/roles/backup_source/defaults/main.yml
 create mode 100644 ansible/roles/backup_source/handlers/main.yml
 create mode 100644 ansible/roles/backup_source/tasks/main.yml
 create mode 100644 ansible/roles/backup_source/templates/backup.service.j2
 create mode 100644 ansible/roles/backup_source/templates/backup.sh.j2
 create mode 100644 ansible/roles/backup_source/templates/backup.timer.j2
 create mode 100644 ansible/roles/backup_store/README.md
 create mode 100644 ansible/roles/backup_store/defaults/main.yml
 create mode 100644 ansible/roles/backup_store/handlers/main.yml
 create mode 100644 ansible/roles/backup_store/tasks/main.yml
 create mode 100644 ansible/roles/backup_store/templates/pull-backups.service.j2
 create mode 100644 ansible/roles/backup_store/templates/pull-backups.sh.j2
 create mode 100644 ansible/roles/backup_store/templates/pull-backups.timer.j2
 create mode 100644 ansible/services/headscale/setup_backup_headscale.yml

diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml
new file mode 100644
index 0000000..842942e
--- /dev/null
+++ b/ansible/playbooks/backups.yml
@@ -0,0 +1,16 @@
+- name: Configure the offsite backup pull
+  hosts: backup_store
+  gather_facts: yes
+
+  tasks:
+    - name: Ensure the box pulls every source on a timer
+      ansible.builtin.include_role:
+        name: backup_store
+      vars:
+        backup_store_sources:
+          - name: arbret
+            source: "arbret@prd-arbret:/opt/arbret/backups/"
+            retention_days: 90
+          - name: headscale
+            source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
+            retention_days: 90
diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md
new file mode 100644
index 0000000..065e1fc
--- /dev/null
+++ b/ansible/roles/backup_source/README.md
@@ -0,0 +1,76 @@
+# `backup_source`
+
+Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a
+local directory, prune, on a systemd timer. `small-backups-box` pulls the
+directory later (see `backup_store`).
+
+Modelled on `prd-arbret`, which has been doing exactly this correctly since
+before the rest of the estate was migrated.
+
+## Usage
+
+```yaml
+- ansible.builtin.include_role:
+    name: backup_source
+  vars:
+    backup_source_name: headscale
+    backup_source_description: "Headscale"
+    backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale"
+    backup_source_stop_service: headscale
+    backup_source_retention_days: 7
+```
+
+Produces `/opt/backups/headscale/headscale_<YYYYmmdd_HHMMSS>.tar.gz.age`,
+`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`.
+
+## Why the source encrypts, not the destination
+
+`age -r <recipient>` is asymmetric and the host holds only the **public** key, so
+a compromised host cannot read its own backups — or anyone else's. The scripts
+this replaces encrypted with GPG *on the laptop, after the data had already
+crossed the network*, which protects the artefact at rest but not in transit.
+
+The matching identity lives only on lapy and is escrowed. **Lose it and every
+artefact everywhere becomes noise**, including arbret's.
+
+## `backup_source_dump_command` writes to STDOUT
+
+The role pipes it into `age`, so plaintext never touches the disk. Use `-C /`
+with relative paths in `tar` rather than absolute ones: it avoids tar's "removing
+leading /" and makes the restore target explicit.
+
+## The trap is the reason this role exists
+
+When `backup_source_stop_service` is set, the script stops the unit and installs
+an EXIT trap that starts it again. Without it, a failed dump leaves the service
+down until the next timer fires — **every hand-written script this replaced had
+that bug**, and it was only ever masked because their `systemctl stop` failed
+first, before anything was stopped.
+
+Verified on spacey: with the dump forced to fail, the log shows
+`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit
+failed rather than hiding it), and headscale is `active` afterwards.
+
+If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed —
+which is correct, because nothing was stopped.
+
+## `.partial`
+
+The dump writes `<artifact>.partial` and only `mv`s it into place on success, so
+a truncated file is never mistaken for a backup. A failure inside the pipeline
+does leave one behind, and the prune glob cannot match it (it ends `.partial`,
+not `.tar.gz.age`), so the script clears stale partials at the **start** of each
+run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run.
+
+## `backup_source_stop_service` may be a bare name
+
+`headscale` and `headscale.service` both work. The unit template normalises it,
+because systemd rejects a bare name in `After=` with
+`Failed to add dependency ... Invalid argument` — which it logs and then ignores,
+so the unit appears to work while carrying no ordering at all.
+
+## Retention is two-tier
+
+`backup_source_retention_days` is **local** and short — these hosts are
+disk-constrained. The long tail lives on `small-backups-box`, which decides its
+own retention per source. Losing the local copy is expected and fine.
diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml
new file mode 100644
index 0000000..77d17e0
--- /dev/null
+++ b/ansible/roles/backup_source/defaults/main.yml
@@ -0,0 +1,27 @@
+---
+# Required
+backup_source_name: ""                  # "headscale" -> headscale_<ts>.tar.gz.age
+backup_source_description: ""           # "Headscale"
+backup_source_dump_command: ""          # must write the payload to STDOUT
+
+# Placement
+backup_source_dir: "/opt/backups/{{ backup_source_name }}"
+backup_source_artifact_suffix: "tar.gz.age"
+
+# Encryption. Asymmetric: the host holds only the public key and cannot decrypt
+# what it produces.
+backup_source_recipient: "{{ age_backup_recipient }}"
+
+# The unprivileged account small-backups-box pulls as. It owns the dump
+# directory and nothing else; it deliberately has no sudo.
+backup_source_pull_user: backup-pull
+backup_source_pull_key: "{{ backup_pull_public_key }}"
+
+# Safety
+backup_source_stop_service: ""          # local unit stopped for the dump, restored by a trap
+
+# Retention here is LOCAL and short; small-backups-box keeps the long tail.
+backup_source_retention_days: 7
+
+# Schedule. The box pulls at 04:00, so dumps must land before that.
+backup_source_on_calendar: "*-*-* 02:00:00"
diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml
new file mode 100644
index 0000000..37a7f2b
--- /dev/null
+++ b/ansible/roles/backup_source/handlers/main.yml
@@ -0,0 +1,4 @@
+---
+- name: Reload systemd for backup units
+  ansible.builtin.systemd:
+    daemon_reload: yes
diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml
new file mode 100644
index 0000000..54d6ac1
--- /dev/null
+++ b/ansible/roles/backup_source/tasks/main.yml
@@ -0,0 +1,89 @@
+---
+- name: Assert backup_source parameters are sane
+  ansible.builtin.assert:
+    that:
+      - backup_source_name | length > 0
+      - backup_source_description | length > 0
+      - backup_source_dump_command | length > 0
+      - backup_source_recipient | length > 0
+      - backup_source_recipient is match('^age1[0-9a-z]{58}$')
+    fail_msg: >-
+      backup_source: '{{ backup_source_name | default("<unnamed>") }}' needs a name,
+      description, dump command and a valid age recipient (age1... 62 chars).
+    quiet: true
+
+# Declared here rather than assumed. Stage 1 installed it by hand; this is what
+# makes a rebuilt host get it too.
+- name: Ensure age is installed
+  ansible.builtin.apt:
+    name: age
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
+# The pull account: unprivileged, no sudo, exists only so small-backups-box can
+# read the dump directory. Trust points one way — the box can read backups, and
+# can do nothing else on this host.
+- name: "Ensure the {{ backup_source_pull_user }} account exists"
+  ansible.builtin.user:
+    name: "{{ backup_source_pull_user }}"
+    system: yes
+    shell: /bin/sh          # rsync-over-ssh needs a shell; nologin breaks it
+    home: "/var/lib/{{ backup_source_pull_user }}"
+    create_home: yes
+    password: '!'           # no password login, ever
+  when: backup_source_pull_user | length > 0
+
+- name: "Authorise the backup box's key for {{ backup_source_pull_user }}"
+  ansible.posix.authorized_key:
+    user: "{{ backup_source_pull_user }}"
+    key: "{{ backup_source_pull_key }}"
+    key_options: "restrict"   # no pty, no forwarding, no user rc
+    exclusive: yes
+    state: present
+  when: backup_source_pull_user | length > 0
+
+# The shared container above the per-service directories. It must be traversable
+# or the pull account cannot reach its own directory. The script's `mkdir -p`
+# runs under `umask 077` and would otherwise create this 0700.
+- name: "Ensure {{ backup_source_dir | dirname }} is traversable"
+  ansible.builtin.file:
+    path: "{{ backup_source_dir | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: "Ensure {{ backup_source_dir }} exists"
+  ansible.builtin.file:
+    path: "{{ backup_source_dir }}"
+    state: directory
+    owner: root
+    group: "{{ backup_source_pull_user | default('root', true) }}"
+    mode: '0750'
+
+- name: "Install the {{ backup_source_name }} backup script"
+  ansible.builtin.template:
+    src: backup.sh.j2
+    dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh"
+    owner: root
+    group: root
+    mode: '0750'
+    validate: "bash -n %s"
+
+- name: "Install the {{ backup_source_name }}-backup systemd units"
+  ansible.builtin.template:
+    src: "backup.{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop: [service, timer]
+  notify: Reload systemd for backup units
+
+- name: "Enable the {{ backup_source_name }}-backup timer"
+  ansible.builtin.systemd:
+    name: "{{ backup_source_name }}-backup.timer"
+    enabled: yes
+    state: started
+    daemon_reload: yes
diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2
new file mode 100644
index 0000000..91ad063
--- /dev/null
+++ b/ansible/roles/backup_source/templates/backup.service.j2
@@ -0,0 +1,14 @@
+[Unit]
+Description={{ backup_source_description }} backup
+{% if backup_source_stop_service %}
+{# systemd rejects a bare name here ("Failed to add dependency ... Invalid
+   argument"), so normalise to a full unit name. #}
+After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }}
+{% endif %}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier={{ backup_source_name }}-backup
diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2
new file mode 100644
index 0000000..35e4398
--- /dev/null
+++ b/ansible/roles/backup_source/templates/backup.sh.j2
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source)
+#
+# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}.
+# The host holds only the age PUBLIC key, so it cannot read its own backups.
+set -euo pipefail
+umask 077
+
+BACKUP_DIR="{{ backup_source_dir }}"
+RETENTION_DAYS={{ backup_source_retention_days }}
+RECIPIENT="{{ backup_source_recipient }}"
+SUFFIX="{{ backup_source_artifact_suffix }}"
+NAME="{{ backup_source_name }}"
+{% if backup_source_stop_service %}
+SERVICE="{{ backup_source_stop_service }}"
+{% endif %}
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}"
+
+die() { echo "FATAL: $*" >&2; exit 1; }
+log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
+
+# --- Pre-flight ---
+[[ -n "$RECIPIENT" ]] || die "no age recipient configured"
+command -v age >/dev/null || die "age is not installed"
+
+# Mode must agree with what the role sets, or each undoes the other every run.
+mkdir -p "$BACKUP_DIR"
+{% if backup_source_pull_user %}
+chown root:{{ backup_source_pull_user }} "$BACKUP_DIR"
+chmod 750 "$BACKUP_DIR"
+{% else %}
+chmod 700 "$BACKUP_DIR"
+{% endif %}
+
+# A run that died mid-dump leaves a .partial. It is not a backup, and the prune
+# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them
+# here or they accumulate forever.
+rm -f "${BACKUP_DIR}/${NAME}_"*.partial
+
+{% if backup_source_stop_service %}
+# --- Stop the service, and guarantee it comes back ---
+# The trap is the point: without it a failed dump leaves the service down until
+# the next timer fires. Every hand-written script this replaced had that bug.
+log "Stopping ${SERVICE}..."
+systemctl stop "$SERVICE"
+trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT
+{% endif %}
+
+# --- Dump straight into age; plaintext never touches the disk ---
+log "Writing ${ARTIFACT}..."
+{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial"
+mv "${ARTIFACT}.partial" "$ARTIFACT"
+{% if backup_source_pull_user %}
+# Readable by the pull account and nobody else. The contents are age-encrypted
+# regardless, so this is depth rather than the actual protection.
+chown root:{{ backup_source_pull_user }} "$ARTIFACT"
+chmod 640 "$ARTIFACT"
+{% else %}
+chmod 600 "$ARTIFACT"
+{% endif %}
+log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))"
+
+# --- Prune ---
+log "Pruning local artefacts older than ${RETENTION_DAYS} days..."
+find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete
+
+log "Done."
diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2
new file mode 100644
index 0000000..d9a7e6a
--- /dev/null
+++ b/ansible/roles/backup_source/templates/backup.timer.j2
@@ -0,0 +1,11 @@
+[Unit]
+Description={{ backup_source_description }} backup
+
+[Timer]
+OnCalendar={{ backup_source_on_calendar }}
+# Persistent: a window missed while the host was down runs on next boot. cron on
+# a laptop had no equivalent, which is how two backups went unnoticed for months.
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md
new file mode 100644
index 0000000..13d9f5a
--- /dev/null
+++ b/ansible/roles/backup_store/README.md
@@ -0,0 +1,55 @@
+# `backup_store`
+
+Pulls already-encrypted backup artefacts from every source host onto
+`small-backups-box`, on a timer, and expires them per source.
+
+Generalises the hand-written `pull-backups.sh` that had one hardcoded source
+(`arbret`). That job's behaviour is preserved exactly: same source path, same
+90 days, same destination directory.
+
+## This host holds no key
+
+Everything pulled here is ciphertext produced by `backup_source` on the source
+host. The box cannot read any of it — the age identity lives only on lapy. That
+is deliberate: the machine holding every backup should not also be able to open
+them.
+
+## One failing source must not stop the others
+
+The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own
+function, failures are counted, and the script exits non-zero at the end so
+systemd marks the unit failed. A dead host costs you that one source, not the
+whole run.
+
+This is the specific failure the whole plan exists to prevent: the laptop jobs
+aborted on first error and then silently produced empty directories for nine
+months.
+
+## Trust points one way
+
+The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated
+account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised
+with `restrict`. That account can read one directory and do nothing else — no
+sudo, no pty, no forwarding. A compromised backup box cannot reach into
+production.
+
+## Addressing: names, never IPs
+
+Sources are addressed by name. The job this replaced hardcoded spacey's IP; the
+droplet was later rebuilt, the address was recycled to a stranger, and the
+backup failed silently from 2025-12-01 while the directory listing still looked
+healthy.
+
+Two kinds of name are in play:
+
+- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a
+  headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without
+  it the box cannot even resolve the peer, let alone reach it.
+- **spacey** is *not* a tailnet member — it is the headscale control server — so
+  its backup is pulled over the public internet via `headscale.contrapeso.xyz`,
+  which follows the host if the droplet is rebuilt.
+
+## Retention here is the long tail
+
+Sources keep a few days locally; this box keeps 90 (or whatever the source entry
+says). Losing the source's local copy is expected.
diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml
new file mode 100644
index 0000000..c0c128b
--- /dev/null
+++ b/ansible/roles/backup_store/defaults/main.yml
@@ -0,0 +1,11 @@
+---
+backup_store_dir: "{{ ansible_env.HOME }}/backups"
+backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull"
+backup_store_on_calendar: "*-*-* 04:00:00"
+
+# One entry per source. `retention_days` is the LONG tail; the source keeps its
+# own short local retention.
+#   - name: headscale
+#     source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
+#     retention_days: 90
+backup_store_sources: []
diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml
new file mode 100644
index 0000000..632be35
--- /dev/null
+++ b/ansible/roles/backup_store/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: Reload systemd for pull-backups
+  ansible.builtin.systemd:
+    daemon_reload: yes
+  become: yes
diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml
new file mode 100644
index 0000000..d138830
--- /dev/null
+++ b/ansible/roles/backup_store/tasks/main.yml
@@ -0,0 +1,53 @@
+---
+- name: Assert backup_store sources are sane
+  ansible.builtin.assert:
+    that:
+      - backup_store_sources | length > 0
+      - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length
+      - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length
+      - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length
+    fail_msg: "backup_store: every source needs name, source and retention_days"
+    quiet: true
+
+- name: Ensure rsync is installed
+  ansible.builtin.apt:
+    name: rsync
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+  become: yes
+
+- name: Ensure the backup store directory exists
+  ansible.builtin.file:
+    path: "{{ backup_store_dir }}"
+    state: directory
+    mode: '0700'
+
+- name: Install the pull-backups script
+  ansible.builtin.template:
+    src: pull-backups.sh.j2
+    dest: /usr/local/bin/pull-backups.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+  become: yes
+
+- name: Install the pull-backups systemd units
+  ansible.builtin.template:
+    src: "pull-backups.{{ item }}.j2"
+    dest: "/etc/systemd/system/pull-backups.{{ item }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop: [service, timer]
+  become: yes
+  notify: Reload systemd for pull-backups
+
+- name: Enable the pull-backups timer
+  ansible.builtin.systemd:
+    name: pull-backups.timer
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  become: yes
diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2
new file mode 100644
index 0000000..c65516c
--- /dev/null
+++ b/ansible/roles/backup_store/templates/pull-backups.service.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Pull encrypted backups from production
+
+[Service]
+Type=oneshot
+User={{ ansible_user_id }}
+ExecStart=/usr/local/bin/pull-backups.sh
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=pull-backups
diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2
new file mode 100644
index 0000000..267900c
--- /dev/null
+++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Pull encrypted backups from production — managed by Ansible (roles/backup_store)
+#
+# Everything here is already ciphertext: this host only moves and expires files,
+# and holds no key that can read them.
+set -uo pipefail          # deliberately NOT -e; see the loop below
+
+SSH_KEY="{{ backup_store_ssh_key }}"
+STORE="{{ backup_store_dir }}"
+
+log()  { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
+fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); }
+
+failures=0
+
+# One source failing must not stop the others. The whole point of this box is
+# that a single dead host cannot silently take the rest of the backups with it —
+# which is exactly how the laptop-based jobs failed unnoticed for nine months.
+{% for src in backup_store_sources %}
+# --- {{ src.name }} ---
+pull_{{ src.name | replace('-', '_') }}() {
+  local dir="${STORE}/{{ src.name }}"
+  mkdir -p "$dir"
+  log "Pulling {{ src.name }} from {{ src.source }}..."
+  if rsync -az --timeout=120 \
+       -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
+       "{{ src.source }}" "$dir/"; then
+    log "  {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"
+  else
+    fail "{{ src.name }}: rsync failed"
+    return 1
+  fi
+  log "  {{ src.name }}: pruning older than {{ src.retention_days }} days"
+  find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete
+}
+pull_{{ src.name | replace('-', '_') }} || true
+
+{% endfor %}
+if [ "$failures" -gt 0 ]; then
+  log "FAILED: $failures source(s) did not pull"
+  exit 1
+fi
+log "All sources pulled."
diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2
new file mode 100644
index 0000000..336e1ca
--- /dev/null
+++ b/ansible/roles/backup_store/templates/pull-backups.timer.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily offsite backup pull
+
+[Timer]
+OnCalendar={{ backup_store_on_calendar }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml
new file mode 100644
index 0000000..9c72d1d
--- /dev/null
+++ b/ansible/services/headscale/setup_backup_headscale.yml
@@ -0,0 +1,21 @@
+---
+- name: Configure the Headscale backup on the vpn_control host
+  hosts: vpn_control
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./headscale_vars.yml
+
+  tasks:
+    - name: Ensure Headscale dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: headscale
+        backup_source_description: "Headscale"
+        # -C / with relative paths: avoids tar's "removing leading /" and makes
+        # the restore target explicit.
+        backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale"
+        backup_source_stop_service: headscale
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:00:00"

From 394f2519ff261f518473f6df171c736d51a164a2 Mon Sep 17 00:00:00 2001
From: counterweight <counterweightoperator@protonmail.com>
Date: Sat, 12 Sep 2026 16:20:42 +0200
Subject: [PATCH 3/3] age backups everywhere

---
 ansible/group_vars/all/main.yml               |  7 +++
 ansible/group_vars/nodito_vms.yml             | 23 ++++++---
 ansible/playbooks/backups.yml                 | 12 +++++
 ansible/roles/backup_source/README.md         | 50 +++++++++++++++++++
 ansible/roles/backup_source/defaults/main.yml |  8 ++-
 ansible/roles/backup_source/tasks/main.yml    |  2 +
 .../backup_source/templates/backup.sh.j2      | 12 +++--
 .../services/forgejo/setup_backup_forgejo.yml | 25 ++++++++++
 .../services/lnbits/setup_backup_lnbits.yml   | 25 ++++++++++
 ansible/services/memos/setup_backup_memos.yml | 26 ++++++++++
 .../vaultwarden/setup_backup_vaultwarden.yml  | 25 ++++++++++
 11 files changed, 200 insertions(+), 15 deletions(-)
 create mode 100644 ansible/services/forgejo/setup_backup_forgejo.yml
 create mode 100644 ansible/services/lnbits/setup_backup_lnbits.yml
 create mode 100644 ansible/services/memos/setup_backup_memos.yml
 create mode 100644 ansible/services/vaultwarden/setup_backup_vaultwarden.yml

diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml
index 36d35f8..0be0162 100644
--- a/ansible/group_vars/all/main.yml
+++ b/ansible/group_vars/all/main.yml
@@ -7,3 +7,10 @@ root_domain: contrapeso.xyz
 # playbooks are kept deliberately — the check logic is meant to be rewired to
 # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
 uptime_kuma_enabled: false
+
+# age recipient for all backup artefacts
+age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza"
+
+# Public key small-backups-box pulls with
+# Authorised on each source host for an unprivileged, dedicated user only
+backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull"
diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml
index 2ff6994..1072b51 100644
--- a/ansible/group_vars/nodito_vms.yml
+++ b/ansible/group_vars/nodito_vms.yml
@@ -2,16 +2,23 @@
 # Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down.
 #
 # ansible_host is a MagicDNS name. If tailscaled is not running on the control
-# node that name does not resolve, so `nc %h %p` fails fast and the second nc
-# takes over on the LAN address recorded as lan_ip in inventory.ini.
+# node that name does not resolve, the probe fails, and the LAN address recorded
+# as lan_ip in inventory.ini takes over.
 #
-# This is safe against the LAN addresses drifting again (which is how
-# fulcrum/mempool came to be transposed): known_hosts is keyed to the MagicDNS
-# NAME, so if lan_ip ever points at a different machine the host key will not
-# match and ssh aborts. Verified 2026-09-12 by pointing fulcrum-box at
-# mempool-box's address: "Host key verification failed."
+# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`:
+# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a
+# single `nc -w5` silently tears down the SSH session after five quiet seconds.
+# That produced intermittent "Data could not be sent to remote host" failures on
+# exactly the long, quiet operations (apt) where a dropped connection costs most.
+# `nc -z` probes, then `exec nc` carries the session with no timeout at all.
+#
+# Safe against the LAN addresses drifting again (which is how fulcrum/mempool
+# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if
+# lan_ip ever points at a different machine the host key will not match and ssh
+# aborts. Verified by pointing fulcrum-box at mempool-box's address:
+# "Host key verification failed."
 #
 # lan_ip is a convenience, not an identity. If it goes stale the fallback stops
 # working; it will never connect you to the wrong box.
 ansible_ssh_common_args: >-
-  -o ProxyCommand="sh -c 'nc -w2 %h %p 2>/dev/null || nc -w4 {{ lan_ip }} %p'"
+  -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'"
diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml
index 842942e..0686ab6 100644
--- a/ansible/playbooks/backups.yml
+++ b/ansible/playbooks/backups.yml
@@ -14,3 +14,15 @@
           - name: headscale
             source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
             retention_days: 90
+          - name: memos
+            source: "backup-pull@memos-box:/opt/backups/memos/"
+            retention_days: 90
+          - name: vaultwarden
+            source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/"
+            retention_days: 90
+          - name: lnbits
+            source: "backup-pull@prd-vipy:/opt/backups/lnbits/"
+            retention_days: 90
+          - name: forgejo
+            source: "backup-pull@prd-vipy:/opt/backups/forgejo/"
+            retention_days: 14
diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md
index 065e1fc..a7794ab 100644
--- a/ansible/roles/backup_source/README.md
+++ b/ansible/roles/backup_source/README.md
@@ -39,6 +39,56 @@ The role pipes it into `age`, so plaintext never touches the disk. Use `-C /`
 with relative paths in `tar` rather than absolute ones: it avoids tar's "removing
 leading /" and makes the restore target explicit.
 
+## Services that are not systemd
+
+`backup_source_stop_service` runs `systemctl stop/start`. For anything else,
+give the pair explicitly — vaultwarden is a docker compose stack, so
+`systemctl stop vaultwarden` silently does nothing:
+
+```yaml
+backup_source_stop_command:  "docker compose -f /opt/vaultwarden/docker-compose.yml stop"
+backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start"
+```
+
+The same EXIT trap wraps both forms. The assert refuses a stop command without a
+matching start command, because that combination fails in the one way you would
+not notice: the service stops and never comes back.
+
+## More than one thing to back up
+
+`tar` takes several paths, so multiple files or directories are normally **one**
+artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together,
+lnbits captures its data directory and its `.env`.
+
+Prefer one artefact. A backup should be a consistent snapshot, and two artefacts
+written by two runs can drift — you can end up restoring an `.env` that does not
+match the database it configures. Pulling a single file back out needs no
+unpacking:
+
+```bash
+age -d -i <identity> <artefact> | tar -xzO opt/lnbits/lnbits/.env
+```
+
+If you genuinely need separate artefacts, call the role twice with different
+`backup_source_name`s rather than extending it — but only one call may set
+`backup_source_stop_service`, or the service is stopped twice per night.
+
+The case this shape cannot express is a **database dump plus a file tree**
+(`pg_dump` and a media directory, say): you cannot merge those into one stream
+without staging plaintext on disk, which is exactly what this design avoids.
+None of the current services need it — all are file trees, all stopped for the
+dump. A future one that does should use two role calls.
+
+## Everything here is sqlite, so everything stops
+
+All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files
+present). A live copy of a WAL-mode database can be torn or stale, so each is
+stopped for the duration. Measured downtime: under a second for headscale and
+memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos
+and database. That last one is the real cost of a consistent snapshot; if it
+becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy,
+not skipping the stop.
+
 ## The trap is the reason this role exists
 
 When `backup_source_stop_service` is set, the script stops the unit and installs
diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml
index 77d17e0..7de6006 100644
--- a/ansible/roles/backup_source/defaults/main.yml
+++ b/ansible/roles/backup_source/defaults/main.yml
@@ -17,8 +17,12 @@ backup_source_recipient: "{{ age_backup_recipient }}"
 backup_source_pull_user: backup-pull
 backup_source_pull_key: "{{ backup_pull_public_key }}"
 
-# Safety
-backup_source_stop_service: ""          # local unit stopped for the dump, restored by a trap
+# Safety. Give either a systemd unit, or an explicit pair of commands for
+# services that are not systemd-managed (vaultwarden is a docker compose stack).
+# Whichever is used, a trap guarantees the restart.
+backup_source_stop_service: ""          # systemd unit stopped for the dump
+backup_source_stop_command: ""          # overrides stop_service when set
+backup_source_start_command: ""         # required alongside stop_command
 
 # Retention here is LOCAL and short; small-backups-box keeps the long tail.
 backup_source_retention_days: 7
diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml
index 54d6ac1..4283b84 100644
--- a/ansible/roles/backup_source/tasks/main.yml
+++ b/ansible/roles/backup_source/tasks/main.yml
@@ -7,9 +7,11 @@
       - backup_source_dump_command | length > 0
       - backup_source_recipient | length > 0
       - backup_source_recipient is match('^age1[0-9a-z]{58}$')
+      - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0)
     fail_msg: >-
       backup_source: '{{ backup_source_name | default("<unnamed>") }}' needs a name,
       description, dump command and a valid age recipient (age1... 62 chars).
+      backup_source_stop_command must be paired with backup_source_start_command.
     quiet: true
 
 # Declared here rather than assumed. Stage 1 installed it by hand; this is what
diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2
index 35e4398..18fabd6 100644
--- a/ansible/roles/backup_source/templates/backup.sh.j2
+++ b/ansible/roles/backup_source/templates/backup.sh.j2
@@ -11,8 +11,10 @@ RETENTION_DAYS={{ backup_source_retention_days }}
 RECIPIENT="{{ backup_source_recipient }}"
 SUFFIX="{{ backup_source_artifact_suffix }}"
 NAME="{{ backup_source_name }}"
-{% if backup_source_stop_service %}
-SERVICE="{{ backup_source_stop_service }}"
+{% if backup_source_stop_service or backup_source_stop_command %}
+STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }}
+START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }}
+SERVICE="{{ backup_source_stop_service or backup_source_description }}"   # label for the log only
 {% endif %}
 
 TIMESTAMP=$(date +%Y%m%d_%H%M%S)
@@ -39,13 +41,13 @@ chmod 700 "$BACKUP_DIR"
 # here or they accumulate forever.
 rm -f "${BACKUP_DIR}/${NAME}_"*.partial
 
-{% if backup_source_stop_service %}
+{% if backup_source_stop_service or backup_source_stop_command %}
 # --- Stop the service, and guarantee it comes back ---
 # The trap is the point: without it a failed dump leaves the service down until
 # the next timer fires. Every hand-written script this replaced had that bug.
 log "Stopping ${SERVICE}..."
-systemctl stop "$SERVICE"
-trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT
+eval "$STOP_CMD"
+trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT
 {% endif %}
 
 # --- Dump straight into age; plaintext never touches the disk ---
diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml
new file mode 100644
index 0000000..d6aafef
--- /dev/null
+++ b/ansible/services/forgejo/setup_backup_forgejo.yml
@@ -0,0 +1,25 @@
+---
+# Forgejo backup: dumps locally on vipy, encrypted with age.
+#
+# The biggest artefact in the estate (~2.7 G) and the reason retention here is
+# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14.
+# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the
+# old job did the same.
+- name: Configure the Forgejo backup on the edge host
+  hosts: edge
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./forgejo_vars.yml
+
+  tasks:
+    - name: Ensure Forgejo dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: forgejo
+        backup_source_description: "Forgejo"
+        backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo"
+        backup_source_stop_service: forgejo
+        backup_source_retention_days: 2
+        backup_source_on_calendar: "*-*-* 02:30:00"
diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml
new file mode 100644
index 0000000..46abf31
--- /dev/null
+++ b/ansible/services/lnbits/setup_backup_lnbits.yml
@@ -0,0 +1,25 @@
+---
+# LNBits backup: dumps locally on vipy, encrypted with age.
+# The old job produced TWO gpg artefacts (data, then .env separately). They are
+# folded into one tar here so the wallet database and the .env that configures
+# it are always the same point in time; two artefacts written by two runs can
+# drift. Pulling one file back out needs no unpacking:
+#   age -d -i <identity> <artefact> | tar -xzO opt/lnbits/lnbits/.env
+- name: Configure the LNBits backup on the edge host
+  hosts: edge
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./lnbits_vars.yml
+
+  tasks:
+    - name: Ensure LNBits dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: lnbits
+        backup_source_description: "LNBits"
+        backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env"
+        backup_source_stop_service: lnbits
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:20:00"
diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml
new file mode 100644
index 0000000..2131924
--- /dev/null
+++ b/ansible/services/memos/setup_backup_memos.yml
@@ -0,0 +1,26 @@
+---
+# Memos backup: dumps locally on memos-box, encrypted with age.
+# Replaces the lapy pull, which had been writing EMPTY directories since
+# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned
+# to a different machine that has no rsync.
+- name: Configure the Memos backup on its own host
+  hosts: memos
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./memos_vars.yml
+
+  tasks:
+    - name: Ensure Memos dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: memos
+        backup_source_description: "Memos"
+        backup_source_dump_command: "tar -czf - -C / var/lib/memos"
+        # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a
+        # consistent database rather than a torn mid-write copy. The old rsync
+        # job did not stop it.
+        backup_source_stop_service: memos
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:00:00"
diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml
new file mode 100644
index 0000000..8430475
--- /dev/null
+++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml
@@ -0,0 +1,25 @@
+---
+# Vaultwarden backup: dumps locally on vipy, encrypted with age.
+# Previously rsynced to lapy in the CLEAR; the artefact now never exists
+# unencrypted, on disk or on the wire.
+- name: Configure the Vaultwarden backup on the edge host
+  hosts: edge
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./vaultwarden_vars.yml
+
+  tasks:
+    - name: Ensure Vaultwarden dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: vaultwarden
+        backup_source_description: "Vaultwarden"
+        backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data"
+        # Not systemd — a docker compose stack — so stop/start explicitly.
+        # sqlite in WAL mode, hence stopping at all.
+        backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop"
+        backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start"
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:10:00"