backup stuff

ansible/playbooks/backups.yml

@@ -0,0 +1,16 @@
+- name: Configure the offsite backup pull
+  hosts: backup_store
+  gather_facts: yes
+
+  tasks:
+    - name: Ensure the box pulls every source on a timer
+      ansible.builtin.include_role:
+        name: backup_store
+      vars:
+        backup_store_sources:
+          - name: arbret
+            source: "arbret@prd-arbret:/opt/arbret/backups/"
+            retention_days: 90
+          - name: headscale
+            source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
+            retention_days: 90

ansible/roles/backup_source/README.md

@@ -0,0 +1,76 @@
+# `backup_source`
+
+Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a
+local directory, prune, on a systemd timer. `small-backups-box` pulls the
+directory later (see `backup_store`).
+
+Modelled on `prd-arbret`, which has been doing exactly this correctly since
+before the rest of the estate was migrated.
+
+## Usage
+
+```yaml
+- ansible.builtin.include_role:
+    name: backup_source
+  vars:
+    backup_source_name: headscale
+    backup_source_description: "Headscale"
+    backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale"
+    backup_source_stop_service: headscale
+    backup_source_retention_days: 7
+```
+
+Produces `/opt/backups/headscale/headscale_<YYYYmmdd_HHMMSS>.tar.gz.age`,
+`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`.
+
+## Why the source encrypts, not the destination
+
+`age -r <recipient>` is asymmetric and the host holds only the **public** key, so
+a compromised host cannot read its own backups — or anyone else's. The scripts
+this replaces encrypted with GPG *on the laptop, after the data had already
+crossed the network*, which protects the artefact at rest but not in transit.
+
+The matching identity lives only on lapy and is escrowed. **Lose it and every
+artefact everywhere becomes noise**, including arbret's.
+
+## `backup_source_dump_command` writes to STDOUT
+
+The role pipes it into `age`, so plaintext never touches the disk. Use `-C /`
+with relative paths in `tar` rather than absolute ones: it avoids tar's "removing
+leading /" and makes the restore target explicit.
+
+## The trap is the reason this role exists
+
+When `backup_source_stop_service` is set, the script stops the unit and installs
+an EXIT trap that starts it again. Without it, a failed dump leaves the service
+down until the next timer fires — **every hand-written script this replaced had
+that bug**, and it was only ever masked because their `systemctl stop` failed
+first, before anything was stopped.
+
+Verified on spacey: with the dump forced to fail, the log shows
+`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit
+failed rather than hiding it), and headscale is `active` afterwards.
+
+If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed —
+which is correct, because nothing was stopped.
+
+## `.partial`
+
+The dump writes `<artifact>.partial` and only `mv`s it into place on success, so
+a truncated file is never mistaken for a backup. A failure inside the pipeline
+does leave one behind, and the prune glob cannot match it (it ends `.partial`,
+not `.tar.gz.age`), so the script clears stale partials at the **start** of each
+run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run.
+
+## `backup_source_stop_service` may be a bare name
+
+`headscale` and `headscale.service` both work. The unit template normalises it,
+because systemd rejects a bare name in `After=` with
+`Failed to add dependency ... Invalid argument` — which it logs and then ignores,
+so the unit appears to work while carrying no ordering at all.
+
+## Retention is two-tier
+
+`backup_source_retention_days` is **local** and short — these hosts are
+disk-constrained. The long tail lives on `small-backups-box`, which decides its
+own retention per source. Losing the local copy is expected and fine.

ansible/roles/backup_source/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# Required
+backup_source_name: ""                  # "headscale" -> headscale_<ts>.tar.gz.age
+backup_source_description: ""           # "Headscale"
+backup_source_dump_command: ""          # must write the payload to STDOUT
+
+# Placement
+backup_source_dir: "/opt/backups/{{ backup_source_name }}"
+backup_source_artifact_suffix: "tar.gz.age"
+
+# Encryption. Asymmetric: the host holds only the public key and cannot decrypt
+# what it produces.
+backup_source_recipient: "{{ age_backup_recipient }}"
+
+# The unprivileged account small-backups-box pulls as. It owns the dump
+# directory and nothing else; it deliberately has no sudo.
+backup_source_pull_user: backup-pull
+backup_source_pull_key: "{{ backup_pull_public_key }}"
+
+# Safety
+backup_source_stop_service: ""          # local unit stopped for the dump, restored by a trap
+
+# Retention here is LOCAL and short; small-backups-box keeps the long tail.
+backup_source_retention_days: 7
+
+# Schedule. The box pulls at 04:00, so dumps must land before that.
+backup_source_on_calendar: "*-*-* 02:00:00"

ansible/roles/backup_source/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd for backup units
+  ansible.builtin.systemd:
+    daemon_reload: yes

ansible/roles/backup_source/tasks/main.yml

@@ -0,0 +1,89 @@
+---
+- name: Assert backup_source parameters are sane
+  ansible.builtin.assert:
+    that:
+      - backup_source_name | length > 0
+      - backup_source_description | length > 0
+      - backup_source_dump_command | length > 0
+      - backup_source_recipient | length > 0
+      - backup_source_recipient is match('^age1[0-9a-z]{58}$')
+    fail_msg: >-
+      backup_source: '{{ backup_source_name | default("<unnamed>") }}' needs a name,
+      description, dump command and a valid age recipient (age1... 62 chars).
+    quiet: true
+
+# Declared here rather than assumed. Stage 1 installed it by hand; this is what
+# makes a rebuilt host get it too.
+- name: Ensure age is installed
+  ansible.builtin.apt:
+    name: age
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
+# The pull account: unprivileged, no sudo, exists only so small-backups-box can
+# read the dump directory. Trust points one way — the box can read backups, and
+# can do nothing else on this host.
+- name: "Ensure the {{ backup_source_pull_user }} account exists"
+  ansible.builtin.user:
+    name: "{{ backup_source_pull_user }}"
+    system: yes
+    shell: /bin/sh          # rsync-over-ssh needs a shell; nologin breaks it
+    home: "/var/lib/{{ backup_source_pull_user }}"
+    create_home: yes
+    password: '!'           # no password login, ever
+  when: backup_source_pull_user | length > 0
+
+- name: "Authorise the backup box's key for {{ backup_source_pull_user }}"
+  ansible.posix.authorized_key:
+    user: "{{ backup_source_pull_user }}"
+    key: "{{ backup_source_pull_key }}"
+    key_options: "restrict"   # no pty, no forwarding, no user rc
+    exclusive: yes
+    state: present
+  when: backup_source_pull_user | length > 0
+
+# The shared container above the per-service directories. It must be traversable
+# or the pull account cannot reach its own directory. The script's `mkdir -p`
+# runs under `umask 077` and would otherwise create this 0700.
+- name: "Ensure {{ backup_source_dir | dirname }} is traversable"
+  ansible.builtin.file:
+    path: "{{ backup_source_dir | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: "Ensure {{ backup_source_dir }} exists"
+  ansible.builtin.file:
+    path: "{{ backup_source_dir }}"
+    state: directory
+    owner: root
+    group: "{{ backup_source_pull_user | default('root', true) }}"
+    mode: '0750'
+
+- name: "Install the {{ backup_source_name }} backup script"
+  ansible.builtin.template:
+    src: backup.sh.j2
+    dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh"
+    owner: root
+    group: root
+    mode: '0750'
+    validate: "bash -n %s"
+
+- name: "Install the {{ backup_source_name }}-backup systemd units"
+  ansible.builtin.template:
+    src: "backup.{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop: [service, timer]
+  notify: Reload systemd for backup units
+
+- name: "Enable the {{ backup_source_name }}-backup timer"
+  ansible.builtin.systemd:
+    name: "{{ backup_source_name }}-backup.timer"
+    enabled: yes
+    state: started
+    daemon_reload: yes

ansible/roles/backup_source/templates/backup.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description={{ backup_source_description }} backup
+{% if backup_source_stop_service %}
+{# systemd rejects a bare name here ("Failed to add dependency ... Invalid
+   argument"), so normalise to a full unit name. #}
+After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }}
+{% endif %}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier={{ backup_source_name }}-backup

ansible/roles/backup_source/templates/backup.sh.j2

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source)
+#
+# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}.
+# The host holds only the age PUBLIC key, so it cannot read its own backups.
+set -euo pipefail
+umask 077
+
+BACKUP_DIR="{{ backup_source_dir }}"
+RETENTION_DAYS={{ backup_source_retention_days }}
+RECIPIENT="{{ backup_source_recipient }}"
+SUFFIX="{{ backup_source_artifact_suffix }}"
+NAME="{{ backup_source_name }}"
+{% if backup_source_stop_service %}
+SERVICE="{{ backup_source_stop_service }}"
+{% endif %}
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}"
+
+die() { echo "FATAL: $*" >&2; exit 1; }
+log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
+
+# --- Pre-flight ---
+[[ -n "$RECIPIENT" ]] || die "no age recipient configured"
+command -v age >/dev/null || die "age is not installed"
+
+# Mode must agree with what the role sets, or each undoes the other every run.
+mkdir -p "$BACKUP_DIR"
+{% if backup_source_pull_user %}
+chown root:{{ backup_source_pull_user }} "$BACKUP_DIR"
+chmod 750 "$BACKUP_DIR"
+{% else %}
+chmod 700 "$BACKUP_DIR"
+{% endif %}
+
+# A run that died mid-dump leaves a .partial. It is not a backup, and the prune
+# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them
+# here or they accumulate forever.
+rm -f "${BACKUP_DIR}/${NAME}_"*.partial
+
+{% if backup_source_stop_service %}
+# --- Stop the service, and guarantee it comes back ---
+# The trap is the point: without it a failed dump leaves the service down until
+# the next timer fires. Every hand-written script this replaced had that bug.
+log "Stopping ${SERVICE}..."
+systemctl stop "$SERVICE"
+trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT
+{% endif %}
+
+# --- Dump straight into age; plaintext never touches the disk ---
+log "Writing ${ARTIFACT}..."
+{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial"
+mv "${ARTIFACT}.partial" "$ARTIFACT"
+{% if backup_source_pull_user %}
+# Readable by the pull account and nobody else. The contents are age-encrypted
+# regardless, so this is depth rather than the actual protection.
+chown root:{{ backup_source_pull_user }} "$ARTIFACT"
+chmod 640 "$ARTIFACT"
+{% else %}
+chmod 600 "$ARTIFACT"
+{% endif %}
+log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))"
+
+# --- Prune ---
+log "Pruning local artefacts older than ${RETENTION_DAYS} days..."
+find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete
+
+log "Done."

ansible/roles/backup_source/templates/backup.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description={{ backup_source_description }} backup
+
+[Timer]
+OnCalendar={{ backup_source_on_calendar }}
+# Persistent: a window missed while the host was down runs on next boot. cron on
+# a laptop had no equivalent, which is how two backups went unnoticed for months.
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/backup_store/README.md

@@ -0,0 +1,55 @@
+# `backup_store`
+
+Pulls already-encrypted backup artefacts from every source host onto
+`small-backups-box`, on a timer, and expires them per source.
+
+Generalises the hand-written `pull-backups.sh` that had one hardcoded source
+(`arbret`). That job's behaviour is preserved exactly: same source path, same
+90 days, same destination directory.
+
+## This host holds no key
+
+Everything pulled here is ciphertext produced by `backup_source` on the source
+host. The box cannot read any of it — the age identity lives only on lapy. That
+is deliberate: the machine holding every backup should not also be able to open
+them.
+
+## One failing source must not stop the others
+
+The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own
+function, failures are counted, and the script exits non-zero at the end so
+systemd marks the unit failed. A dead host costs you that one source, not the
+whole run.
+
+This is the specific failure the whole plan exists to prevent: the laptop jobs
+aborted on first error and then silently produced empty directories for nine
+months.
+
+## Trust points one way
+
+The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated
+account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised
+with `restrict`. That account can read one directory and do nothing else — no
+sudo, no pty, no forwarding. A compromised backup box cannot reach into
+production.
+
+## Addressing: names, never IPs
+
+Sources are addressed by name. The job this replaced hardcoded spacey's IP; the
+droplet was later rebuilt, the address was recycled to a stranger, and the
+backup failed silently from 2025-12-01 while the directory listing still looked
+healthy.
+
+Two kinds of name are in play:
+
+- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a
+  headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without
+  it the box cannot even resolve the peer, let alone reach it.
+- **spacey** is *not* a tailnet member — it is the headscale control server — so
+  its backup is pulled over the public internet via `headscale.contrapeso.xyz`,
+  which follows the host if the droplet is rebuilt.
+
+## Retention here is the long tail
+
+Sources keep a few days locally; this box keeps 90 (or whatever the source entry
+says). Losing the source's local copy is expected.

ansible/roles/backup_store/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+backup_store_dir: "{{ ansible_env.HOME }}/backups"
+backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull"
+backup_store_on_calendar: "*-*-* 04:00:00"
+
+# One entry per source. `retention_days` is the LONG tail; the source keeps its
+# own short local retention.
+#   - name: headscale
+#     source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
+#     retention_days: 90
+backup_store_sources: []

ansible/roles/backup_store/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reload systemd for pull-backups
+  ansible.builtin.systemd:
+    daemon_reload: yes
+  become: yes

ansible/roles/backup_store/tasks/main.yml

@@ -0,0 +1,53 @@
+---
+- name: Assert backup_store sources are sane
+  ansible.builtin.assert:
+    that:
+      - backup_store_sources | length > 0
+      - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length
+      - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length
+      - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length
+    fail_msg: "backup_store: every source needs name, source and retention_days"
+    quiet: true
+
+- name: Ensure rsync is installed
+  ansible.builtin.apt:
+    name: rsync
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+  become: yes
+
+- name: Ensure the backup store directory exists
+  ansible.builtin.file:
+    path: "{{ backup_store_dir }}"
+    state: directory
+    mode: '0700'
+
+- name: Install the pull-backups script
+  ansible.builtin.template:
+    src: pull-backups.sh.j2
+    dest: /usr/local/bin/pull-backups.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+  become: yes
+
+- name: Install the pull-backups systemd units
+  ansible.builtin.template:
+    src: "pull-backups.{{ item }}.j2"
+    dest: "/etc/systemd/system/pull-backups.{{ item }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop: [service, timer]
+  become: yes
+  notify: Reload systemd for pull-backups
+
+- name: Enable the pull-backups timer
+  ansible.builtin.systemd:
+    name: pull-backups.timer
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  become: yes

ansible/roles/backup_store/templates/pull-backups.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Pull encrypted backups from production
+
+[Service]
+Type=oneshot
+User={{ ansible_user_id }}
+ExecStart=/usr/local/bin/pull-backups.sh
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=pull-backups

ansible/roles/backup_store/templates/pull-backups.sh.j2

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Pull encrypted backups from production — managed by Ansible (roles/backup_store)
+#
+# Everything here is already ciphertext: this host only moves and expires files,
+# and holds no key that can read them.
+set -uo pipefail          # deliberately NOT -e; see the loop below
+
+SSH_KEY="{{ backup_store_ssh_key }}"
+STORE="{{ backup_store_dir }}"
+
+log()  { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
+fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); }
+
+failures=0
+
+# One source failing must not stop the others. The whole point of this box is
+# that a single dead host cannot silently take the rest of the backups with it —
+# which is exactly how the laptop-based jobs failed unnoticed for nine months.
+{% for src in backup_store_sources %}
+# --- {{ src.name }} ---
+pull_{{ src.name | replace('-', '_') }}() {
+  local dir="${STORE}/{{ src.name }}"
+  mkdir -p "$dir"
+  log "Pulling {{ src.name }} from {{ src.source }}..."
+  if rsync -az --timeout=120 \
+       -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
+       "{{ src.source }}" "$dir/"; then
+    log "  {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"
+  else
+    fail "{{ src.name }}: rsync failed"
+    return 1
+  fi
+  log "  {{ src.name }}: pruning older than {{ src.retention_days }} days"
+  find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete
+}
+pull_{{ src.name | replace('-', '_') }} || true
+
+{% endfor %}
+if [ "$failures" -gt 0 ]; then
+  log "FAILED: $failures source(s) did not pull"
+  exit 1
+fi
+log "All sources pulled."

ansible/roles/backup_store/templates/pull-backups.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily offsite backup pull
+
+[Timer]
+OnCalendar={{ backup_store_on_calendar }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/services/headscale/setup_backup_headscale.yml

@@ -0,0 +1,21 @@
+---
+- name: Configure the Headscale backup on the vpn_control host
+  hosts: vpn_control
+  become: yes
+  vars_files:
+    - ../../group_vars/all/main.yml
+    - ./headscale_vars.yml
+
+  tasks:
+    - name: Ensure Headscale dumps itself, encrypted, on a timer
+      ansible.builtin.include_role:
+        name: backup_source
+      vars:
+        backup_source_name: headscale
+        backup_source_description: "Headscale"
+        # -C / with relative paths: avoids tar's "removing leading /" and makes
+        # the restore target explicit.
+        backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale"
+        backup_source_stop_service: headscale
+        backup_source_retention_days: 7
+        backup_source_on_calendar: "*-*-* 02:00:00"