backup stuff

ansible/roles/backup_store/README.md

@@ -0,0 +1,55 @@
+# `backup_store`
+
+Pulls already-encrypted backup artefacts from every source host onto
+`small-backups-box`, on a timer, and expires them per source.
+
+Generalises the hand-written `pull-backups.sh` that had one hardcoded source
+(`arbret`). That job's behaviour is preserved exactly: same source path, same
+90 days, same destination directory.
+
+## This host holds no key
+
+Everything pulled here is ciphertext produced by `backup_source` on the source
+host. The box cannot read any of it — the age identity lives only on lapy. That
+is deliberate: the machine holding every backup should not also be able to open
+them.
+
+## One failing source must not stop the others
+
+The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own
+function, failures are counted, and the script exits non-zero at the end so
+systemd marks the unit failed. A dead host costs you that one source, not the
+whole run.
+
+This is the specific failure the whole plan exists to prevent: the laptop jobs
+aborted on first error and then silently produced empty directories for nine
+months.
+
+## Trust points one way
+
+The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated
+account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised
+with `restrict`. That account can read one directory and do nothing else — no
+sudo, no pty, no forwarding. A compromised backup box cannot reach into
+production.
+
+## Addressing: names, never IPs
+
+Sources are addressed by name. The job this replaced hardcoded spacey's IP; the
+droplet was later rebuilt, the address was recycled to a stranger, and the
+backup failed silently from 2025-12-01 while the directory listing still looked
+healthy.
+
+Two kinds of name are in play:
+
+- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a
+  headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without
+  it the box cannot even resolve the peer, let alone reach it.
+- **spacey** is *not* a tailnet member — it is the headscale control server — so
+  its backup is pulled over the public internet via `headscale.contrapeso.xyz`,
+  which follows the host if the droplet is rebuilt.
+
+## Retention here is the long tail
+
+Sources keep a few days locally; this box keeps 90 (or whatever the source entry
+says). Losing the source's local copy is expected.

ansible/roles/backup_store/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+backup_store_dir: "{{ ansible_env.HOME }}/backups"
+backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull"
+backup_store_on_calendar: "*-*-* 04:00:00"
+
+# One entry per source. `retention_days` is the LONG tail; the source keeps its
+# own short local retention.
+#   - name: headscale
+#     source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
+#     retention_days: 90
+backup_store_sources: []

ansible/roles/backup_store/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reload systemd for pull-backups
+  ansible.builtin.systemd:
+    daemon_reload: yes
+  become: yes

ansible/roles/backup_store/tasks/main.yml

@@ -0,0 +1,53 @@
+---
+- name: Assert backup_store sources are sane
+  ansible.builtin.assert:
+    that:
+      - backup_store_sources | length > 0
+      - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length
+      - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length
+      - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length
+    fail_msg: "backup_store: every source needs name, source and retention_days"
+    quiet: true
+
+- name: Ensure rsync is installed
+  ansible.builtin.apt:
+    name: rsync
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+  become: yes
+
+- name: Ensure the backup store directory exists
+  ansible.builtin.file:
+    path: "{{ backup_store_dir }}"
+    state: directory
+    mode: '0700'
+
+- name: Install the pull-backups script
+  ansible.builtin.template:
+    src: pull-backups.sh.j2
+    dest: /usr/local/bin/pull-backups.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+  become: yes
+
+- name: Install the pull-backups systemd units
+  ansible.builtin.template:
+    src: "pull-backups.{{ item }}.j2"
+    dest: "/etc/systemd/system/pull-backups.{{ item }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop: [service, timer]
+  become: yes
+  notify: Reload systemd for pull-backups
+
+- name: Enable the pull-backups timer
+  ansible.builtin.systemd:
+    name: pull-backups.timer
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  become: yes

ansible/roles/backup_store/templates/pull-backups.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Pull encrypted backups from production
+
+[Service]
+Type=oneshot
+User={{ ansible_user_id }}
+ExecStart=/usr/local/bin/pull-backups.sh
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=pull-backups

ansible/roles/backup_store/templates/pull-backups.sh.j2

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Pull encrypted backups from production — managed by Ansible (roles/backup_store)
+#
+# Everything here is already ciphertext: this host only moves and expires files,
+# and holds no key that can read them.
+set -uo pipefail          # deliberately NOT -e; see the loop below
+
+SSH_KEY="{{ backup_store_ssh_key }}"
+STORE="{{ backup_store_dir }}"
+
+log()  { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }
+fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); }
+
+failures=0
+
+# One source failing must not stop the others. The whole point of this box is
+# that a single dead host cannot silently take the rest of the backups with it —
+# which is exactly how the laptop-based jobs failed unnoticed for nine months.
+{% for src in backup_store_sources %}
+# --- {{ src.name }} ---
+pull_{{ src.name | replace('-', '_') }}() {
+  local dir="${STORE}/{{ src.name }}"
+  mkdir -p "$dir"
+  log "Pulling {{ src.name }} from {{ src.source }}..."
+  if rsync -az --timeout=120 \
+       -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
+       "{{ src.source }}" "$dir/"; then
+    log "  {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"
+  else
+    fail "{{ src.name }}: rsync failed"
+    return 1
+  fi
+  log "  {{ src.name }}: pruning older than {{ src.retention_days }} days"
+  find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete
+}
+pull_{{ src.name | replace('-', '_') }} || true
+
+{% endfor %}
+if [ "$failures" -gt 0 ]; then
+  log "FAILED: $failures source(s) did not pull"
+  exit 1
+fi
+log "All sources pulled."

ansible/roles/backup_store/templates/pull-backups.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily offsite backup pull
+
+[Timer]
+OnCalendar={{ backup_store_on_calendar }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target