diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml new file mode 100644 index 0000000..842942e --- /dev/null +++ b/ansible/playbooks/backups.yml @@ -0,0 +1,16 @@ +- name: Configure the offsite backup pull + hosts: backup_store + gather_facts: yes + + tasks: + - name: Ensure the box pulls every source on a timer + ansible.builtin.include_role: + name: backup_store + vars: + backup_store_sources: + - name: arbret + source: "arbret@prd-arbret:/opt/arbret/backups/" + retention_days: 90 + - name: headscale + source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" + retention_days: 90 diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md new file mode 100644 index 0000000..065e1fc --- /dev/null +++ b/ansible/roles/backup_source/README.md @@ -0,0 +1,76 @@ +# `backup_source` + +Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a +local directory, prune, on a systemd timer. `small-backups-box` pulls the +directory later (see `backup_store`). + +Modelled on `prd-arbret`, which has been doing exactly this correctly since +before the rest of the estate was migrated. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 +``` + +Produces `/opt/backups/headscale/headscale_.tar.gz.age`, +`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`. + +## Why the source encrypts, not the destination + +`age -r ` is asymmetric and the host holds only the **public** key, so +a compromised host cannot read its own backups — or anyone else's. The scripts +this replaces encrypted with GPG *on the laptop, after the data had already +crossed the network*, which protects the artefact at rest but not in transit. + +The matching identity lives only on lapy and is escrowed. **Lose it and every +artefact everywhere becomes noise**, including arbret's. + +## `backup_source_dump_command` writes to STDOUT + +The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` +with relative paths in `tar` rather than absolute ones: it avoids tar's "removing +leading /" and makes the restore target explicit. + +## The trap is the reason this role exists + +When `backup_source_stop_service` is set, the script stops the unit and installs +an EXIT trap that starts it again. Without it, a failed dump leaves the service +down until the next timer fires — **every hand-written script this replaced had +that bug**, and it was only ever masked because their `systemctl stop` failed +first, before anything was stopped. + +Verified on spacey: with the dump forced to fail, the log shows +`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit +failed rather than hiding it), and headscale is `active` afterwards. + +If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed — +which is correct, because nothing was stopped. + +## `.partial` + +The dump writes `.partial` and only `mv`s it into place on success, so +a truncated file is never mistaken for a backup. A failure inside the pipeline +does leave one behind, and the prune glob cannot match it (it ends `.partial`, +not `.tar.gz.age`), so the script clears stale partials at the **start** of each +run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run. + +## `backup_source_stop_service` may be a bare name + +`headscale` and `headscale.service` both work. The unit template normalises it, +because systemd rejects a bare name in `After=` with +`Failed to add dependency ... Invalid argument` — which it logs and then ignores, +so the unit appears to work while carrying no ordering at all. + +## Retention is two-tier + +`backup_source_retention_days` is **local** and short — these hosts are +disk-constrained. The long tail lives on `small-backups-box`, which decides its +own retention per source. Losing the local copy is expected and fine. diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml new file mode 100644 index 0000000..77d17e0 --- /dev/null +++ b/ansible/roles/backup_source/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# Required +backup_source_name: "" # "headscale" -> headscale_.tar.gz.age +backup_source_description: "" # "Headscale" +backup_source_dump_command: "" # must write the payload to STDOUT + +# Placement +backup_source_dir: "/opt/backups/{{ backup_source_name }}" +backup_source_artifact_suffix: "tar.gz.age" + +# Encryption. Asymmetric: the host holds only the public key and cannot decrypt +# what it produces. +backup_source_recipient: "{{ age_backup_recipient }}" + +# The unprivileged account small-backups-box pulls as. It owns the dump +# directory and nothing else; it deliberately has no sudo. +backup_source_pull_user: backup-pull +backup_source_pull_key: "{{ backup_pull_public_key }}" + +# Safety +backup_source_stop_service: "" # local unit stopped for the dump, restored by a trap + +# Retention here is LOCAL and short; small-backups-box keeps the long tail. +backup_source_retention_days: 7 + +# Schedule. The box pulls at 04:00, so dumps must land before that. +backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml new file mode 100644 index 0000000..37a7f2b --- /dev/null +++ b/ansible/roles/backup_source/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd for backup units + ansible.builtin.systemd: + daemon_reload: yes diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml new file mode 100644 index 0000000..54d6ac1 --- /dev/null +++ b/ansible/roles/backup_source/tasks/main.yml @@ -0,0 +1,89 @@ +--- +- name: Assert backup_source parameters are sane + ansible.builtin.assert: + that: + - backup_source_name | length > 0 + - backup_source_description | length > 0 + - backup_source_dump_command | length > 0 + - backup_source_recipient | length > 0 + - backup_source_recipient is match('^age1[0-9a-z]{58}$') + fail_msg: >- + backup_source: '{{ backup_source_name | default("") }}' needs a name, + description, dump command and a valid age recipient (age1... 62 chars). + quiet: true + +# Declared here rather than assumed. Stage 1 installed it by hand; this is what +# makes a rebuilt host get it too. +- name: Ensure age is installed + ansible.builtin.apt: + name: age + state: present + update_cache: yes + cache_valid_time: 3600 + +# The pull account: unprivileged, no sudo, exists only so small-backups-box can +# read the dump directory. Trust points one way — the box can read backups, and +# can do nothing else on this host. +- name: "Ensure the {{ backup_source_pull_user }} account exists" + ansible.builtin.user: + name: "{{ backup_source_pull_user }}" + system: yes + shell: /bin/sh # rsync-over-ssh needs a shell; nologin breaks it + home: "/var/lib/{{ backup_source_pull_user }}" + create_home: yes + password: '!' # no password login, ever + when: backup_source_pull_user | length > 0 + +- name: "Authorise the backup box's key for {{ backup_source_pull_user }}" + ansible.posix.authorized_key: + user: "{{ backup_source_pull_user }}" + key: "{{ backup_source_pull_key }}" + key_options: "restrict" # no pty, no forwarding, no user rc + exclusive: yes + state: present + when: backup_source_pull_user | length > 0 + +# The shared container above the per-service directories. It must be traversable +# or the pull account cannot reach its own directory. The script's `mkdir -p` +# runs under `umask 077` and would otherwise create this 0700. +- name: "Ensure {{ backup_source_dir | dirname }} is traversable" + ansible.builtin.file: + path: "{{ backup_source_dir | dirname }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: "Ensure {{ backup_source_dir }} exists" + ansible.builtin.file: + path: "{{ backup_source_dir }}" + state: directory + owner: root + group: "{{ backup_source_pull_user | default('root', true) }}" + mode: '0750' + +- name: "Install the {{ backup_source_name }} backup script" + ansible.builtin.template: + src: backup.sh.j2 + dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh" + owner: root + group: root + mode: '0750' + validate: "bash -n %s" + +- name: "Install the {{ backup_source_name }}-backup systemd units" + ansible.builtin.template: + src: "backup.{{ item }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + notify: Reload systemd for backup units + +- name: "Enable the {{ backup_source_name }}-backup timer" + ansible.builtin.systemd: + name: "{{ backup_source_name }}-backup.timer" + enabled: yes + state: started + daemon_reload: yes diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 new file mode 100644 index 0000000..91ad063 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description={{ backup_source_description }} backup +{% if backup_source_stop_service %} +{# systemd rejects a bare name here ("Failed to add dependency ... Invalid + argument"), so normalise to a full unit name. #} +After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }} +{% endif %} + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 new file mode 100644 index 0000000..35e4398 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source) +# +# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}. +# The host holds only the age PUBLIC key, so it cannot read its own backups. +set -euo pipefail +umask 077 + +BACKUP_DIR="{{ backup_source_dir }}" +RETENTION_DAYS={{ backup_source_retention_days }} +RECIPIENT="{{ backup_source_recipient }}" +SUFFIX="{{ backup_source_artifact_suffix }}" +NAME="{{ backup_source_name }}" +{% if backup_source_stop_service %} +SERVICE="{{ backup_source_stop_service }}" +{% endif %} + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}" + +die() { echo "FATAL: $*" >&2; exit 1; } +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } + +# --- Pre-flight --- +[[ -n "$RECIPIENT" ]] || die "no age recipient configured" +command -v age >/dev/null || die "age is not installed" + +# Mode must agree with what the role sets, or each undoes the other every run. +mkdir -p "$BACKUP_DIR" +{% if backup_source_pull_user %} +chown root:{{ backup_source_pull_user }} "$BACKUP_DIR" +chmod 750 "$BACKUP_DIR" +{% else %} +chmod 700 "$BACKUP_DIR" +{% endif %} + +# A run that died mid-dump leaves a .partial. It is not a backup, and the prune +# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them +# here or they accumulate forever. +rm -f "${BACKUP_DIR}/${NAME}_"*.partial + +{% if backup_source_stop_service %} +# --- Stop the service, and guarantee it comes back --- +# The trap is the point: without it a failed dump leaves the service down until +# the next timer fires. Every hand-written script this replaced had that bug. +log "Stopping ${SERVICE}..." +systemctl stop "$SERVICE" +trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT +{% endif %} + +# --- Dump straight into age; plaintext never touches the disk --- +log "Writing ${ARTIFACT}..." +{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +mv "${ARTIFACT}.partial" "$ARTIFACT" +{% if backup_source_pull_user %} +# Readable by the pull account and nobody else. The contents are age-encrypted +# regardless, so this is depth rather than the actual protection. +chown root:{{ backup_source_pull_user }} "$ARTIFACT" +chmod 640 "$ARTIFACT" +{% else %} +chmod 600 "$ARTIFACT" +{% endif %} +log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))" + +# --- Prune --- +log "Pruning local artefacts older than ${RETENTION_DAYS} days..." +find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete + +log "Done." diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2 new file mode 100644 index 0000000..d9a7e6a --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description={{ backup_source_description }} backup + +[Timer] +OnCalendar={{ backup_source_on_calendar }} +# Persistent: a window missed while the host was down runs on next boot. cron on +# a laptop had no equivalent, which is how two backups went unnoticed for months. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md new file mode 100644 index 0000000..13d9f5a --- /dev/null +++ b/ansible/roles/backup_store/README.md @@ -0,0 +1,55 @@ +# `backup_store` + +Pulls already-encrypted backup artefacts from every source host onto +`small-backups-box`, on a timer, and expires them per source. + +Generalises the hand-written `pull-backups.sh` that had one hardcoded source +(`arbret`). That job's behaviour is preserved exactly: same source path, same +90 days, same destination directory. + +## This host holds no key + +Everything pulled here is ciphertext produced by `backup_source` on the source +host. The box cannot read any of it — the age identity lives only on lapy. That +is deliberate: the machine holding every backup should not also be able to open +them. + +## One failing source must not stop the others + +The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own +function, failures are counted, and the script exits non-zero at the end so +systemd marks the unit failed. A dead host costs you that one source, not the +whole run. + +This is the specific failure the whole plan exists to prevent: the laptop jobs +aborted on first error and then silently produced empty directories for nine +months. + +## Trust points one way + +The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated +account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised +with `restrict`. That account can read one directory and do nothing else — no +sudo, no pty, no forwarding. A compromised backup box cannot reach into +production. + +## Addressing: names, never IPs + +Sources are addressed by name. The job this replaced hardcoded spacey's IP; the +droplet was later rebuilt, the address was recycled to a stranger, and the +backup failed silently from 2025-12-01 while the directory listing still looked +healthy. + +Two kinds of name are in play: + +- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a + headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without + it the box cannot even resolve the peer, let alone reach it. +- **spacey** is *not* a tailnet member — it is the headscale control server — so + its backup is pulled over the public internet via `headscale.contrapeso.xyz`, + which follows the host if the droplet is rebuilt. + +## Retention here is the long tail + +Sources keep a few days locally; this box keeps 90 (or whatever the source entry +says). Losing the source's local copy is expected. diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml new file mode 100644 index 0000000..c0c128b --- /dev/null +++ b/ansible/roles/backup_store/defaults/main.yml @@ -0,0 +1,11 @@ +--- +backup_store_dir: "{{ ansible_env.HOME }}/backups" +backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull" +backup_store_on_calendar: "*-*-* 04:00:00" + +# One entry per source. `retention_days` is the LONG tail; the source keeps its +# own short local retention. +# - name: headscale +# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" +# retention_days: 90 +backup_store_sources: [] diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml new file mode 100644 index 0000000..632be35 --- /dev/null +++ b/ansible/roles/backup_store/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload systemd for pull-backups + ansible.builtin.systemd: + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml new file mode 100644 index 0000000..d138830 --- /dev/null +++ b/ansible/roles/backup_store/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Assert backup_store sources are sane + ansible.builtin.assert: + that: + - backup_store_sources | length > 0 + - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length + fail_msg: "backup_store: every source needs name, source and retention_days" + quiet: true + +- name: Ensure rsync is installed + ansible.builtin.apt: + name: rsync + state: present + update_cache: yes + cache_valid_time: 3600 + become: yes + +- name: Ensure the backup store directory exists + ansible.builtin.file: + path: "{{ backup_store_dir }}" + state: directory + mode: '0700' + +- name: Install the pull-backups script + ansible.builtin.template: + src: pull-backups.sh.j2 + dest: /usr/local/bin/pull-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + +- name: Install the pull-backups systemd units + ansible.builtin.template: + src: "pull-backups.{{ item }}.j2" + dest: "/etc/systemd/system/pull-backups.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + become: yes + notify: Reload systemd for pull-backups + +- name: Enable the pull-backups timer + ansible.builtin.systemd: + name: pull-backups.timer + enabled: yes + state: started + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2 new file mode 100644 index 0000000..c65516c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Pull encrypted backups from production + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/pull-backups.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier=pull-backups diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 new file mode 100644 index 0000000..267900c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Pull encrypted backups from production — managed by Ansible (roles/backup_store) +# +# Everything here is already ciphertext: this host only moves and expires files, +# and holds no key that can read them. +set -uo pipefail # deliberately NOT -e; see the loop below + +SSH_KEY="{{ backup_store_ssh_key }}" +STORE="{{ backup_store_dir }}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } +fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); } + +failures=0 + +# One source failing must not stop the others. The whole point of this box is +# that a single dead host cannot silently take the rest of the backups with it — +# which is exactly how the laptop-based jobs failed unnoticed for nine months. +{% for src in backup_store_sources %} +# --- {{ src.name }} --- +pull_{{ src.name | replace('-', '_') }}() { + local dir="${STORE}/{{ src.name }}" + mkdir -p "$dir" + log "Pulling {{ src.name }} from {{ src.source }}..." + if rsync -az --timeout=120 \ + -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ + "{{ src.source }}" "$dir/"; then + log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" + else + fail "{{ src.name }}: rsync failed" + return 1 + fi + log " {{ src.name }}: pruning older than {{ src.retention_days }} days" + find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete +} +pull_{{ src.name | replace('-', '_') }} || true + +{% endfor %} +if [ "$failures" -gt 0 ]; then + log "FAILED: $failures source(s) did not pull" + exit 1 +fi +log "All sources pulled." diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2 new file mode 100644 index 0000000..336e1ca --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Daily offsite backup pull + +[Timer] +OnCalendar={{ backup_store_on_calendar }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml new file mode 100644 index 0000000..9c72d1d --- /dev/null +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -0,0 +1,21 @@ +--- +- name: Configure the Headscale backup on the vpn_control host + hosts: vpn_control + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./headscale_vars.yml + + tasks: + - name: Ensure Headscale dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + # -C / with relative paths: avoids tar's "removing leading /" and makes + # the restore target explicit. + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00"