From 2ebb2f9a64c81ac1a2ae161ba74dd81d26c58acb Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 12 Sep 2026 17:57:43 +0200 Subject: [PATCH] brushing up backups --- ansible/roles/backup_source/tasks/main.yml | 14 +- .../backup_source/templates/backup.sh.j2 | 6 + ansible/roles/backup_store/tasks/main.yml | 13 ++ .../templates/check-backups.sh.j2 | 148 ++++++++++++++++++ .../backup_store/templates/pull-backups.sh.j2 | 7 +- 5 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 ansible/roles/backup_store/templates/check-backups.sh.j2 diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index 4283b84..ec2d6cf 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -16,12 +16,22 @@ # Declared here rather than assumed. Stage 1 installed it by hand; this is what # makes a rebuilt host get it too. +# Cache refresh is best-effort on purpose. An unrelated third-party repo with a +# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource +# key) makes `apt-get update` return warnings, which the apt module treats as a +# hard failure — and that must not stop backups being configured. Installing the +# package is NOT best-effort: if age is genuinely unavailable, the next task fails. +- name: Refresh the apt cache (best effort) + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 3600 + failed_when: false + changed_when: false + - name: Ensure age is installed ansible.builtin.apt: name: age state: present - update_cache: yes - cache_valid_time: 3600 # The pull account: unprivileged, no sudo, exists only so small-backups-box can # read the dump directory. Trust points one way — the box can read backups, and diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index 18fabd6..abdadfe 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -53,6 +53,12 @@ trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT # --- Dump straight into age; plaintext never touches the disk --- log "Writing ${ARTIFACT}..." {{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +{% if backup_source_pull_user %} +# Match the final ownership immediately, so even a partial left by a later +# failure is not an unreadable obstacle to the pull. +chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial" +chmod 640 "${ARTIFACT}.partial" +{% endif %} mv "${ARTIFACT}.partial" "$ARTIFACT" {% if backup_source_pull_user %} # Readable by the pull account and nobody else. The contents are age-encrypted diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml index d138830..2596dbc 100644 --- a/ansible/roles/backup_store/tasks/main.yml +++ b/ansible/roles/backup_store/tasks/main.yml @@ -33,6 +33,19 @@ validate: "bash -n %s" become: yes +# A human-run assertion that last night actually worked. Generated from the same +# source list as the puller, so it can never drift out of sync with what is +# supposed to be arriving. +- name: Install the backup check script + ansible.builtin.template: + src: check-backups.sh.j2 + dest: /usr/local/bin/check-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + - name: Install the pull-backups systemd units ansible.builtin.template: src: "pull-backups.{{ item }}.j2" diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 new file mode 100644 index 0000000..f7a8830 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.sh.j2 @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Assert the nightly backups actually worked. +# +# Run as {{ ansible_user_id }} on this host. Needs no sudo. +# +# What it CANNOT do: verify contents. The age identity lives only on lapy, so +# this host cannot decrypt anything it holds — by design. These are freshness, +# completeness and integrity checks. To verify content, decrypt on lapy: +# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups//" \ +# | age -d -i ~/.age/counterweight_age | tar -tzf - | head +# +# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL. +set -uo pipefail + +STORE="{{ backup_store_dir }}" +MAX_AGE_H="${1:-26}" # an artefact older than this is stale +NOW=$(date +%s) +fails=0; warns=0 + +# Colour only when attached to a terminal: this gets piped into files and, later, +# probably into a notification. +if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' +else R=''; Y=''; G=''; N=''; fi + +red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); } +yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } +ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; } + +hours_since() { echo $(( (NOW - $1) / 3600 )); } + +# Pull the dump timestamp out of _YYYYmmdd_HHMMSS.. This is when +# the SOURCE produced it, which is the thing that actually matters: a source +# whose timer died still pulls "ok" forever, because yesterday's artefact is +# still sitting there. Checking only the pull would miss exactly that. +dump_epoch() { + local base ts + base=$(basename "$1") + ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1 + [ -n "$ts" ] || return 1 + date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null +} + +check_source() { + local name="$1" keep="$2" dir="$STORE/$1" + printf '\n%s\n' "== $name" + + [ -d "$dir" ] || { red "$name: no directory $dir"; return; } + + local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l) + [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; } + + local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l) + [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these" + + local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1) + local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p) + + # 1. Did the SOURCE dump recently? + local de; de=$(dump_epoch "$newest") + if [ -z "${de:-}" ]; then + yell "$name: cannot parse a dump timestamp from $(basename "$newest")" + else + local dh; dh=$(hours_since "$de") + if [ "$dh" -lt 0 ]; then + # A future-dated artefact would otherwise stay "fresh" forever and the + # staleness check would never fire again — the exact silent failure this + # script exists to catch. + red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?" + elif [ "$dh" -gt "$MAX_AGE_H" ]; then + red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run" + else + ok "$name: dumped ${dh}h ago" + fi + fi + + # 2. Did the PULL bring it over recently? + local ph; ph=$(hours_since "$(stat -c %Y "$newest")") + if [ "$ph" -gt "$MAX_AGE_H" ]; then + red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)" + else + ok "$name: pulled ${ph}h ago" + fi + + # 3. Is it plausibly a real backup? + local sz; sz=$(stat -c %s "$newest") + if [ "$sz" -eq 0 ]; then + red "$name: newest artefact is ZERO bytes" + elif [ -n "$prev" ]; then + local psz; psz=$(stat -c %s "$prev") + if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then + # Not automatically wrong: headscale legitimately shrank 297K -> 20K when + # a clean stop checkpointed its write-ahead log into the database. + yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect" + else + ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)" + fi + else + ok "$name: $(numfmt --to=iec "$sz") (first artefact)" + fi + + # 4. Is retention pruning? Allow generous slack for multiple dumps per day. + if [ "$n" -gt $(( keep * 3 + 10 )) ]; then + yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working" + fi +} + +echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" +echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." + +# --- the pull job itself --- +printf '\n%s\n' "== pull-backups.service" +result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) +status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) +when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null) +[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})" +if [ -n "$when" ]; then + wh=$(hours_since "$(date -d "$when" +%s)") + [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?" +fi +systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ + && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \ + || red "pull-backups.timer is NOT enabled" + +# --- each source --- +{% for src in backup_store_sources %} +check_source "{{ src.name }}" {{ src.retention_days }} +{% endfor %} + +# --- capacity --- +printf '\n%s\n' "== disk" +use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') +avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') +if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" +elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" +else ok "store is ${use}% full, ${avail} free"; fi + +printf '\n%s\n' "-----" +if [ "$fails" -gt 0 ]; then + echo "RESULT: $fails failure(s), $warns warning(s)" + echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager" + exit 1 +fi +if [ "$warns" -gt 0 ]; then + echo "RESULT: all checks passed, $warns warning(s)" +else + echo "RESULT: all checks passed" +fi +exit 0 diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 index 267900c..1db7d9c 100644 --- a/ansible/roles/backup_store/templates/pull-backups.sh.j2 +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -22,7 +22,12 @@ pull_{{ src.name | replace('-', '_') }}() { local dir="${STORE}/{{ src.name }}" mkdir -p "$dir" log "Pulling {{ src.name }} from {{ src.source }}..." - if rsync -az --timeout=120 \ + # --exclude '*.partial': a dump that died mid-write leaves one behind, owned + # root:root 0600 because the chown only happens after a successful mv. Without + # this exclude the pull account cannot read it and rsync fails for the WHOLE + # source — so one failed dump would silently block every subsequent pull of + # that service. An incomplete artefact is never worth transferring anyway. + if rsync -az --timeout=120 --exclude '*.partial' \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ "{{ src.source }}" "$dir/"; then log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"