brushing up backups

ansible/roles/backup_source/tasks/main.yml

@@ -16,12 +16,22 @@
 
 # Declared here rather than assumed. Stage 1 installed it by hand; this is what
 # makes a rebuilt host get it too.
+# Cache refresh is best-effort on purpose. An unrelated third-party repo with a
+# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource
+# key) makes `apt-get update` return warnings, which the apt module treats as a
+# hard failure — and that must not stop backups being configured. Installing the
+# package is NOT best-effort: if age is genuinely unavailable, the next task fails.
+- name: Refresh the apt cache (best effort)
+  ansible.builtin.apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  failed_when: false
+  changed_when: false
+
 - name: Ensure age is installed
   ansible.builtin.apt:
     name: age
     state: present
-    update_cache: yes
-    cache_valid_time: 3600
 
 # The pull account: unprivileged, no sudo, exists only so small-backups-box can
 # read the dump directory. Trust points one way — the box can read backups, and

ansible/roles/backup_source/templates/backup.sh.j2

@@ -53,6 +53,12 @@ trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT
 # --- Dump straight into age; plaintext never touches the disk ---
 log "Writing ${ARTIFACT}..."
 {{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial"
+{% if backup_source_pull_user %}
+# Match the final ownership immediately, so even a partial left by a later
+# failure is not an unreadable obstacle to the pull.
+chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial"
+chmod 640 "${ARTIFACT}.partial"
+{% endif %}
 mv "${ARTIFACT}.partial" "$ARTIFACT"
 {% if backup_source_pull_user %}
 # Readable by the pull account and nobody else. The contents are age-encrypted

ansible/roles/backup_store/tasks/main.yml

@@ -33,6 +33,19 @@
     validate: "bash -n %s"
   become: yes
 
+# A human-run assertion that last night actually worked. Generated from the same
+# source list as the puller, so it can never drift out of sync with what is
+# supposed to be arriving.
+- name: Install the backup check script
+  ansible.builtin.template:
+    src: check-backups.sh.j2
+    dest: /usr/local/bin/check-backups.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+  become: yes
+
 - name: Install the pull-backups systemd units
   ansible.builtin.template:
     src: "pull-backups.{{ item }}.j2"

ansible/roles/backup_store/templates/check-backups.sh.j2

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+# Assert the nightly backups actually worked.
+#
+# Run as {{ ansible_user_id }} on this host. Needs no sudo.
+#
+# What it CANNOT do: verify contents. The age identity lives only on lapy, so
+# this host cannot decrypt anything it holds — by design. These are freshness,
+# completeness and integrity checks. To verify content, decrypt on lapy:
+#     ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups/<svc>/<artefact>" \
+#       | age -d -i ~/.age/counterweight_age | tar -tzf - | head
+#
+# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL.
+set -uo pipefail
+
+STORE="{{ backup_store_dir }}"
+MAX_AGE_H="${1:-26}"          # an artefact older than this is stale
+NOW=$(date +%s)
+fails=0; warns=0
+
+# Colour only when attached to a terminal: this gets piped into files and, later,
+# probably into a notification.
+if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m'
+else R=''; Y=''; G=''; N=''; fi
+
+red()  { printf '  %sFAIL%s  %s\n' "$R" "$N" "$*"; fails=$((fails+1)); }
+yell() { printf '  %sWARN%s  %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); }
+ok()   { printf '  %sok%s    %s\n' "$G" "$N" "$*"; }
+
+hours_since() { echo $(( (NOW - $1) / 3600 )); }
+
+# Pull the dump timestamp out of <name>_YYYYmmdd_HHMMSS.<suffix>. This is when
+# the SOURCE produced it, which is the thing that actually matters: a source
+# whose timer died still pulls "ok" forever, because yesterday's artefact is
+# still sitting there. Checking only the pull would miss exactly that.
+dump_epoch() {
+  local base ts
+  base=$(basename "$1")
+  ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1
+  [ -n "$ts" ] || return 1
+  date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null
+}
+
+check_source() {
+  local name="$1" keep="$2" dir="$STORE/$1"
+  printf '\n%s\n' "== $name"
+
+  [ -d "$dir" ] || { red "$name: no directory $dir"; return; }
+
+  local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l)
+  [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; }
+
+  local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l)
+  [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these"
+
+  local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1)
+  local prev;   prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p)
+
+  # 1. Did the SOURCE dump recently?
+  local de; de=$(dump_epoch "$newest")
+  if [ -z "${de:-}" ]; then
+    yell "$name: cannot parse a dump timestamp from $(basename "$newest")"
+  else
+    local dh; dh=$(hours_since "$de")
+    if [ "$dh" -lt 0 ]; then
+      # A future-dated artefact would otherwise stay "fresh" forever and the
+      # staleness check would never fire again — the exact silent failure this
+      # script exists to catch.
+      red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?"
+    elif [ "$dh" -gt "$MAX_AGE_H" ]; then
+      red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run"
+    else
+      ok "$name: dumped ${dh}h ago"
+    fi
+  fi
+
+  # 2. Did the PULL bring it over recently?
+  local ph; ph=$(hours_since "$(stat -c %Y "$newest")")
+  if [ "$ph" -gt "$MAX_AGE_H" ]; then
+    red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)"
+  else
+    ok "$name: pulled ${ph}h ago"
+  fi
+
+  # 3. Is it plausibly a real backup?
+  local sz; sz=$(stat -c %s "$newest")
+  if [ "$sz" -eq 0 ]; then
+    red "$name: newest artefact is ZERO bytes"
+  elif [ -n "$prev" ]; then
+    local psz; psz=$(stat -c %s "$prev")
+    if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then
+      # Not automatically wrong: headscale legitimately shrank 297K -> 20K when
+      # a clean stop checkpointed its write-ahead log into the database.
+      yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect"
+    else
+      ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)"
+    fi
+  else
+    ok "$name: $(numfmt --to=iec "$sz") (first artefact)"
+  fi
+
+  # 4. Is retention pruning? Allow generous slack for multiple dumps per day.
+  if [ "$n" -gt $(( keep * 3 + 10 )) ]; then
+    yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working"
+  fi
+}
+
+echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')"
+echo "Artefacts older than ${MAX_AGE_H}h are treated as stale."
+
+# --- the pull job itself ---
+printf '\n%s\n' "== pull-backups.service"
+result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null)
+status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null)
+when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null)
+[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})"
+if [ -n "$when" ]; then
+  wh=$(hours_since "$(date -d "$when" +%s)")
+  [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?"
+fi
+systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \
+  && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \
+  || red "pull-backups.timer is NOT enabled"
+
+# --- each source ---
+{% for src in backup_store_sources %}
+check_source "{{ src.name }}" {{ src.retention_days }}
+{% endfor %}
+
+# --- capacity ---
+printf '\n%s\n' "== disk"
+use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9')
+avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ')
+if   [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free"
+elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free"
+else ok "store is ${use}% full, ${avail} free"; fi
+
+printf '\n%s\n' "-----"
+if [ "$fails" -gt 0 ]; then
+  echo "RESULT: $fails failure(s), $warns warning(s)"
+  echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager"
+  exit 1
+fi
+if [ "$warns" -gt 0 ]; then
+  echo "RESULT: all checks passed, $warns warning(s)"
+else
+  echo "RESULT: all checks passed"
+fi
+exit 0

ansible/roles/backup_store/templates/pull-backups.sh.j2

@@ -22,7 +22,12 @@ pull_{{ src.name | replace('-', '_') }}() {
   local dir="${STORE}/{{ src.name }}"
   mkdir -p "$dir"
   log "Pulling {{ src.name }} from {{ src.source }}..."
-  if rsync -az --timeout=120 \
+  # --exclude '*.partial': a dump that died mid-write leaves one behind, owned
+  # root:root 0600 because the chown only happens after a successful mv. Without
+  # this exclude the pull account cannot read it and rsync fails for the WHOLE
+  # source — so one failed dump would silently block every subsequent pull of
+  # that service. An incomplete artefact is never worth transferring anyway.
+  if rsync -az --timeout=120 --exclude '*.partial' \
        -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
        "{{ src.source }}" "$dir/"; then
     log "  {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"