personal_infra/ansible/roles/backup_store/templates/check-backups.sh.j2

#!/usr/bin/env bash
# Assert the nightly backups actually worked.
#
# Run as {{ ansible_user_id }} on this host. Needs no sudo.
#
# What it CANNOT do: verify contents. The age identity lives only on lapy, so
# this host cannot decrypt anything it holds — by design. These are freshness,
# completeness and integrity checks. To verify content, decrypt on lapy:
#     ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups/<svc>/<artefact>" \
#       | age -d -i ~/.age/counterweight_age | tar -tzf - | head
#
# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL.
set -uo pipefail

STORE="{{ backup_store_dir }}"
MAX_AGE_H="${1:-26}"          # an artefact older than this is stale
NOW=$(date +%s)
fails=0; warns=0

# Colour only when attached to a terminal: this gets piped into files and, later,
# probably into a notification.
if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m'
else R=''; Y=''; G=''; N=''; fi

# Per-source verdicts, so each source can be reported independently. A single
# aggregate red light tells you backups are broken; it does not tell you which
# one, which is the thing you need at 3am.
declare -A SRC_FAIL SRC_MSG
CURRENT=""

red()  { printf '  %sFAIL%s  %s\n' "$R" "$N" "$*"; fails=$((fails+1));
         [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; }
yell() { printf '  %sWARN%s  %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); }
ok()   { printf '  %sok%s    %s\n' "$G" "$N" "$*";
         [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }

# --- Reporting -------------------------------------------------------------
# Each source gets its own Gatus external endpoint, plus one for the store
# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means
# report nowhere, which is valid: the exit code is still the whole answer.
PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}"
PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}"

report() {
    local key="$1" success="$2" message="$3"
    [ -n "$PUSH_BASE" ] || return 0
    local encoded
    encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g')
    curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \
        -H "Authorization: Bearer ${PUSH_TOKEN}" \
        "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true
}

hours_since() { echo $(( (NOW - $1) / 3600 )); }

# Pull the dump timestamp out of <name>_YYYYmmdd_HHMMSS.<suffix>. This is when
# the SOURCE produced it, which is the thing that actually matters: a source
# whose timer died still pulls "ok" forever, because yesterday's artefact is
# still sitting there. Checking only the pull would miss exactly that.
dump_epoch() {
  local base ts
  base=$(basename "$1")
  ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1
  [ -n "$ts" ] || return 1
  date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null
}

check_source() {
  local name="$1" keep="$2" dir="$STORE/$1"
  printf '\n%s\n' "== $name"
  CURRENT="$name"
  SRC_FAIL[$name]=0
  SRC_MSG[$name]=""

  [ -d "$dir" ] || { red "$name: no directory $dir"; return; }

  local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l)
  [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; }

  local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l)
  [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these"

  local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1)
  local prev;   prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p)

  # 1. Did the SOURCE dump recently?
  local de; de=$(dump_epoch "$newest")
  if [ -z "${de:-}" ]; then
    yell "$name: cannot parse a dump timestamp from $(basename "$newest")"
  else
    local dh; dh=$(hours_since "$de")
    if [ "$dh" -lt 0 ]; then
      # A future-dated artefact would otherwise stay "fresh" forever and the
      # staleness check would never fire again — the exact silent failure this
      # script exists to catch.
      red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?"
    elif [ "$dh" -gt "$MAX_AGE_H" ]; then
      red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run"
    else
      ok "$name: dumped ${dh}h ago"
    fi
  fi

  # 2. Did the PULL bring it over recently?
  local ph; ph=$(hours_since "$(stat -c %Y "$newest")")
  if [ "$ph" -gt "$MAX_AGE_H" ]; then
    red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)"
  else
    ok "$name: pulled ${ph}h ago"
  fi

  # 3. Is it plausibly a real backup?
  local sz; sz=$(stat -c %s "$newest")
  if [ "$sz" -eq 0 ]; then
    red "$name: newest artefact is ZERO bytes"
  elif [ -n "$prev" ]; then
    local psz; psz=$(stat -c %s "$prev")
    if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then
      # Not automatically wrong: headscale legitimately shrank 297K -> 20K when
      # a clean stop checkpointed its write-ahead log into the database.
      yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect"
    else
      ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)"
    fi
  else
    ok "$name: $(numfmt --to=iec "$sz") (first artefact)"
  fi

  # 4. Is retention pruning? Allow generous slack for multiple dumps per day.
  if [ "$n" -gt $(( keep * 3 + 10 )) ]; then
    yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working"
  fi
}

echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')"
echo "Artefacts older than ${MAX_AGE_H}h are treated as stale."

# --- the pull job itself ---
# Reserved key, reported as backup-store_pull-job. The store's own machinery is
# a different alarm from any one source being stale, and it is a LEADING
# indicator where the per-source checks are lagging ones: those only fire once
# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer,
# a failed pull job or a filling disk are all visible here immediately, and they
# name the cause instead of showing six stale sources with no explanation.
CURRENT="__store"
SRC_FAIL[__store]=0
SRC_MSG[__store]=""
printf '\n%s\n' "== pull-backups.service"
result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null)
status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null)
when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null)
[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})"
if [ -n "$when" ]; then
  wh=$(hours_since "$(date -d "$when" +%s)")
  [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?"
fi
systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \
  && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \
  || red "pull-backups.timer is NOT enabled"

# --- each source ---
{% for src in backup_store_sources %}
check_source "{{ src.name }}" {{ src.retention_days }}
CURRENT=""
# The store key is what Gatus computes from group+name: sanitize("backup-store")
# + "_" + sanitize("{{ src.name }}").
report "backup-store_{{ src.name }}" \
       "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \
       "${SRC_MSG[{{ src.name }}]:-no result}"
{% endfor %}

# --- capacity ---
CURRENT="__store"
printf '\n%s\n' "== disk"
use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9')
avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ')
if   [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free"
elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free"
else ok "store is ${use}% full, ${avail} free"; fi

CURRENT=""
report "backup-store_pull-job" \
       "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \
       "${SRC_MSG[__store]:-no result}"

printf '\n%s\n' "-----"
if [ "$fails" -gt 0 ]; then
  echo "RESULT: $fails failure(s), $warns warning(s)"
  echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager"
  exit 1
fi
if [ "$warns" -gt 0 ]; then
  echo "RESULT: all checks passed, $warns warning(s)"
else
  echo "RESULT: all checks passed"
fi
exit 0