brushing up backups
This commit is contained in:
parent
e9bb90f8f8
commit
2ebb2f9a64
GPG key ID:
883EDBAA726BD96C
5 changed files with 185 additions and 3 deletions
148
ansible/roles/backup_store/templates/check-backups.sh.j2
Normal file
148
ansible/roles/backup_store/templates/check-backups.sh.j2
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
#!/usr/bin/env bash
|
||||
# Assert the nightly backups actually worked.
|
||||
#
|
||||
# Run as {{ ansible_user_id }} on this host. Needs no sudo.
|
||||
#
|
||||
# What it CANNOT do: verify contents. The age identity lives only on lapy, so
|
||||
# this host cannot decrypt anything it holds — by design. These are freshness,
|
||||
# completeness and integrity checks. To verify content, decrypt on lapy:
|
||||
# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups/<svc>/<artefact>" \
|
||||
# | age -d -i ~/.age/counterweight_age | tar -tzf - | head
|
||||
#
|
||||
# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL.
|
||||
set -uo pipefail
|
||||
|
||||
STORE="{{ backup_store_dir }}"
|
||||
MAX_AGE_H="${1:-26}" # an artefact older than this is stale
|
||||
NOW=$(date +%s)
|
||||
fails=0; warns=0
|
||||
|
||||
# Colour only when attached to a terminal: this gets piped into files and, later,
|
||||
# probably into a notification.
|
||||
if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m'
|
||||
else R=''; Y=''; G=''; N=''; fi
|
||||
|
||||
red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); }
|
||||
yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); }
|
||||
ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; }
|
||||
|
||||
hours_since() { echo $(( (NOW - $1) / 3600 )); }
|
||||
|
||||
# Pull the dump timestamp out of <name>_YYYYmmdd_HHMMSS.<suffix>. This is when
|
||||
# the SOURCE produced it, which is the thing that actually matters: a source
|
||||
# whose timer died still pulls "ok" forever, because yesterday's artefact is
|
||||
# still sitting there. Checking only the pull would miss exactly that.
|
||||
dump_epoch() {
|
||||
local base ts
|
||||
base=$(basename "$1")
|
||||
ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1
|
||||
[ -n "$ts" ] || return 1
|
||||
date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null
|
||||
}
|
||||
|
||||
check_source() {
|
||||
local name="$1" keep="$2" dir="$STORE/$1"
|
||||
printf '\n%s\n' "== $name"
|
||||
|
||||
[ -d "$dir" ] || { red "$name: no directory $dir"; return; }
|
||||
|
||||
local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l)
|
||||
[ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; }
|
||||
|
||||
local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l)
|
||||
[ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these"
|
||||
|
||||
local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1)
|
||||
local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p)
|
||||
|
||||
# 1. Did the SOURCE dump recently?
|
||||
local de; de=$(dump_epoch "$newest")
|
||||
if [ -z "${de:-}" ]; then
|
||||
yell "$name: cannot parse a dump timestamp from $(basename "$newest")"
|
||||
else
|
||||
local dh; dh=$(hours_since "$de")
|
||||
if [ "$dh" -lt 0 ]; then
|
||||
# A future-dated artefact would otherwise stay "fresh" forever and the
|
||||
# staleness check would never fire again — the exact silent failure this
|
||||
# script exists to catch.
|
||||
red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?"
|
||||
elif [ "$dh" -gt "$MAX_AGE_H" ]; then
|
||||
red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run"
|
||||
else
|
||||
ok "$name: dumped ${dh}h ago"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Did the PULL bring it over recently?
|
||||
local ph; ph=$(hours_since "$(stat -c %Y "$newest")")
|
||||
if [ "$ph" -gt "$MAX_AGE_H" ]; then
|
||||
red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)"
|
||||
else
|
||||
ok "$name: pulled ${ph}h ago"
|
||||
fi
|
||||
|
||||
# 3. Is it plausibly a real backup?
|
||||
local sz; sz=$(stat -c %s "$newest")
|
||||
if [ "$sz" -eq 0 ]; then
|
||||
red "$name: newest artefact is ZERO bytes"
|
||||
elif [ -n "$prev" ]; then
|
||||
local psz; psz=$(stat -c %s "$prev")
|
||||
if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then
|
||||
# Not automatically wrong: headscale legitimately shrank 297K -> 20K when
|
||||
# a clean stop checkpointed its write-ahead log into the database.
|
||||
yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect"
|
||||
else
|
||||
ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)"
|
||||
fi
|
||||
else
|
||||
ok "$name: $(numfmt --to=iec "$sz") (first artefact)"
|
||||
fi
|
||||
|
||||
# 4. Is retention pruning? Allow generous slack for multiple dumps per day.
|
||||
if [ "$n" -gt $(( keep * 3 + 10 )) ]; then
|
||||
yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
echo "Artefacts older than ${MAX_AGE_H}h are treated as stale."
|
||||
|
||||
# --- the pull job itself ---
|
||||
printf '\n%s\n' "== pull-backups.service"
|
||||
result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null)
|
||||
status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null)
|
||||
when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null)
|
||||
[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})"
|
||||
if [ -n "$when" ]; then
|
||||
wh=$(hours_since "$(date -d "$when" +%s)")
|
||||
[ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?"
|
||||
fi
|
||||
systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \
|
||||
&& ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \
|
||||
|| red "pull-backups.timer is NOT enabled"
|
||||
|
||||
# --- each source ---
|
||||
{% for src in backup_store_sources %}
|
||||
check_source "{{ src.name }}" {{ src.retention_days }}
|
||||
{% endfor %}
|
||||
|
||||
# --- capacity ---
|
||||
printf '\n%s\n' "== disk"
|
||||
use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9')
|
||||
avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ')
|
||||
if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free"
|
||||
elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free"
|
||||
else ok "store is ${use}% full, ${avail} free"; fi
|
||||
|
||||
printf '\n%s\n' "-----"
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "RESULT: $fails failure(s), $warns warning(s)"
|
||||
echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$warns" -gt 0 ]; then
|
||||
echo "RESULT: all checks passed, $warns warning(s)"
|
||||
else
|
||||
echo "RESULT: all checks passed"
|
||||
fi
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue