diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 0686ab6..61c005d 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -7,6 +7,10 @@ ansible.builtin.include_role: name: backup_store vars: + # check-backups.sh reports one result per source plus one for the store + # itself, so it needs the collection URL and appends each key. + backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" backup_store_sources: - name: arbret source: "arbret@prd-arbret:/opt/arbret/backups/" @@ -26,3 +30,66 @@ - name: forgejo source: "backup-pull@prd-vipy:/opt/backups/forgejo/" retention_days: 14 + +# ───────────────────────────────────────────────────────────────────────────── +# Register the backup checks with Gatus. +# +# Two groups on purpose, because they answer different questions and fail for +# different reasons: +# +# backup-dump did the SOURCE produce an artefact? Pushed by each dump right +# after it runs, so a broken dump is visible within minutes. +# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is +# retention pruning? Pushed by check-backups.sh at 05:30. +# +# The store alone could catch almost everything, because the artefact filename +# carries the source's dump timestamp - a source whose timer died still pulls +# "ok" forever, but the timestamp gives it away. What the source side adds is +# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it +# cannot tell you whether the dump broke or the pull did. +# +# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` +# deliberately excludes, so nothing of ours runs there. It is store-checked only. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the backup checks with Gatus + hosts: observability + become: yes + vars: + # Sources we deploy the dump for, and the host each one runs on. + dump_sources: + - {name: headscale, host: spacey} + - {name: memos, host: memos_box_local} + - {name: vaultwarden, host: vipy} + - {name: lnbits, host: vipy} + - {name: forgejo, host: vipy} + store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] + + tasks: + - name: Build the dump endpoint list + ansible.builtin.set_fact: + dump_endpoints: "{{ dump_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'backup-dump', + 'token': gatus_push_tokens[item.host], + 'heartbeat': '30h'}] }}" + loop: "{{ dump_sources }}" + + - name: Build the store endpoint list + ansible.builtin.set_fact: + store_endpoints: "{{ store_endpoints | default([]) + [{ + 'name': item, + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '30h'}] }}" + loop: "{{ store_sources }}" + + - name: Register the backup endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: backups + gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ + 'name': 'pull job', + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '30h'}] }}" diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml index 7de6006..b195190 100644 --- a/ansible/roles/backup_source/defaults/main.yml +++ b/ansible/roles/backup_source/defaults/main.yml @@ -29,3 +29,16 @@ backup_source_retention_days: 7 # Schedule. The box pulls at 04:00, so dumps must land before that. backup_source_on_calendar: "*-*-* 02:00:00" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Where to report that this dump ran and produced a plausible artefact. +# Gatus external endpoint: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty is valid and is not an error: the unit's exit code is still the answer, +# and the STORE will independently notice a stale dump within ~26h because the +# artefact filename carries this dump's timestamp. Reporting here only buys +# earlier detection and tells you it was the DUMP that broke rather than the +# pull. +backup_source_push_url: "" +backup_source_push_token: "" diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index ec2d6cf..a964c5d 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -30,7 +30,12 @@ - name: Ensure age is installed ansible.builtin.apt: - name: age + name: + - age + # curl is needed only when backup_source_push_url is set, but installing it + # unconditionally keeps the task idempotent and it is present on every + # Debian host here anyway. + - curl state: present # The pull account: unprivileged, no sudo, exists only so small-backups-box can @@ -83,14 +88,18 @@ mode: '0750' validate: "bash -n %s" +# The .service carries the push token in an Environment= line, so it is 0600. +# The .timer holds nothing secret and stays world-readable. - name: "Install the {{ backup_source_name }}-backup systemd units" ansible.builtin.template: - src: "backup.{{ item }}.j2" - dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" + src: "backup.{{ item.unit }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}" owner: root group: root - mode: '0644' - loop: [service, timer] + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} notify: Reload systemd for backup units - name: "Enable the {{ backup_source_name }}-backup timer" diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 index 91ad063..7a1df21 100644 --- a/ansible/roles/backup_source/templates/backup.service.j2 +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -9,6 +9,8 @@ After={{ backup_source_stop_service if '.' in backup_source_stop_service else ba [Service] Type=oneshot ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +Environment=BACKUP_PUSH_URL={{ backup_source_push_url }} +Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }} StandardOutput=journal StandardError=journal SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index abdadfe..018c313 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -41,13 +41,65 @@ chmod 700 "$BACKUP_DIR" # here or they accumulate forever. rm -f "${BACKUP_DIR}/${NAME}_"*.partial +# --- Reporting ------------------------------------------------------------- +# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed +# backup even though the script "finished". Both are reported as failures. +PUSH_URL="${BACKUP_PUSH_URL:-}" +PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}" + +report() { + local success="$1" message="$2" + [ -n "$PUSH_URL" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true +} + +# Reports on ANY exit path, so a dump that dies halfway still reports rather +# than going quiet. The size of the FINISHED artefact decides success, not +# merely reaching the end of the script. +# +# This is called FROM the single EXIT trap below - it must never register an +# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather +# than adding to it, so a second trap here silently discards the one that +# restarts the service, and a backup run leaves the service stopped. That is +# precisely the failure the restart trap exists to prevent. +report_outcome() { + local rc="$1" + if [ "$rc" -ne 0 ]; then + report "false" "${NAME} dump exited ${rc}" + elif [ ! -s "$ARTIFACT" ]; then + report "false" "${NAME} produced no artefact at ${ARTIFACT}" + else + report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)" + fi +} + +# --- One EXIT handler, doing both jobs ------------------------------------- +# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous +# handler rather than adding to it. Registering a second one here would +# silently discard the service restart and leave the service stopped after +# every backup - which is the exact bug the restart exists to prevent, and it +# is invisible until someone notices the service is down. +on_exit() { + local rc=$? {% if backup_source_stop_service or backup_source_stop_command %} -# --- Stop the service, and guarantee it comes back --- + log "Restarting ${SERVICE}..." + eval "$START_CMD" || true +{% endif %} + report_outcome "$rc" +} +trap on_exit EXIT + +{% if backup_source_stop_service or backup_source_stop_command %} +# --- Stop the service; the trap above guarantees it comes back ------------- # The trap is the point: without it a failed dump leaves the service down until # the next timer fires. Every hand-written script this replaced had that bug. +# It is armed BEFORE the stop, so even a failure during the stop restarts. log "Stopping ${SERVICE}..." eval "$STOP_CMD" -trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT {% endif %} # --- Dump straight into age; plaintext never touches the disk --- diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml index c0c128b..3a0754c 100644 --- a/ansible/roles/backup_store/defaults/main.yml +++ b/ansible/roles/backup_store/defaults/main.yml @@ -9,3 +9,18 @@ backup_store_on_calendar: "*-*-* 04:00:00" # source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" # retention_days: 90 backup_store_sources: [] + +# ── Reporting ──────────────────────────────────────────────────────────────── +# check-backups.sh reports one result PER SOURCE plus one for the store itself, +# so the base URL is the endpoints collection and the script appends each key. +# Empty is valid: the script still prints its report and exits 0/1. +backup_store_check_push_base: "" +backup_store_check_push_token: "" + +# Runs after the 04:00 pull. Late enough that a slow pull has finished, early +# enough that a failure is visible before the working day. +backup_store_check_on_calendar: "*-*-* 05:30:00" + +# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the +# pull is at 04:00, so 26h tolerates exactly one missed night before alarming. +backup_store_check_max_age_hours: 26 diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml index 2596dbc..55500c6 100644 --- a/ansible/roles/backup_store/tasks/main.yml +++ b/ansible/roles/backup_store/tasks/main.yml @@ -33,9 +33,9 @@ validate: "bash -n %s" become: yes -# A human-run assertion that last night actually worked. Generated from the same -# source list as the puller, so it can never drift out of sync with what is -# supposed to be arriving. +# An assertion that last night actually worked. Generated from the same source +# list as the puller, so it can never drift out of sync with what is supposed to +# be arriving. Runs on a timer AND is useful by hand. - name: Install the backup check script ansible.builtin.template: src: check-backups.sh.j2 @@ -46,6 +46,29 @@ validate: "bash -n %s" become: yes +# The .service carries the push token, so it is 0600; the .timer is not secret. +- name: Install the check-backups systemd units + ansible.builtin.template: + src: "check-backups.{{ item.unit }}.j2" + dest: "/etc/systemd/system/check-backups.{{ item.unit }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} + become: yes + +# restarted, not started: `started` is a no-op on an already-active timer, so a +# changed schedule would never be picked up. +- name: Enable the check-backups timer + ansible.builtin.systemd: + name: check-backups.timer + enabled: yes + state: restarted + daemon_reload: yes + become: yes + - name: Install the pull-backups systemd units ansible.builtin.template: src: "pull-backups.{{ item }}.j2" diff --git a/ansible/roles/backup_store/templates/check-backups.service.j2 b/ansible/roles/backup_store/templates/check-backups.service.j2 new file mode 100644 index 0000000..3e50b6a --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Verify the nightly backup pull actually worked +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }} +Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }} +Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 index f7a8830..37c9916 100644 --- a/ansible/roles/backup_store/templates/check-backups.sh.j2 +++ b/ansible/roles/backup_store/templates/check-backups.sh.j2 @@ -22,9 +22,34 @@ fails=0; warns=0 if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' else R=''; Y=''; G=''; N=''; fi -red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); } +# Per-source verdicts, so each source can be reported independently. A single +# aggregate red light tells you backups are broken; it does not tell you which +# one, which is the thing you need at 3am. +declare -A SRC_FAIL SRC_MSG +CURRENT="" + +red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); + [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; } yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } -ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; } +ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; + [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; } + +# --- Reporting ------------------------------------------------------------- +# Each source gets its own Gatus external endpoint, plus one for the store +# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means +# report nowhere, which is valid: the exit code is still the whole answer. +PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}" +PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}" + +report() { + local key="$1" success="$2" message="$3" + [ -n "$PUSH_BASE" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true +} hours_since() { echo $(( (NOW - $1) / 3600 )); } @@ -43,6 +68,9 @@ dump_epoch() { check_source() { local name="$1" keep="$2" dir="$STORE/$1" printf '\n%s\n' "== $name" + CURRENT="$name" + SRC_FAIL[$name]=0 + SRC_MSG[$name]="" [ -d "$dir" ] || { red "$name: no directory $dir"; return; } @@ -108,6 +136,15 @@ echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." # --- the pull job itself --- +# Reserved key, reported as backup-store_pull-job. The store's own machinery is +# a different alarm from any one source being stale, and it is a LEADING +# indicator where the per-source checks are lagging ones: those only fire once +# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer, +# a failed pull job or a filling disk are all visible here immediately, and they +# name the cause instead of showing six stale sources with no explanation. +CURRENT="__store" +SRC_FAIL[__store]=0 +SRC_MSG[__store]="" printf '\n%s\n' "== pull-backups.service" result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) @@ -124,9 +161,16 @@ systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ # --- each source --- {% for src in backup_store_sources %} check_source "{{ src.name }}" {{ src.retention_days }} +CURRENT="" +# The store key is what Gatus computes from group+name: sanitize("backup-store") +# + "_" + sanitize("{{ src.name }}"). +report "backup-store_{{ src.name }}" \ + "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[{{ src.name }}]:-no result}" {% endfor %} # --- capacity --- +CURRENT="__store" printf '\n%s\n' "== disk" use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') @@ -134,6 +178,11 @@ if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" else ok "store is ${use}% full, ${avail} free"; fi +CURRENT="" +report "backup-store_pull-job" \ + "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[__store]:-no result}" + printf '\n%s\n' "-----" if [ "$fails" -gt 0 ]; then echo "RESULT: $fails failure(s), $warns warning(s)" diff --git a/ansible/roles/backup_store/templates/check-backups.timer.j2 b/ansible/roles/backup_store/templates/check-backups.timer.j2 new file mode 100644 index 0000000..ba5c5a8 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run the backup verification after the nightly pull +Requires=check-backups.service + +[Timer] +OnCalendar={{ backup_store_check_on_calendar }} +# Run a missed occurrence on the next boot rather than skipping the day. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml index ee1769b..5329e2a 100644 --- a/ansible/services/forgejo/setup_backup_forgejo.yml +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -22,3 +22,7 @@ backup_source_stop_service: forgejo backup_source_retention_days: 2 backup_source_on_calendar: "*-*-* 02:30:00" + # Reported to Gatus as backup-dump_forgejo. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_forgejo/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml index 5ae3ad1..15ba5b7 100644 --- a/ansible/services/headscale/setup_backup_headscale.yml +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -18,3 +18,7 @@ backup_source_stop_service: headscale backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_headscale. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_headscale/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml index 0c45b29..2da0ac0 100644 --- a/ansible/services/lnbits/setup_backup_lnbits.yml +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -22,3 +22,7 @@ backup_source_stop_service: lnbits backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:20:00" + # Reported to Gatus as backup-dump_lnbits. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_lnbits/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml index 36b8cf6..53ab85d 100644 --- a/ansible/services/memos/setup_backup_memos.yml +++ b/ansible/services/memos/setup_backup_memos.yml @@ -23,3 +23,7 @@ backup_source_stop_service: memos backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_memos. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_memos/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml index 54fe147..d0dfdfa 100644 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -22,3 +22,7 @@ backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:10:00" + # Reported to Gatus as backup-dump_vaultwarden. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_vaultwarden/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"