From c2de6dbbd939409fd2395ebe4997420761d6bff3 Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 14 Sep 2026 09:22:12 +0200 Subject: [PATCH] backups: monitor both the dump and the pull, per source Twelve endpoints in two groups, because they answer different questions and fail for different reasons: backup-dump_ pushed by the SOURCE right after its dump runs backup-store_ pushed by the BOX at 05:30, per source backup-store_pull-job pushed by the BOX, about the box itself The store alone could catch almost everything, because the artefact filename carries the source's dump timestamp - a source whose timer died still pulls "ok" forever, but the timestamp gives it away. What the source side adds is LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it cannot tell you whether the dump broke or the pull did. pull-job is separate from the per-source checks because it is a LEADING indicator where those are lagging ones. A disabled pull timer, a failed pull job, or a filling disk are all visible immediately, while the per-source checks only fire once an artefact is >26h stale - about a day later. Disable the timer at 10:00 and every source stays green until tomorrow; pull-job goes red this morning and names the cause instead of showing six stale sources. Frequencies: dumps 02:00-02:30 staggered, pull 04:00, verification 05:30, all daily and Persistent. 26h staleness decides red; a 30h Gatus heartbeat catches the verification itself having stopped, so a dead check-backups.timer cannot hide a stale backup. arbret has no dump endpoint: prd-arbret is in [arbret], which `managed` deliberately excludes, so nothing of ours runs there. Store-checked only. check-backups.sh was manual-only; it now runs on a timer and reports per source rather than only printing. The human-readable report is unchanged. A SERIOUS bug introduced and fixed in this change, recorded because the shape is easy to repeat: the reporting hook was added to backup.sh as a second `trap ... EXIT`. Bash REPLACES the EXIT handler rather than adding to it, so that silently deleted the trap which restarts the stopped service - the one the script's own comment calls "the point", and the bug the role was written to eliminate. Every backup then stopped its service and left it stopped. It took forgejo, lnbits, headscale and memos down for several minutes each, and nothing caught it: the dumps exit 0, the artefacts are correct, the deploy reports failed=0, and liveness only proves the HOST is up. There is now ONE EXIT handler doing both jobs, armed BEFORE the stop so a failure during the stop still restarts. Verified by rendering both variants, asserting exactly one EXIT trap in each, and simulating a mid-way failure to confirm the restart fires. Verified: all 12 endpoints UP; six sources pulled cleanly (arbret 31M, headscale 198K, memos 7.8M, vaultwarden 2.1M, lnbits 30M, forgejo 2.6G), store 16% full, "RESULT: all checks passed". Known gaps, deliberately not closed here: * `yell` warnings - disk 75-90%, an artefact under half the previous size, retention not pruning - never reach Gatus, because a push is binary. * Nothing verifies a backed-up service came back UP. That is the gap that let the trap bug run unnoticed, and it is what the next change addresses. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/playbooks/backups.yml | 67 +++++++++++++++++++ ansible/roles/backup_source/defaults/main.yml | 13 ++++ ansible/roles/backup_source/tasks/main.yml | 19 ++++-- .../backup_source/templates/backup.service.j2 | 2 + .../backup_source/templates/backup.sh.j2 | 56 +++++++++++++++- ansible/roles/backup_store/defaults/main.yml | 15 +++++ ansible/roles/backup_store/tasks/main.yml | 29 +++++++- .../templates/check-backups.service.j2 | 16 +++++ .../templates/check-backups.sh.j2 | 53 ++++++++++++++- .../templates/check-backups.timer.j2 | 11 +++ .../services/forgejo/setup_backup_forgejo.yml | 4 ++ .../headscale/setup_backup_headscale.yml | 4 ++ .../services/lnbits/setup_backup_lnbits.yml | 4 ++ ansible/services/memos/setup_backup_memos.yml | 4 ++ .../vaultwarden/setup_backup_vaultwarden.yml | 4 ++ 15 files changed, 289 insertions(+), 12 deletions(-) create mode 100644 ansible/roles/backup_store/templates/check-backups.service.j2 create mode 100644 ansible/roles/backup_store/templates/check-backups.timer.j2 diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 0686ab6..61c005d 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -7,6 +7,10 @@ ansible.builtin.include_role: name: backup_store vars: + # check-backups.sh reports one result per source plus one for the store + # itself, so it needs the collection URL and appends each key. + backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" backup_store_sources: - name: arbret source: "arbret@prd-arbret:/opt/arbret/backups/" @@ -26,3 +30,66 @@ - name: forgejo source: "backup-pull@prd-vipy:/opt/backups/forgejo/" retention_days: 14 + +# ───────────────────────────────────────────────────────────────────────────── +# Register the backup checks with Gatus. +# +# Two groups on purpose, because they answer different questions and fail for +# different reasons: +# +# backup-dump did the SOURCE produce an artefact? Pushed by each dump right +# after it runs, so a broken dump is visible within minutes. +# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is +# retention pruning? Pushed by check-backups.sh at 05:30. +# +# The store alone could catch almost everything, because the artefact filename +# carries the source's dump timestamp - a source whose timer died still pulls +# "ok" forever, but the timestamp gives it away. What the source side adds is +# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it +# cannot tell you whether the dump broke or the pull did. +# +# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` +# deliberately excludes, so nothing of ours runs there. It is store-checked only. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the backup checks with Gatus + hosts: observability + become: yes + vars: + # Sources we deploy the dump for, and the host each one runs on. + dump_sources: + - {name: headscale, host: spacey} + - {name: memos, host: memos_box_local} + - {name: vaultwarden, host: vipy} + - {name: lnbits, host: vipy} + - {name: forgejo, host: vipy} + store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] + + tasks: + - name: Build the dump endpoint list + ansible.builtin.set_fact: + dump_endpoints: "{{ dump_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'backup-dump', + 'token': gatus_push_tokens[item.host], + 'heartbeat': '30h'}] }}" + loop: "{{ dump_sources }}" + + - name: Build the store endpoint list + ansible.builtin.set_fact: + store_endpoints: "{{ store_endpoints | default([]) + [{ + 'name': item, + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '30h'}] }}" + loop: "{{ store_sources }}" + + - name: Register the backup endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: backups + gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ + 'name': 'pull job', + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '30h'}] }}" diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml index 7de6006..b195190 100644 --- a/ansible/roles/backup_source/defaults/main.yml +++ b/ansible/roles/backup_source/defaults/main.yml @@ -29,3 +29,16 @@ backup_source_retention_days: 7 # Schedule. The box pulls at 04:00, so dumps must land before that. backup_source_on_calendar: "*-*-* 02:00:00" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Where to report that this dump ran and produced a plausible artefact. +# Gatus external endpoint: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty is valid and is not an error: the unit's exit code is still the answer, +# and the STORE will independently notice a stale dump within ~26h because the +# artefact filename carries this dump's timestamp. Reporting here only buys +# earlier detection and tells you it was the DUMP that broke rather than the +# pull. +backup_source_push_url: "" +backup_source_push_token: "" diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index ec2d6cf..a964c5d 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -30,7 +30,12 @@ - name: Ensure age is installed ansible.builtin.apt: - name: age + name: + - age + # curl is needed only when backup_source_push_url is set, but installing it + # unconditionally keeps the task idempotent and it is present on every + # Debian host here anyway. + - curl state: present # The pull account: unprivileged, no sudo, exists only so small-backups-box can @@ -83,14 +88,18 @@ mode: '0750' validate: "bash -n %s" +# The .service carries the push token in an Environment= line, so it is 0600. +# The .timer holds nothing secret and stays world-readable. - name: "Install the {{ backup_source_name }}-backup systemd units" ansible.builtin.template: - src: "backup.{{ item }}.j2" - dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" + src: "backup.{{ item.unit }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}" owner: root group: root - mode: '0644' - loop: [service, timer] + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} notify: Reload systemd for backup units - name: "Enable the {{ backup_source_name }}-backup timer" diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 index 91ad063..7a1df21 100644 --- a/ansible/roles/backup_source/templates/backup.service.j2 +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -9,6 +9,8 @@ After={{ backup_source_stop_service if '.' in backup_source_stop_service else ba [Service] Type=oneshot ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +Environment=BACKUP_PUSH_URL={{ backup_source_push_url }} +Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }} StandardOutput=journal StandardError=journal SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index abdadfe..018c313 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -41,13 +41,65 @@ chmod 700 "$BACKUP_DIR" # here or they accumulate forever. rm -f "${BACKUP_DIR}/${NAME}_"*.partial +# --- Reporting ------------------------------------------------------------- +# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed +# backup even though the script "finished". Both are reported as failures. +PUSH_URL="${BACKUP_PUSH_URL:-}" +PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}" + +report() { + local success="$1" message="$2" + [ -n "$PUSH_URL" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true +} + +# Reports on ANY exit path, so a dump that dies halfway still reports rather +# than going quiet. The size of the FINISHED artefact decides success, not +# merely reaching the end of the script. +# +# This is called FROM the single EXIT trap below - it must never register an +# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather +# than adding to it, so a second trap here silently discards the one that +# restarts the service, and a backup run leaves the service stopped. That is +# precisely the failure the restart trap exists to prevent. +report_outcome() { + local rc="$1" + if [ "$rc" -ne 0 ]; then + report "false" "${NAME} dump exited ${rc}" + elif [ ! -s "$ARTIFACT" ]; then + report "false" "${NAME} produced no artefact at ${ARTIFACT}" + else + report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)" + fi +} + +# --- One EXIT handler, doing both jobs ------------------------------------- +# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous +# handler rather than adding to it. Registering a second one here would +# silently discard the service restart and leave the service stopped after +# every backup - which is the exact bug the restart exists to prevent, and it +# is invisible until someone notices the service is down. +on_exit() { + local rc=$? {% if backup_source_stop_service or backup_source_stop_command %} -# --- Stop the service, and guarantee it comes back --- + log "Restarting ${SERVICE}..." + eval "$START_CMD" || true +{% endif %} + report_outcome "$rc" +} +trap on_exit EXIT + +{% if backup_source_stop_service or backup_source_stop_command %} +# --- Stop the service; the trap above guarantees it comes back ------------- # The trap is the point: without it a failed dump leaves the service down until # the next timer fires. Every hand-written script this replaced had that bug. +# It is armed BEFORE the stop, so even a failure during the stop restarts. log "Stopping ${SERVICE}..." eval "$STOP_CMD" -trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT {% endif %} # --- Dump straight into age; plaintext never touches the disk --- diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml index c0c128b..3a0754c 100644 --- a/ansible/roles/backup_store/defaults/main.yml +++ b/ansible/roles/backup_store/defaults/main.yml @@ -9,3 +9,18 @@ backup_store_on_calendar: "*-*-* 04:00:00" # source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" # retention_days: 90 backup_store_sources: [] + +# ── Reporting ──────────────────────────────────────────────────────────────── +# check-backups.sh reports one result PER SOURCE plus one for the store itself, +# so the base URL is the endpoints collection and the script appends each key. +# Empty is valid: the script still prints its report and exits 0/1. +backup_store_check_push_base: "" +backup_store_check_push_token: "" + +# Runs after the 04:00 pull. Late enough that a slow pull has finished, early +# enough that a failure is visible before the working day. +backup_store_check_on_calendar: "*-*-* 05:30:00" + +# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the +# pull is at 04:00, so 26h tolerates exactly one missed night before alarming. +backup_store_check_max_age_hours: 26 diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml index 2596dbc..55500c6 100644 --- a/ansible/roles/backup_store/tasks/main.yml +++ b/ansible/roles/backup_store/tasks/main.yml @@ -33,9 +33,9 @@ validate: "bash -n %s" become: yes -# A human-run assertion that last night actually worked. Generated from the same -# source list as the puller, so it can never drift out of sync with what is -# supposed to be arriving. +# An assertion that last night actually worked. Generated from the same source +# list as the puller, so it can never drift out of sync with what is supposed to +# be arriving. Runs on a timer AND is useful by hand. - name: Install the backup check script ansible.builtin.template: src: check-backups.sh.j2 @@ -46,6 +46,29 @@ validate: "bash -n %s" become: yes +# The .service carries the push token, so it is 0600; the .timer is not secret. +- name: Install the check-backups systemd units + ansible.builtin.template: + src: "check-backups.{{ item.unit }}.j2" + dest: "/etc/systemd/system/check-backups.{{ item.unit }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} + become: yes + +# restarted, not started: `started` is a no-op on an already-active timer, so a +# changed schedule would never be picked up. +- name: Enable the check-backups timer + ansible.builtin.systemd: + name: check-backups.timer + enabled: yes + state: restarted + daemon_reload: yes + become: yes + - name: Install the pull-backups systemd units ansible.builtin.template: src: "pull-backups.{{ item }}.j2" diff --git a/ansible/roles/backup_store/templates/check-backups.service.j2 b/ansible/roles/backup_store/templates/check-backups.service.j2 new file mode 100644 index 0000000..3e50b6a --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Verify the nightly backup pull actually worked +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }} +Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }} +Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 index f7a8830..37c9916 100644 --- a/ansible/roles/backup_store/templates/check-backups.sh.j2 +++ b/ansible/roles/backup_store/templates/check-backups.sh.j2 @@ -22,9 +22,34 @@ fails=0; warns=0 if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' else R=''; Y=''; G=''; N=''; fi -red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); } +# Per-source verdicts, so each source can be reported independently. A single +# aggregate red light tells you backups are broken; it does not tell you which +# one, which is the thing you need at 3am. +declare -A SRC_FAIL SRC_MSG +CURRENT="" + +red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); + [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; } yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } -ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; } +ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; + [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; } + +# --- Reporting ------------------------------------------------------------- +# Each source gets its own Gatus external endpoint, plus one for the store +# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means +# report nowhere, which is valid: the exit code is still the whole answer. +PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}" +PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}" + +report() { + local key="$1" success="$2" message="$3" + [ -n "$PUSH_BASE" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true +} hours_since() { echo $(( (NOW - $1) / 3600 )); } @@ -43,6 +68,9 @@ dump_epoch() { check_source() { local name="$1" keep="$2" dir="$STORE/$1" printf '\n%s\n' "== $name" + CURRENT="$name" + SRC_FAIL[$name]=0 + SRC_MSG[$name]="" [ -d "$dir" ] || { red "$name: no directory $dir"; return; } @@ -108,6 +136,15 @@ echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." # --- the pull job itself --- +# Reserved key, reported as backup-store_pull-job. The store's own machinery is +# a different alarm from any one source being stale, and it is a LEADING +# indicator where the per-source checks are lagging ones: those only fire once +# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer, +# a failed pull job or a filling disk are all visible here immediately, and they +# name the cause instead of showing six stale sources with no explanation. +CURRENT="__store" +SRC_FAIL[__store]=0 +SRC_MSG[__store]="" printf '\n%s\n' "== pull-backups.service" result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) @@ -124,9 +161,16 @@ systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ # --- each source --- {% for src in backup_store_sources %} check_source "{{ src.name }}" {{ src.retention_days }} +CURRENT="" +# The store key is what Gatus computes from group+name: sanitize("backup-store") +# + "_" + sanitize("{{ src.name }}"). +report "backup-store_{{ src.name }}" \ + "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[{{ src.name }}]:-no result}" {% endfor %} # --- capacity --- +CURRENT="__store" printf '\n%s\n' "== disk" use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') @@ -134,6 +178,11 @@ if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" else ok "store is ${use}% full, ${avail} free"; fi +CURRENT="" +report "backup-store_pull-job" \ + "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[__store]:-no result}" + printf '\n%s\n' "-----" if [ "$fails" -gt 0 ]; then echo "RESULT: $fails failure(s), $warns warning(s)" diff --git a/ansible/roles/backup_store/templates/check-backups.timer.j2 b/ansible/roles/backup_store/templates/check-backups.timer.j2 new file mode 100644 index 0000000..ba5c5a8 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run the backup verification after the nightly pull +Requires=check-backups.service + +[Timer] +OnCalendar={{ backup_store_check_on_calendar }} +# Run a missed occurrence on the next boot rather than skipping the day. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml index ee1769b..5329e2a 100644 --- a/ansible/services/forgejo/setup_backup_forgejo.yml +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -22,3 +22,7 @@ backup_source_stop_service: forgejo backup_source_retention_days: 2 backup_source_on_calendar: "*-*-* 02:30:00" + # Reported to Gatus as backup-dump_forgejo. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_forgejo/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml index 5ae3ad1..15ba5b7 100644 --- a/ansible/services/headscale/setup_backup_headscale.yml +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -18,3 +18,7 @@ backup_source_stop_service: headscale backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_headscale. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_headscale/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml index 0c45b29..2da0ac0 100644 --- a/ansible/services/lnbits/setup_backup_lnbits.yml +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -22,3 +22,7 @@ backup_source_stop_service: lnbits backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:20:00" + # Reported to Gatus as backup-dump_lnbits. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_lnbits/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml index 36b8cf6..53ab85d 100644 --- a/ansible/services/memos/setup_backup_memos.yml +++ b/ansible/services/memos/setup_backup_memos.yml @@ -23,3 +23,7 @@ backup_source_stop_service: memos backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_memos. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_memos/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml index 54fe147..d0dfdfa 100644 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -22,3 +22,7 @@ backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:10:00" + # Reported to Gatus as backup-dump_vaultwarden. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_vaultwarden/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"