personal_infra/ansible/playbooks/backups.yml

- name: Configure the offsite backup pull
  hosts: backup_store
  gather_facts: yes

  tasks:
    - name: Ensure the box pulls every source on a timer
      ansible.builtin.include_role:
        name: backup_store
      vars:
        # check-backups.sh reports one result per source plus one for the store
        # itself, so it needs the collection URL and appends each key.
        backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
        backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"
        backup_store_sources:
          - name: arbret
            source: "arbret@prd-arbret:/opt/arbret/backups/"
            retention_days: 90
          - name: headscale
            source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
            retention_days: 90
          - name: memos
            source: "backup-pull@memos-box:/opt/backups/memos/"
            retention_days: 90
          - name: vaultwarden
            source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/"
            retention_days: 90
          - name: lnbits
            source: "backup-pull@prd-vipy:/opt/backups/lnbits/"
            retention_days: 90
          - name: forgejo
            source: "backup-pull@prd-vipy:/opt/backups/forgejo/"
            retention_days: 14
          # Staging: disposable data, so the long tail is as short as the local one.
          - name: "{{ hostvars['jar_lid_staging_local'].jar_lid_archive_instance }}"
            source: "backup-pull@{{ hostvars['jar_lid_staging_local'].ansible_host }}:/opt/backups/{{ hostvars['jar_lid_staging_local'].jar_lid_archive_instance }}/"
            retention_days: "{{ hostvars['jar_lid_staging_local'].jar_lid_archive_backup_retention_days }}"

# ─────────────────────────────────────────────────────────────────────────────
# Register the backup checks with Gatus.
#
# Two groups on purpose, because they answer different questions and fail for
# different reasons:
#
#   backup-dump   did the SOURCE produce an artefact? Pushed by each dump right
#                 after it runs, so a broken dump is visible within minutes.
#   backup-store  did it ARRIVE, is it fresh, non-zero, plausibly sized, and is
#                 retention pruning? Pushed by check-backups.sh at 05:30.
#
# The store alone could catch almost everything, because the artefact filename
# carries the source's dump timestamp - a source whose timer died still pulls
# "ok" forever, but the timestamp gives it away. What the source side adds is
# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it
# cannot tell you whether the dump broke or the pull did.
#
# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed`
# deliberately excludes, so nothing of ours runs there. It is store-checked only.
# ─────────────────────────────────────────────────────────────────────────────
- name: Register the backup checks with Gatus
  hosts: observability
  become: yes
  vars:
    # Sources we deploy the dump for, and the host each one runs on.
    dump_sources:
      - {name: headscale,   host: spacey}
      - {name: memos,       host: memos_box_local}
      - {name: vaultwarden, host: vipy}
      - {name: lnbits,      host: vipy}
      - {name: forgejo,     host: vipy}
      - {name: jar-lid-archive-staging, host: jar_lid_staging_local}
    store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo, jar-lid-archive-staging]

  tasks:
    # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter
    # than a day plus slack. The store-side check catches the same fault within
    # 6h by reading the artefact's dump timestamp out of the filename, so this is
    # the slow backstop rather than the primary signal.
    - name: Build the dump endpoint list
      ansible.builtin.set_fact:
        dump_endpoints: "{{ dump_endpoints | default([]) + [{
          'name': item.name,
          'group': 'backup-dump',
          'token': gatus_push_tokens[item.host],
          'heartbeat': '26h'}] }}"
      loop: "{{ dump_sources }}"

    - name: Build the store endpoint list
      ansible.builtin.set_fact:
        store_endpoints: "{{ store_endpoints | default([]) + [{
          'name': item,
          'group': 'backup-store',
          'token': gatus_push_tokens['small_backups_local'],
          'heartbeat': '7h'}] }}"
      loop: "{{ store_sources }}"

    - name: Register the backup endpoints
      ansible.builtin.include_role:
        name: gatus_endpoint
      vars:
        # Push endpoints: the heartbeat window is the tolerance, so alert on
        # the first failure rather than waiting for three 7h windows to pass.
        gatus_endpoint_default_alerts:
          - type: signal
            failure-threshold: 1
            success-threshold: 2
            send-on-resolved: true
            minimum-reminder-interval: 12h
        gatus_endpoint_name: backups
        gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{
          'name': 'pull job',
          'group': 'backup-store',
          'token': gatus_push_tokens['small_backups_local'],
          'heartbeat': '7h'}] }}"