personal_infra/ansible/infra/403_service_probe_registration.yml

---
# The per-service health probes.
#
# These are NOT the same thing as the systemd checks in infra/401. Those answer
# "is the unit running"; these answer "does the service actually respond" - an
# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch
# from the Mempool backend. A process can be perfectly `active` and useless,
# which is precisely the gap these close.
#
# The checks themselves live in each service's own role, deployed by that
# service's playbook. This play only registers where they report, because the
# endpoints must exist in Gatus before the first push arrives.
#
# They used to push to Uptime Kuma. The scripts now POST with a bearer token
# instead of GETting ?status=up, and each host uses its own token.

# ─────────────────────────────────────────────────────────────────────────────
# Alerting thresholds, and why they differ by check type.
#
# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
# different amount of wall-clock time per check:
#
#   push/heartbeat endpoints   a failure is produced once per heartbeat window
#   pulled endpoints           a failure is produced once per interval
#
# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
# threshold of 3 on top triples a tolerance that was already chosen.
#
# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
# have no built-in tolerance, so the threshold is where it belongs for them.
# ─────────────────────────────────────────────────────────────────────────────
- name: Register the per-service probes with Gatus
  hosts: observability
  become: yes

  vars:
    probes:
      - {name: bitcoin-knots,    host: knots_box_local}
      - {name: datum-gateway,    host: knots_box_local}
      - {name: fulcrum,          host: fulcrum_box_local}
      - {name: phoenixd,         host: vipy}
      - {name: forgejo-runner,   host: forgejo_runner_local}
      - {name: mempool-mariadb,  host: mempool_box_local}
      - {name: mempool-backend,  host: mempool_box_local}
      - {name: mempool-frontend, host: mempool_box_local}
      # The two Monero probes push ONCE A DAY, not every 5 minutes, so they
      # need a window wider than a day. 26h, not 24h: the window must exceed
      # the push interval or a check that reports exactly on time races its own
      # deadline and alerts on jitter. The timers are OnCalendar 05:00 and
      # 05:30 with RandomizedDelaySec=900, so the true worst case is ~24h15m.
      # Cadence lives in roles/{monero,xmrblocks}/defaults/main.yml; the two
      # are one decision and must move together.
      - {name: monerod,          host: monerod_box_local, heartbeat: '26h'}
      - {name: xmrblocks,        host: monerod_box_local, heartbeat: '26h'}
      - {name: navidrome,        host: music_box_local}
      - {name: jar-lid-archive-staging, host: jar_lid_staging_local}

  tasks:
    - name: Build the probe endpoint list
      ansible.builtin.set_fact:
        probe_endpoints: "{{ probe_endpoints | default([]) + [{
          'name': item.name,
          'group': 'probe',
          'token': gatus_push_tokens[item.host],
          'heartbeat': item.heartbeat | default('11m')}] }}"
      loop: "{{ probes }}"

    - name: Register the probe endpoints
      ansible.builtin.include_role:
        name: gatus_endpoint
      vars:
        gatus_endpoint_default_alerts:
          - type: signal
            # 1, not 3: the heartbeat window is the tolerance. See the note above.
            failure-threshold: 1
            success-threshold: 2
            send-on-resolved: true
            minimum-reminder-interval: 6h
        gatus_endpoint_name: probes
        gatus_endpoint_external: "{{ probe_endpoints }}"