personal_infra/ansible/infra/403_service_probe_registration.yml

---
# The per-service health probes.
#
# These are NOT the same thing as the systemd checks in infra/401. Those answer
# "is the unit running"; these answer "does the service actually respond" - an
# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch
# from the Mempool backend. A process can be perfectly `active` and useless,
# which is precisely the gap these close.
#
# The checks themselves live in each service's own role, deployed by that
# service's playbook. This play only registers where they report, because the
# endpoints must exist in Gatus before the first push arrives.
#
# They used to push to Uptime Kuma. The scripts now POST with a bearer token
# instead of GETting ?status=up, and each host uses its own token.

# ─────────────────────────────────────────────────────────────────────────────
# Alerting thresholds, and why they differ by check type.
#
# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
# different amount of wall-clock time per check:
#
#   push/heartbeat endpoints   a failure is produced once per heartbeat window
#   pulled endpoints           a failure is produced once per interval
#
# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
# threshold of 3 on top triples a tolerance that was already chosen.
#
# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
# have no built-in tolerance, so the threshold is where it belongs for them.
# ─────────────────────────────────────────────────────────────────────────────
- name: Register the per-service probes with Gatus
  hosts: observability
  become: yes

  vars:
    probes:
      # failure_threshold 4: this probe runs every MINUTE, and bitcoind turns RPC
      # away for 1-3 minutes when a block lands - validation on the saturated
      # HDD plus Fulcrum/mempool/DATUM all fetching the block at once (93
      # "work queue depth exceeded" in one day, 2026-10-18). At 1, every slow
      # block paged. 4 consecutive failed minutes is past any block burst seen,
      # and a real outage still alerts within ~5 minutes.
      - {name: bitcoin-knots,    host: knots_box_local, failure_threshold: 4}
      - {name: datum-gateway,    host: knots_box_local}
      - {name: fulcrum,          host: fulcrum_box_local}
      - {name: phoenixd,         host: vipy}
      - {name: forgejo-runner,   host: forgejo_runner_local}
      - {name: mempool-mariadb,  host: mempool_box_local}
      - {name: mempool-backend,  host: mempool_box_local}
      - {name: mempool-frontend, host: mempool_box_local}
      # Every 5 minutes like the rest since 2026-10-17 (they were daily while
      # the node synced on a rotating disk). Cadence lives in
      # roles/{monero,xmrblocks}/defaults/main.yml; it and this window are one
      # decision and must move together.
      - {name: monerod,          host: monerod_box_local}
      - {name: xmrblocks,        host: monerod_box_local}
      - {name: navidrome,        host: music_box_local}
      - {name: jar-lid-archive-staging, host: jar_lid_staging_local}

    # The alert every probe gets. failure-threshold is 1 - the heartbeat window
    # is the tolerance, see the note above - unless a probe sets
    # failure_threshold, for checks that legitimately fail for a minute or two.
    probe_alert:
      type: signal
      failure-threshold: 1
      success-threshold: 2
      send-on-resolved: true
      minimum-reminder-interval: 6h

  tasks:
    - name: Build the probe endpoint list
      ansible.builtin.set_fact:
        probe_endpoints: "{{ probe_endpoints | default([]) + [{
          'name': item.name,
          'group': 'probe',
          'token': gatus_push_tokens[item.host],
          'heartbeat': item.heartbeat | default('11m'),
          'alerts': [probe_alert | combine({'failure-threshold': item.failure_threshold | default(1)})]}] }}"
      loop: "{{ probes }}"

    - name: Register the probe endpoints
      ansible.builtin.include_role:
        name: gatus_endpoint
      vars:
        gatus_endpoint_name: probes
        gatus_endpoint_external: "{{ probe_endpoints }}"