diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml index c181ba1..088cc7b 100644 --- a/ansible/infra/401_service_monitoring.yml +++ b/ansible/infra/401_service_monitoring.yml @@ -44,6 +44,13 @@ # # Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints # have no built-in tolerance, so the threshold is where it belongs for them. +# +# EXCEPT here, where the threshold is 2. The heartbeat only forgives a MISSING +# push. These checks also push an explicit `false` whenever a unit is caught +# mid-restart - a deploy, an unattended upgrade, a backup stopping the service +# for its dump - and at 1 every one of those pages, then resolves ten minutes +# later. 2 means "down on two consecutive 5-minute runs", i.e. 5-10 minutes. +# A host that stops reporting altogether still alarms on the heartbeat alone. # ───────────────────────────────────────────────────────────────────────────── - name: Register the service checks with Gatus hosts: observability @@ -73,8 +80,9 @@ vars: gatus_endpoint_default_alerts: - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 + # 2: tolerate one explicit `false` from a unit caught mid-restart. + # See the note above. + failure-threshold: 2 success-threshold: 2 send-on-resolved: true minimum-reminder-interval: 6h