From a3d3f063f8d4cf492e9b6e45fb983c8f1d4a8c23 Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 19 Oct 2026 22:10:24 +0200 Subject: [PATCH] gatus: service checks tolerate one failed run The systemd unit checks alerted on failure-threshold 1, on the reasoning that the heartbeat window is the tolerance. That covers a MISSING push, but a unit caught mid-restart - a deploy, an upgrade, a backup stopping it - pushes an explicit false, and each of those paged and resolved ten minutes later. 2 means down on two consecutive 5-minute runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- ansible/infra/401_service_monitoring.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml index c181ba1..088cc7b 100644 --- a/ansible/infra/401_service_monitoring.yml +++ b/ansible/infra/401_service_monitoring.yml @@ -44,6 +44,13 @@ # # Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints # have no built-in tolerance, so the threshold is where it belongs for them. +# +# EXCEPT here, where the threshold is 2. The heartbeat only forgives a MISSING +# push. These checks also push an explicit `false` whenever a unit is caught +# mid-restart - a deploy, an unattended upgrade, a backup stopping the service +# for its dump - and at 1 every one of those pages, then resolves ten minutes +# later. 2 means "down on two consecutive 5-minute runs", i.e. 5-10 minutes. +# A host that stops reporting altogether still alarms on the heartbeat alone. # ───────────────────────────────────────────────────────────────────────────── - name: Register the service checks with Gatus hosts: observability @@ -73,8 +80,9 @@ vars: gatus_endpoint_default_alerts: - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 + # 2: tolerate one explicit `false` from a unit caught mid-restart. + # See the note above. + failure-threshold: 2 success-threshold: 2 send-on-resolved: true minimum-reminder-interval: 6h