From f33f1a4b67cc167e13012d3fa12cf46c4f69dc54 Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 19 Oct 2026 22:10:24 +0200 Subject: [PATCH] monero: check every five minutes again The probes, the unit check and the public checks ran once a day with 26h windows, justified as "a multi-week initial sync on a rotating disk". That stopped being true: the node is synced, and a cold explorer page now renders in 29s and a warm one in 0.2s - polling every 5 minutes is what keeps it warm. A dead monerod could go unnoticed for a day. Back to the standard 5m cadence and 11m heartbeat; the per-host service_health override on monerod-box is gone. Co-Authored-By: Claude Opus 5.5 (1M context) --- ansible/host_vars/monerod_box_local/main.yml | 21 ------------------- ansible/infra/401_service_monitoring.yml | 6 +++--- ansible/infra/402_public_monitoring.yml | 19 ++++------------- .../infra/403_service_probe_registration.yml | 15 ++++++------- ansible/roles/monero/defaults/main.yml | 10 +++++---- .../monero/templates/healthcheck.timer.j2 | 16 ++++++-------- ansible/roles/xmrblocks/defaults/main.yml | 11 +++++----- .../xmrblocks/templates/healthcheck.timer.j2 | 11 +++++----- 8 files changed, 36 insertions(+), 73 deletions(-) diff --git a/ansible/host_vars/monerod_box_local/main.yml b/ansible/host_vars/monerod_box_local/main.yml index c05830b..d9bfef4 100644 --- a/ansible/host_vars/monerod_box_local/main.yml +++ b/ansible/host_vars/monerod_box_local/main.yml @@ -29,24 +29,3 @@ monero_data_device: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1 monitored_services: - monerod - xmrblocks - -# ...but on THIS host, once a day rather than every five minutes. -# -# Both units are a Monero node in the middle of a multi-week initial sync on a -# rotating disk. There is nothing a 5-minute poll reports that a daily one does -# not, and every failing poll was another Signal message - 83 of them in six -# hours while the explorer was wedged. -# -# This is safe to scope to the host because monerod-box runs ONLY Monero units. -# Do not copy it to a box with a mixed workload: the timer is one per host and -# iterates every unit in monitored_services, so slowing it slows all of them. -# -# 26h, not 24h: the window must EXCEED the push interval, or a check reporting -# exactly on schedule races its own deadline and alerts on jitter alone. The -# 900s randomized delay puts the real worst case at ~24h15m. -# -# Host liveness and disk are deliberately NOT slowed - they answer "is the box -# alive", which is still wanted within minutes. See infra/400. -service_health_on_calendar: "*-*-* 05:15:00" -service_health_randomized_delay: "900" -service_health_heartbeat: "26h" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml index 94828eb..c181ba1 100644 --- a/ansible/infra/401_service_monitoring.yml +++ b/ansible/infra/401_service_monitoring.yml @@ -99,9 +99,9 @@ healthcheck_units: "{{ monitored_services }}" healthcheck_units_key_prefix: "services_{{ host_key }}" # Cadence and heartbeat window are ONE decision, and both are a property - # of the host - monerod-box runs a multi-week sync nobody needs polled - # every five minutes, so it declares a daily schedule in its host_vars - # and a matching window above. Everything else keeps 5min/11m. + # of the host: a host can set service_health_on_calendar and a matching + # service_health_heartbeat in its host_vars. None does today; everything + # runs 5min/11m. (monerod-box was daily during its initial sync.) # # An empty healthcheck_on_calendar means "use healthcheck_interval" - # see the timer template in roles/healthcheck. diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index ee6eb9f..d572171 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -30,11 +30,8 @@ - {sub: "{{ subdomains.memos }}", host: vipy} - {sub: "{{ subdomains.mempool }}", host: vipy} - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - # Daily rather than the 6h default - see the Monero note under - # public_sites. A record that has not moved in a year does not need - # four looks a day while the node behind it is mid-sync. - - {sub: "{{ subdomains.monero }}", host: vipy, interval: '24h'} - - {sub: "{{ subdomains.monero_explorer }}", host: vipy, interval: '24h'} + - {sub: "{{ subdomains.monero }}", host: vipy} + - {sub: "{{ subdomains.monero_explorer }}", host: vipy} - {sub: "{{ subdomains.music }}", host: vipy} - {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy} @@ -58,16 +55,8 @@ # 401 is CORRECT: monerod answers an unauthenticated request with an HTTP # Digest challenge. Asserting 200 here would go green precisely when # rpc-login broke and the node started serving the world for free. - # - # Both Monero sites are polled ONCE A DAY, not every 5 minutes. The node - # is a multi-week initial sync on a rotating disk and the explorer reads - # the same LMDB, so a hard poll measures the disk more than the service - - # and a wedged explorer turned that cadence into 83 Signal messages in - # six hours. `threshold` drops with it: 3 failures at 5m is 15 minutes of - # tolerance, but at 24h it would be THREE DAYS before anyone heard. A - # daily check has no room for a blip, so it alerts on the first one. - - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401, interval: '24h', threshold: 1} - - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200, interval: '24h', threshold: 1} + - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401} + - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200} # /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to # the SPA) and only /app/ returns 200 with the real UI. 200 rather than # 401 because Navidrome has no basic auth in front of it - the Subsonic diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index e1ed107..b716faa 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -45,15 +45,12 @@ - {name: mempool-mariadb, host: mempool_box_local} - {name: mempool-backend, host: mempool_box_local} - {name: mempool-frontend, host: mempool_box_local} - # The two Monero probes push ONCE A DAY, not every 5 minutes, so they - # need a window wider than a day. 26h, not 24h: the window must exceed - # the push interval or a check that reports exactly on time races its own - # deadline and alerts on jitter. The timers are OnCalendar 05:00 and - # 05:30 with RandomizedDelaySec=900, so the true worst case is ~24h15m. - # Cadence lives in roles/{monero,xmrblocks}/defaults/main.yml; the two - # are one decision and must move together. - - {name: monerod, host: monerod_box_local, heartbeat: '26h'} - - {name: xmrblocks, host: monerod_box_local, heartbeat: '26h'} + # Every 5 minutes like the rest since 2026-10-17 (they were daily while + # the node synced on a rotating disk). Cadence lives in + # roles/{monero,xmrblocks}/defaults/main.yml; it and this window are one + # decision and must move together. + - {name: monerod, host: monerod_box_local} + - {name: xmrblocks, host: monerod_box_local} - {name: navidrome, host: music_box_local} - {name: jar-lid-archive-staging, host: jar_lid_staging_local} diff --git a/ansible/roles/monero/defaults/main.yml b/ansible/roles/monero/defaults/main.yml index 0558ce6..686b1c9 100644 --- a/ansible/roles/monero/defaults/main.yml +++ b/ansible/roles/monero/defaults/main.yml @@ -104,15 +104,17 @@ healthcheck_push_url: "" # is set: Gatus rejects an unauthenticated push with 401. healthcheck_push_token: "" -# WHEN TO CHECK. Daily, at a fixed hour, rather than an OnUnitActiveSec -# interval that drifts with every restart. +# WHEN TO CHECK. Every 5 minutes, on the calendar rather than OnUnitActiveSec so +# the cadence does not drift with every restart. (It was daily while the node +# did its initial sync on a rotating disk; since 2026-10-03 the chain is on NVMe +# and fully synced, and get_info answers in well under a second.) # # Prefixed because the generic `healthcheck` role owns the unprefixed # healthcheck_on_calendar; an unprefixed default here would shadow it for any # play that includes both. # -# Paired with the 26h Gatus heartbeat in +# Paired with the standard 11m Gatus heartbeat in # infra/403_service_probe_registration.yml. Change one and you must change the # other: the window has to stay above the interval or the check alerts on its # own jitter. -monero_healthcheck_on_calendar: "*-*-* 05:00:00" +monero_healthcheck_on_calendar: "*:0/5" diff --git a/ansible/roles/monero/templates/healthcheck.timer.j2 b/ansible/roles/monero/templates/healthcheck.timer.j2 index c088af5..306a575 100644 --- a/ansible/roles/monero/templates/healthcheck.timer.j2 +++ b/ansible/roles/monero/templates/healthcheck.timer.j2 @@ -8,17 +8,13 @@ Description=Monero Node Health Check Timer [Timer] OnBootSec=2min -# ONCE A DAY, not every 5 minutes. -# -# A 5-minute poll of monerod buys nothing a daily one does not: the node is a -# multi-week initial sync on a rotating disk, and every failing poll is another -# Signal message. The cadence and the Gatus heartbeat window are ONE decision - -# the window MUST exceed this interval, and it is set to 26h in -# infra/403_service_probe_registration.yml. A 24h push under a 24h window races -# itself and alerts on jitter alone. +# Every 5 minutes (see monero_healthcheck_on_calendar). The cadence and the +# Gatus heartbeat window are ONE decision - the window MUST exceed this +# interval, and it is the standard 11m in +# infra/403_service_probe_registration.yml. OnCalendar={{ monero_healthcheck_on_calendar }} -# Spread against the other daily pushes leaving this box. -RandomizedDelaySec=900 +# Small: a delay comparable to the interval would eat the heartbeat's slack. +RandomizedDelaySec=30 Persistent=true [Install] diff --git a/ansible/roles/xmrblocks/defaults/main.yml b/ansible/roles/xmrblocks/defaults/main.yml index 8254393..8661676 100644 --- a/ansible/roles/xmrblocks/defaults/main.yml +++ b/ansible/roles/xmrblocks/defaults/main.yml @@ -74,8 +74,9 @@ healthcheck_push_url: "" # is set: Gatus rejects an unauthenticated push with 401. healthcheck_push_token: "" -# WHEN TO CHECK. Daily, 30 minutes after monerod's own check so the two do not -# contend for the same cold LMDB pages. See roles/monero/defaults for why the -# name is prefixed and why this is paired with the 26h heartbeat in -# infra/403_service_probe_registration.yml. -xmrblocks_healthcheck_on_calendar: "*-*-* 05:30:00" +# WHEN TO CHECK. Every 5 minutes, offset 2 minutes from monerod's own check so +# the two do not land together. On NVMe a cold page render measured 29s and a +# warm one 0.2s (2026-10-17); a 5-minute poll is what keeps it warm. See +# roles/monero/defaults for why the name is prefixed and why this is paired +# with the 11m heartbeat in infra/403_service_probe_registration.yml. +xmrblocks_healthcheck_on_calendar: "*:2/5" diff --git a/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 index 6edf61d..b6c9ba8 100644 --- a/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 +++ b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 @@ -7,14 +7,13 @@ Description=Onion Monero Blockchain Explorer Health Check Timer [Timer] OnBootSec=2min -# ONCE A DAY, not every 5 minutes. See the note in roles/monero's timer: the -# explorer follows the node it reads from, and this check costs a full page -# render out of the LMDB (a cold one measured 68 seconds), so polling it hard -# is not free either. The Gatus heartbeat window is 26h, set in +# Every 5 minutes (see xmrblocks_healthcheck_on_calendar). The check renders a +# full page out of the LMDB: 29s cold, 0.2s warm on NVMe - frequent polling is +# what keeps it warm. The Gatus heartbeat window is the standard 11m, set in # infra/403_service_probe_registration.yml, and MUST stay above this interval. OnCalendar={{ xmrblocks_healthcheck_on_calendar }} -# Spread against the other daily pushes leaving this box. -RandomizedDelaySec=900 +# Small: a delay comparable to the interval would eat the heartbeat's slack. +RandomizedDelaySec=30 Persistent=true [Install]