monero: check every five minutes again
The probes, the unit check and the public checks ran once a day with 26h windows, justified as "a multi-week initial sync on a rotating disk". That stopped being true: the node is synced, and a cold explorer page now renders in 29s and a warm one in 0.2s - polling every 5 minutes is what keeps it warm. A dead monerod could go unnoticed for a day. Back to the standard 5m cadence and 11m heartbeat; the per-host service_health override on monerod-box is gone. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
532974634e
commit
f33f1a4b67
8 changed files with 36 additions and 73 deletions
|
|
@ -29,24 +29,3 @@ monero_data_device: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1
|
||||||
monitored_services:
|
monitored_services:
|
||||||
- monerod
|
- monerod
|
||||||
- xmrblocks
|
- xmrblocks
|
||||||
|
|
||||||
# ...but on THIS host, once a day rather than every five minutes.
|
|
||||||
#
|
|
||||||
# Both units are a Monero node in the middle of a multi-week initial sync on a
|
|
||||||
# rotating disk. There is nothing a 5-minute poll reports that a daily one does
|
|
||||||
# not, and every failing poll was another Signal message - 83 of them in six
|
|
||||||
# hours while the explorer was wedged.
|
|
||||||
#
|
|
||||||
# This is safe to scope to the host because monerod-box runs ONLY Monero units.
|
|
||||||
# Do not copy it to a box with a mixed workload: the timer is one per host and
|
|
||||||
# iterates every unit in monitored_services, so slowing it slows all of them.
|
|
||||||
#
|
|
||||||
# 26h, not 24h: the window must EXCEED the push interval, or a check reporting
|
|
||||||
# exactly on schedule races its own deadline and alerts on jitter alone. The
|
|
||||||
# 900s randomized delay puts the real worst case at ~24h15m.
|
|
||||||
#
|
|
||||||
# Host liveness and disk are deliberately NOT slowed - they answer "is the box
|
|
||||||
# alive", which is still wanted within minutes. See infra/400.
|
|
||||||
service_health_on_calendar: "*-*-* 05:15:00"
|
|
||||||
service_health_randomized_delay: "900"
|
|
||||||
service_health_heartbeat: "26h"
|
|
||||||
|
|
|
||||||
|
|
@ -99,9 +99,9 @@
|
||||||
healthcheck_units: "{{ monitored_services }}"
|
healthcheck_units: "{{ monitored_services }}"
|
||||||
healthcheck_units_key_prefix: "services_{{ host_key }}"
|
healthcheck_units_key_prefix: "services_{{ host_key }}"
|
||||||
# Cadence and heartbeat window are ONE decision, and both are a property
|
# Cadence and heartbeat window are ONE decision, and both are a property
|
||||||
# of the host - monerod-box runs a multi-week sync nobody needs polled
|
# of the host: a host can set service_health_on_calendar and a matching
|
||||||
# every five minutes, so it declares a daily schedule in its host_vars
|
# service_health_heartbeat in its host_vars. None does today; everything
|
||||||
# and a matching window above. Everything else keeps 5min/11m.
|
# runs 5min/11m. (monerod-box was daily during its initial sync.)
|
||||||
#
|
#
|
||||||
# An empty healthcheck_on_calendar means "use healthcheck_interval" -
|
# An empty healthcheck_on_calendar means "use healthcheck_interval" -
|
||||||
# see the timer template in roles/healthcheck.
|
# see the timer template in roles/healthcheck.
|
||||||
|
|
|
||||||
|
|
@ -30,11 +30,8 @@
|
||||||
- {sub: "{{ subdomains.memos }}", host: vipy}
|
- {sub: "{{ subdomains.memos }}", host: vipy}
|
||||||
- {sub: "{{ subdomains.mempool }}", host: vipy}
|
- {sub: "{{ subdomains.mempool }}", host: vipy}
|
||||||
- {sub: "{{ subdomains.datum_gateway }}", host: vipy}
|
- {sub: "{{ subdomains.datum_gateway }}", host: vipy}
|
||||||
# Daily rather than the 6h default - see the Monero note under
|
- {sub: "{{ subdomains.monero }}", host: vipy}
|
||||||
# public_sites. A record that has not moved in a year does not need
|
- {sub: "{{ subdomains.monero_explorer }}", host: vipy}
|
||||||
# four looks a day while the node behind it is mid-sync.
|
|
||||||
- {sub: "{{ subdomains.monero }}", host: vipy, interval: '24h'}
|
|
||||||
- {sub: "{{ subdomains.monero_explorer }}", host: vipy, interval: '24h'}
|
|
||||||
- {sub: "{{ subdomains.music }}", host: vipy}
|
- {sub: "{{ subdomains.music }}", host: vipy}
|
||||||
- {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy}
|
- {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy}
|
||||||
|
|
||||||
|
|
@ -58,16 +55,8 @@
|
||||||
# 401 is CORRECT: monerod answers an unauthenticated request with an HTTP
|
# 401 is CORRECT: monerod answers an unauthenticated request with an HTTP
|
||||||
# Digest challenge. Asserting 200 here would go green precisely when
|
# Digest challenge. Asserting 200 here would go green precisely when
|
||||||
# rpc-login broke and the node started serving the world for free.
|
# rpc-login broke and the node started serving the world for free.
|
||||||
#
|
- {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401}
|
||||||
# Both Monero sites are polled ONCE A DAY, not every 5 minutes. The node
|
- {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200}
|
||||||
# is a multi-week initial sync on a rotating disk and the explorer reads
|
|
||||||
# the same LMDB, so a hard poll measures the disk more than the service -
|
|
||||||
# and a wedged explorer turned that cadence into 83 Signal messages in
|
|
||||||
# six hours. `threshold` drops with it: 3 failures at 5m is 15 minutes of
|
|
||||||
# tolerance, but at 24h it would be THREE DAYS before anyone heard. A
|
|
||||||
# daily check has no room for a blip, so it alerts on the first one.
|
|
||||||
- {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401, interval: '24h', threshold: 1}
|
|
||||||
- {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200, interval: '24h', threshold: 1}
|
|
||||||
# /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to
|
# /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to
|
||||||
# the SPA) and only /app/ returns 200 with the real UI. 200 rather than
|
# the SPA) and only /app/ returns 200 with the real UI. 200 rather than
|
||||||
# 401 because Navidrome has no basic auth in front of it - the Subsonic
|
# 401 because Navidrome has no basic auth in front of it - the Subsonic
|
||||||
|
|
|
||||||
|
|
@ -45,15 +45,12 @@
|
||||||
- {name: mempool-mariadb, host: mempool_box_local}
|
- {name: mempool-mariadb, host: mempool_box_local}
|
||||||
- {name: mempool-backend, host: mempool_box_local}
|
- {name: mempool-backend, host: mempool_box_local}
|
||||||
- {name: mempool-frontend, host: mempool_box_local}
|
- {name: mempool-frontend, host: mempool_box_local}
|
||||||
# The two Monero probes push ONCE A DAY, not every 5 minutes, so they
|
# Every 5 minutes like the rest since 2026-10-17 (they were daily while
|
||||||
# need a window wider than a day. 26h, not 24h: the window must exceed
|
# the node synced on a rotating disk). Cadence lives in
|
||||||
# the push interval or a check that reports exactly on time races its own
|
# roles/{monero,xmrblocks}/defaults/main.yml; it and this window are one
|
||||||
# deadline and alerts on jitter. The timers are OnCalendar 05:00 and
|
# decision and must move together.
|
||||||
# 05:30 with RandomizedDelaySec=900, so the true worst case is ~24h15m.
|
- {name: monerod, host: monerod_box_local}
|
||||||
# Cadence lives in roles/{monero,xmrblocks}/defaults/main.yml; the two
|
- {name: xmrblocks, host: monerod_box_local}
|
||||||
# are one decision and must move together.
|
|
||||||
- {name: monerod, host: monerod_box_local, heartbeat: '26h'}
|
|
||||||
- {name: xmrblocks, host: monerod_box_local, heartbeat: '26h'}
|
|
||||||
- {name: navidrome, host: music_box_local}
|
- {name: navidrome, host: music_box_local}
|
||||||
- {name: jar-lid-archive-staging, host: jar_lid_staging_local}
|
- {name: jar-lid-archive-staging, host: jar_lid_staging_local}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -104,15 +104,17 @@ healthcheck_push_url: ""
|
||||||
# is set: Gatus rejects an unauthenticated push with 401.
|
# is set: Gatus rejects an unauthenticated push with 401.
|
||||||
healthcheck_push_token: ""
|
healthcheck_push_token: ""
|
||||||
|
|
||||||
# WHEN TO CHECK. Daily, at a fixed hour, rather than an OnUnitActiveSec
|
# WHEN TO CHECK. Every 5 minutes, on the calendar rather than OnUnitActiveSec so
|
||||||
# interval that drifts with every restart.
|
# the cadence does not drift with every restart. (It was daily while the node
|
||||||
|
# did its initial sync on a rotating disk; since 2026-10-03 the chain is on NVMe
|
||||||
|
# and fully synced, and get_info answers in well under a second.)
|
||||||
#
|
#
|
||||||
# Prefixed because the generic `healthcheck` role owns the unprefixed
|
# Prefixed because the generic `healthcheck` role owns the unprefixed
|
||||||
# healthcheck_on_calendar; an unprefixed default here would shadow it for any
|
# healthcheck_on_calendar; an unprefixed default here would shadow it for any
|
||||||
# play that includes both.
|
# play that includes both.
|
||||||
#
|
#
|
||||||
# Paired with the 26h Gatus heartbeat in
|
# Paired with the standard 11m Gatus heartbeat in
|
||||||
# infra/403_service_probe_registration.yml. Change one and you must change the
|
# infra/403_service_probe_registration.yml. Change one and you must change the
|
||||||
# other: the window has to stay above the interval or the check alerts on its
|
# other: the window has to stay above the interval or the check alerts on its
|
||||||
# own jitter.
|
# own jitter.
|
||||||
monero_healthcheck_on_calendar: "*-*-* 05:00:00"
|
monero_healthcheck_on_calendar: "*:0/5"
|
||||||
|
|
|
||||||
|
|
@ -8,17 +8,13 @@ Description=Monero Node Health Check Timer
|
||||||
|
|
||||||
[Timer]
|
[Timer]
|
||||||
OnBootSec=2min
|
OnBootSec=2min
|
||||||
# ONCE A DAY, not every 5 minutes.
|
# Every 5 minutes (see monero_healthcheck_on_calendar). The cadence and the
|
||||||
#
|
# Gatus heartbeat window are ONE decision - the window MUST exceed this
|
||||||
# A 5-minute poll of monerod buys nothing a daily one does not: the node is a
|
# interval, and it is the standard 11m in
|
||||||
# multi-week initial sync on a rotating disk, and every failing poll is another
|
# infra/403_service_probe_registration.yml.
|
||||||
# Signal message. The cadence and the Gatus heartbeat window are ONE decision -
|
|
||||||
# the window MUST exceed this interval, and it is set to 26h in
|
|
||||||
# infra/403_service_probe_registration.yml. A 24h push under a 24h window races
|
|
||||||
# itself and alerts on jitter alone.
|
|
||||||
OnCalendar={{ monero_healthcheck_on_calendar }}
|
OnCalendar={{ monero_healthcheck_on_calendar }}
|
||||||
# Spread against the other daily pushes leaving this box.
|
# Small: a delay comparable to the interval would eat the heartbeat's slack.
|
||||||
RandomizedDelaySec=900
|
RandomizedDelaySec=30
|
||||||
Persistent=true
|
Persistent=true
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|
|
||||||
|
|
@ -74,8 +74,9 @@ healthcheck_push_url: ""
|
||||||
# is set: Gatus rejects an unauthenticated push with 401.
|
# is set: Gatus rejects an unauthenticated push with 401.
|
||||||
healthcheck_push_token: ""
|
healthcheck_push_token: ""
|
||||||
|
|
||||||
# WHEN TO CHECK. Daily, 30 minutes after monerod's own check so the two do not
|
# WHEN TO CHECK. Every 5 minutes, offset 2 minutes from monerod's own check so
|
||||||
# contend for the same cold LMDB pages. See roles/monero/defaults for why the
|
# the two do not land together. On NVMe a cold page render measured 29s and a
|
||||||
# name is prefixed and why this is paired with the 26h heartbeat in
|
# warm one 0.2s (2026-10-17); a 5-minute poll is what keeps it warm. See
|
||||||
# infra/403_service_probe_registration.yml.
|
# roles/monero/defaults for why the name is prefixed and why this is paired
|
||||||
xmrblocks_healthcheck_on_calendar: "*-*-* 05:30:00"
|
# with the 11m heartbeat in infra/403_service_probe_registration.yml.
|
||||||
|
xmrblocks_healthcheck_on_calendar: "*:2/5"
|
||||||
|
|
|
||||||
|
|
@ -7,14 +7,13 @@ Description=Onion Monero Blockchain Explorer Health Check Timer
|
||||||
|
|
||||||
[Timer]
|
[Timer]
|
||||||
OnBootSec=2min
|
OnBootSec=2min
|
||||||
# ONCE A DAY, not every 5 minutes. See the note in roles/monero's timer: the
|
# Every 5 minutes (see xmrblocks_healthcheck_on_calendar). The check renders a
|
||||||
# explorer follows the node it reads from, and this check costs a full page
|
# full page out of the LMDB: 29s cold, 0.2s warm on NVMe - frequent polling is
|
||||||
# render out of the LMDB (a cold one measured 68 seconds), so polling it hard
|
# what keeps it warm. The Gatus heartbeat window is the standard 11m, set in
|
||||||
# is not free either. The Gatus heartbeat window is 26h, set in
|
|
||||||
# infra/403_service_probe_registration.yml, and MUST stay above this interval.
|
# infra/403_service_probe_registration.yml, and MUST stay above this interval.
|
||||||
OnCalendar={{ xmrblocks_healthcheck_on_calendar }}
|
OnCalendar={{ xmrblocks_healthcheck_on_calendar }}
|
||||||
# Spread against the other daily pushes leaving this box.
|
# Small: a delay comparable to the interval would eat the heartbeat's slack.
|
||||||
RandomizedDelaySec=900
|
RandomizedDelaySec=30
|
||||||
Persistent=true
|
Persistent=true
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue