diff --git a/ansible/host_vars/monerod_box_local/main.yml b/ansible/host_vars/monerod_box_local/main.yml index 181beb1..5a1b130 100644 --- a/ansible/host_vars/monerod_box_local/main.yml +++ b/ansible/host_vars/monerod_box_local/main.yml @@ -29,3 +29,24 @@ monero_data_device: /dev/sdb monitored_services: - monerod - xmrblocks + +# ...but on THIS host, once a day rather than every five minutes. +# +# Both units are a Monero node in the middle of a multi-week initial sync on a +# rotating disk. There is nothing a 5-minute poll reports that a daily one does +# not, and every failing poll was another Signal message - 83 of them in six +# hours while the explorer was wedged. +# +# This is safe to scope to the host because monerod-box runs ONLY Monero units. +# Do not copy it to a box with a mixed workload: the timer is one per host and +# iterates every unit in monitored_services, so slowing it slows all of them. +# +# 26h, not 24h: the window must EXCEED the push interval, or a check reporting +# exactly on schedule races its own deadline and alerts on jitter alone. The +# 900s randomized delay puts the real worst case at ~24h15m. +# +# Host liveness and disk are deliberately NOT slowed - they answer "is the box +# alive", which is still wanted within minutes. See infra/400. +service_health_on_calendar: "*-*-* 05:15:00" +service_health_randomized_delay: "900" +service_health_heartbeat: "26h" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml index ab564a1..94828eb 100644 --- a/ansible/infra/401_service_monitoring.yml +++ b/ansible/infra/401_service_monitoring.yml @@ -64,7 +64,7 @@ 'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1, 'group': 'services', 'token': gatus_push_tokens[item.0.host], - 'heartbeat': '11m'}] }}" + 'heartbeat': hostvars[item.0.host].service_health_heartbeat | default('11m')}] }}" loop: "{{ host_units | subelements('units') }}" - name: Register the service endpoints @@ -98,7 +98,16 @@ healthcheck_check: systemd-units healthcheck_units: "{{ monitored_services }}" healthcheck_units_key_prefix: "services_{{ host_key }}" + # Cadence and heartbeat window are ONE decision, and both are a property + # of the host - monerod-box runs a multi-week sync nobody needs polled + # every five minutes, so it declares a daily schedule in its host_vars + # and a matching window above. Everything else keeps 5min/11m. + # + # An empty healthcheck_on_calendar means "use healthcheck_interval" - + # see the timer template in roles/healthcheck. healthcheck_interval: "5min" + healthcheck_on_calendar: "{{ service_health_on_calendar | default('') }}" + healthcheck_randomized_delay: "{{ service_health_randomized_delay | default('30') }}" healthcheck_boot_delay: "2min" # The per-unit results go to keys under this collection; the role's own # single-result push is unused here, so only the base is set. diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index c24df46..821ead5 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -31,8 +31,11 @@ - {sub: "{{ subdomains.memos }}", host: vipy} - {sub: "{{ subdomains.mempool }}", host: vipy} - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - - {sub: "{{ subdomains.monero }}", host: vipy} - - {sub: "{{ subdomains.monero_explorer }}", host: vipy} + # Daily rather than the 6h default - see the Monero note under + # public_sites. A record that has not moved in a year does not need + # four looks a day while the node behind it is mid-sync. + - {sub: "{{ subdomains.monero }}", host: vipy, interval: '24h'} + - {sub: "{{ subdomains.monero_explorer }}", host: vipy, interval: '24h'} # A public resolver on purpose: this must test what the internet sees, not # what a local cache or the tailnet's MagicDNS happens to answer. @@ -55,8 +58,16 @@ # 401 is CORRECT: monerod answers an unauthenticated request with an HTTP # Digest challenge. Asserting 200 here would go green precisely when # rpc-login broke and the node started serving the world for free. - - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401} - - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200} + # + # Both Monero sites are polled ONCE A DAY, not every 5 minutes. The node + # is a multi-week initial sync on a rotating disk and the explorer reads + # the same LMDB, so a hard poll measures the disk more than the service - + # and a wedged explorer turned that cadence into 83 Signal messages in + # six hours. `threshold` drops with it: 3 failures at 5m is 15 minutes of + # tolerance, but at 24h it would be THREE DAYS before anyone heard. A + # daily check has no room for a blip, so it alerts on the first one. + - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401, interval: '24h', threshold: 1} + - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200, interval: '24h', threshold: 1} # Ports published from the edge host by socket_proxy. public_tcp: @@ -99,7 +110,7 @@ 'name': item.sub ~ '.' ~ root_domain, 'group': 'dns', 'url': dns_resolver, - 'interval': '6h', + 'interval': item.interval | default('6h'), 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, 'conditions': ['[DNS_RCODE] == NOERROR', '[BODY] == ' ~ hostvars[item.host].ansible_host], @@ -118,10 +129,10 @@ 'name': item.name, 'group': 'public', 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, - 'interval': '5m', + 'interval': item.interval | default('5m'), 'conditions': ['[STATUS] == ' ~ item.status, '[CERTIFICATE_EXPIRATION] > 168h'], - 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'alerts': [{'type': 'signal', 'failure-threshold': item.threshold | default(3), 'success-threshold': 2, 'send-on-resolved': true, 'minimum-reminder-interval': '6h'}]}] }}" loop: "{{ public_sites }}" diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index f6e6bf4..802df49 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -45,8 +45,15 @@ - {name: mempool-mariadb, host: mempool_box_local} - {name: mempool-backend, host: mempool_box_local} - {name: mempool-frontend, host: mempool_box_local} - - {name: monerod, host: monerod_box_local} - - {name: xmrblocks, host: monerod_box_local} + # The two Monero probes push ONCE A DAY, not every 5 minutes, so they + # need a window wider than a day. 26h, not 24h: the window must exceed + # the push interval or a check that reports exactly on time races its own + # deadline and alerts on jitter. The timers are OnCalendar 05:00 and + # 05:30 with RandomizedDelaySec=900, so the true worst case is ~24h15m. + # Cadence lives in roles/{monero,xmrblocks}/defaults/main.yml; the two + # are one decision and must move together. + - {name: monerod, host: monerod_box_local, heartbeat: '26h'} + - {name: xmrblocks, host: monerod_box_local, heartbeat: '26h'} tasks: - name: Build the probe endpoint list @@ -55,7 +62,7 @@ 'name': item.name, 'group': 'probe', 'token': gatus_push_tokens[item.host], - 'heartbeat': '11m'}] }}" + 'heartbeat': item.heartbeat | default('11m')}] }}" loop: "{{ probes }}" - name: Register the probe endpoints diff --git a/ansible/roles/monero/defaults/main.yml b/ansible/roles/monero/defaults/main.yml index 633f3c0..3d925ed 100644 --- a/ansible/roles/monero/defaults/main.yml +++ b/ansible/roles/monero/defaults/main.yml @@ -94,3 +94,16 @@ healthcheck_push_url: "" # Bearer token for the Gatus external endpoint. Required whenever a push URL # is set: Gatus rejects an unauthenticated push with 401. healthcheck_push_token: "" + +# WHEN TO CHECK. Daily, at a fixed hour, rather than an OnUnitActiveSec +# interval that drifts with every restart. +# +# Prefixed because the generic `healthcheck` role owns the unprefixed +# healthcheck_on_calendar; an unprefixed default here would shadow it for any +# play that includes both. +# +# Paired with the 26h Gatus heartbeat in +# infra/403_service_probe_registration.yml. Change one and you must change the +# other: the window has to stay above the interval or the check alerts on its +# own jitter. +monero_healthcheck_on_calendar: "*-*-* 05:00:00" diff --git a/ansible/roles/monero/templates/healthcheck.timer.j2 b/ansible/roles/monero/templates/healthcheck.timer.j2 index 8c850a9..c088af5 100644 --- a/ansible/roles/monero/templates/healthcheck.timer.j2 +++ b/ansible/roles/monero/templates/healthcheck.timer.j2 @@ -8,7 +8,17 @@ Description=Monero Node Health Check Timer [Timer] OnBootSec=2min -OnUnitActiveSec=5min +# ONCE A DAY, not every 5 minutes. +# +# A 5-minute poll of monerod buys nothing a daily one does not: the node is a +# multi-week initial sync on a rotating disk, and every failing poll is another +# Signal message. The cadence and the Gatus heartbeat window are ONE decision - +# the window MUST exceed this interval, and it is set to 26h in +# infra/403_service_probe_registration.yml. A 24h push under a 24h window races +# itself and alerts on jitter alone. +OnCalendar={{ monero_healthcheck_on_calendar }} +# Spread against the other daily pushes leaving this box. +RandomizedDelaySec=900 Persistent=true [Install] diff --git a/ansible/roles/xmrblocks/defaults/main.yml b/ansible/roles/xmrblocks/defaults/main.yml index d90cab8..8254393 100644 --- a/ansible/roles/xmrblocks/defaults/main.yml +++ b/ansible/roles/xmrblocks/defaults/main.yml @@ -73,3 +73,9 @@ healthcheck_push_url: "" # Bearer token for the Gatus external endpoint. Required whenever a push URL # is set: Gatus rejects an unauthenticated push with 401. healthcheck_push_token: "" + +# WHEN TO CHECK. Daily, 30 minutes after monerod's own check so the two do not +# contend for the same cold LMDB pages. See roles/monero/defaults for why the +# name is prefixed and why this is paired with the 26h heartbeat in +# infra/403_service_probe_registration.yml. +xmrblocks_healthcheck_on_calendar: "*-*-* 05:30:00" diff --git a/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 index 48713a4..6edf61d 100644 --- a/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 +++ b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 @@ -7,7 +7,14 @@ Description=Onion Monero Blockchain Explorer Health Check Timer [Timer] OnBootSec=2min -OnUnitActiveSec=5min +# ONCE A DAY, not every 5 minutes. See the note in roles/monero's timer: the +# explorer follows the node it reads from, and this check costs a full page +# render out of the LMDB (a cold one measured 68 seconds), so polling it hard +# is not free either. The Gatus heartbeat window is 26h, set in +# infra/403_service_probe_registration.yml, and MUST stay above this interval. +OnCalendar={{ xmrblocks_healthcheck_on_calendar }} +# Spread against the other daily pushes leaving this box. +RandomizedDelaySec=900 Persistent=true [Install]