counterweight
f33f1a4b67
The probes, the unit check and the public checks ran once a day with 26h windows, justified as "a multi-week initial sync on a rotating disk". That stopped being true: the node is synced, and a cold explorer page now renders in 29s and a warm one in 0.2s - polling every 5 minutes is what keeps it warm. A dead monerod could go unnoticed for a day. Back to the standard 5m cadence and 11m heartbeat; the per-host service_health override on monerod-box is gone. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
158 lines
9.4 KiB
YAML
158 lines
9.4 KiB
YAML
---
|
|
# Domain expiry, DNS correctness, and public endpoint reachability.
|
|
#
|
|
# These are the first checks in the estate that PULL rather than push, and that
|
|
# is the right way round for them: all three are about how the outside world
|
|
# sees us, so they must be measured from outside. Gatus polls from the
|
|
# observability host and needs nothing installed anywhere else - there is no
|
|
# script, no timer and no token, because nothing is reporting in.
|
|
#
|
|
# That also means these have no heartbeat. A heartbeat answers "did the thing
|
|
# that was supposed to report in do so"; when Gatus does the checking itself,
|
|
# failure is immediate and self-evident.
|
|
|
|
- name: Register the public-facing checks with Gatus
|
|
hosts: observability
|
|
become: yes
|
|
|
|
vars:
|
|
# Expected A records, derived from inventory rather than written down again.
|
|
# The estate's recurring bug is an address recorded in a second place and
|
|
# then left behind when the machine moved, so the check asserts against
|
|
# ansible_host - if a box is renumbered, inventory is the one edit.
|
|
dns_records:
|
|
- {sub: "{{ subdomains.gatus }}", host: spacey}
|
|
- {sub: "{{ subdomains.headscale }}", host: spacey}
|
|
- {sub: "{{ subdomains.vaultwarden }}", host: vipy}
|
|
- {sub: "{{ subdomains.forgejo }}", host: vipy}
|
|
- {sub: "{{ subdomains.lnbits }}", host: vipy}
|
|
- {sub: "{{ subdomains.personal_blog }}", host: vipy}
|
|
- {sub: "{{ subdomains.memos }}", host: vipy}
|
|
- {sub: "{{ subdomains.mempool }}", host: vipy}
|
|
- {sub: "{{ subdomains.datum_gateway }}", host: vipy}
|
|
- {sub: "{{ subdomains.monero }}", host: vipy}
|
|
- {sub: "{{ subdomains.monero_explorer }}", host: vipy}
|
|
- {sub: "{{ subdomains.music }}", host: vipy}
|
|
- {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy}
|
|
|
|
# A public resolver on purpose: this must test what the internet sees, not
|
|
# what a local cache or the tailnet's MagicDNS happens to answer.
|
|
dns_resolver: "1.1.1.1"
|
|
|
|
# Expected status per site, checked live before being written down.
|
|
# 401 is the CORRECT answer for the two behind basic auth - asserting 200
|
|
# there would go green precisely when the auth broke.
|
|
public_sites:
|
|
- {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401}
|
|
- {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200}
|
|
- {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200}
|
|
- {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200}
|
|
- {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200}
|
|
- {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200}
|
|
- {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200}
|
|
- {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200}
|
|
- {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401}
|
|
# 401 is CORRECT: monerod answers an unauthenticated request with an HTTP
|
|
# Digest challenge. Asserting 200 here would go green precisely when
|
|
# rpc-login broke and the node started serving the world for free.
|
|
- {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401}
|
|
- {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200}
|
|
# /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to
|
|
# the SPA) and only /app/ returns 200 with the real UI. 200 rather than
|
|
# 401 because Navidrome has no basic auth in front of it - the Subsonic
|
|
# API the mobile clients speak cannot carry a proxy credential, so its
|
|
# own login is the gate. See roles/navidrome/README.md.
|
|
- {name: music, sub: "{{ subdomains.music }}", path: "/app/", status: 200}
|
|
# /health/ through the whole public path: DNS, vipy's Caddy and cert, the
|
|
# tailnet hop, gunicorn, and the database and bucket behind it (503 if
|
|
# either is down). "/" would only prove the login redirect works.
|
|
- {name: jars-staging, sub: "{{ subdomains.jar_lid_archive_staging }}", path: "/health/", status: 200}
|
|
|
|
# Ports published from the edge host by socket_proxy.
|
|
public_tcp:
|
|
- {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"}
|
|
- {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"}
|
|
- {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"}
|
|
|
|
tasks:
|
|
# ── Domain expiry ────────────────────────────────────────────────────────
|
|
# Each domain needs a URL SCHEME: Gatus derives the endpoint type from the
|
|
# prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole
|
|
# config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so
|
|
# whatever the apex serves - a real site, or the registrar's parking page -
|
|
# is irrelevant.
|
|
#
|
|
# 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway
|
|
# because it uses a free whois service that must not be hammered.
|
|
# 336h = 14 days of runway, because renewal is a manual act at the registrar.
|
|
- name: Build the domain endpoints
|
|
ansible.builtin.set_fact:
|
|
domain_endpoints: "{{ domain_endpoints | default([]) + [{
|
|
'name': item,
|
|
'group': 'domain',
|
|
'url': 'https://' ~ item,
|
|
'interval': '24h',
|
|
'conditions': ['[DOMAIN_EXPIRATION] > 336h'],
|
|
'alerts': [{'type': 'signal', 'failure-threshold': 1,
|
|
'success-threshold': 1, 'send-on-resolved': true,
|
|
'minimum-reminder-interval': '168h'}]}] }}"
|
|
loop: "{{ monitored_domains }}"
|
|
|
|
# ── DNS ──────────────────────────────────────────────────────────────────
|
|
# 6h, not daily: a DNS query is cheap and a wrong record is an outage. The
|
|
# domain check stays at 24h because it does a WHOIS/RDAP lookup against a
|
|
# free service. Alert on the first failure - at a 6h interval, waiting for
|
|
# three would be nearly a day.
|
|
- name: Build the DNS endpoints
|
|
ansible.builtin.set_fact:
|
|
dns_endpoints: "{{ dns_endpoints | default([]) + [{
|
|
'name': item.sub ~ '.' ~ root_domain,
|
|
'group': 'dns',
|
|
'url': dns_resolver,
|
|
'interval': item.interval | default('6h'),
|
|
'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain},
|
|
'conditions': ['[DNS_RCODE] == NOERROR',
|
|
'[BODY] == ' ~ hostvars[item.host].ansible_host],
|
|
'alerts': [{'type': 'signal', 'failure-threshold': 1,
|
|
'success-threshold': 1, 'send-on-resolved': true,
|
|
'minimum-reminder-interval': '24h'}]}] }}"
|
|
loop: "{{ dns_records }}"
|
|
|
|
# ── Public HTTP ──────────────────────────────────────────────────────────
|
|
# failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has
|
|
# no heartbeat window, so unlike the push checks the tolerance has to live
|
|
# in the threshold - and one failed poll of a public site is usually a blip.
|
|
- name: Build the public HTTP endpoints
|
|
ansible.builtin.set_fact:
|
|
http_endpoints: "{{ http_endpoints | default([]) + [{
|
|
'name': item.name,
|
|
'group': 'public',
|
|
'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path,
|
|
'interval': item.interval | default('5m'),
|
|
'conditions': ['[STATUS] == ' ~ item.status,
|
|
'[CERTIFICATE_EXPIRATION] > 168h'],
|
|
'alerts': [{'type': 'signal', 'failure-threshold': item.threshold | default(3),
|
|
'success-threshold': 2, 'send-on-resolved': true,
|
|
'minimum-reminder-interval': '6h'}]}] }}"
|
|
loop: "{{ public_sites }}"
|
|
|
|
# ── Public TCP ───────────────────────────────────────────────────────────
|
|
- name: Build the public TCP endpoints
|
|
ansible.builtin.set_fact:
|
|
tcp_endpoints: "{{ tcp_endpoints | default([]) + [{
|
|
'name': item.name,
|
|
'group': 'public',
|
|
'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port,
|
|
'interval': '5m',
|
|
'conditions': ['[CONNECTED] == true'],
|
|
'alerts': [{'type': 'signal', 'failure-threshold': 3,
|
|
'success-threshold': 2, 'send-on-resolved': true,
|
|
'minimum-reminder-interval': '6h'}]}] }}"
|
|
loop: "{{ public_tcp }}"
|
|
|
|
- name: Register the public-facing endpoints
|
|
ansible.builtin.include_role:
|
|
name: gatus_endpoint
|
|
vars:
|
|
gatus_endpoint_name: public
|
|
gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}"
|