--- # Domain expiry, DNS correctness, and public endpoint reachability. # # These are the first checks in the estate that PULL rather than push, and that # is the right way round for them: all three are about how the outside world # sees us, so they must be measured from outside. Gatus polls from the # observability host and needs nothing installed anywhere else - there is no # script, no timer and no push token, because nothing is reporting in. (The # provider-balance check does carry an API credential - see below.) # # That also means these have no heartbeat. A heartbeat answers "did the thing # that was supposed to report in do so"; when Gatus does the checking itself, # failure is immediate and self-evident. - name: Register the public-facing checks with Gatus hosts: observability become: yes vars: # Expected A records, derived from inventory rather than written down again. # The estate's recurring bug is an address recorded in a second place and # then left behind when the machine moved, so the check asserts against # ansible_host - if a box is renumbered, inventory is the one edit. dns_records: - {sub: "{{ subdomains.gatus }}", host: spacey} - {sub: "{{ subdomains.headscale }}", host: spacey} - {sub: "{{ subdomains.vaultwarden }}", host: vipy} - {sub: "{{ subdomains.forgejo }}", host: vipy} - {sub: "{{ subdomains.lnbits }}", host: vipy} - {sub: "{{ subdomains.personal_blog }}", host: vipy} - {sub: "{{ subdomains.memos }}", host: vipy} - {sub: "{{ subdomains.mempool }}", host: vipy} - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - {sub: "{{ subdomains.monero }}", host: vipy} - {sub: "{{ subdomains.monero_explorer }}", host: vipy} - {sub: "{{ subdomains.music }}", host: vipy} - {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy} # Not under root_domain, so named in full. arbret.com's zone is NOT in # Porkbun (mynymdns) - this is the check that would have caught it still # pointing at prd-arbret's recycled IP after the 2026-10-17 rebuild. - {fqdn: arbret.com, host: vipy} - {fqdn: www.arbret.com, host: vipy} # A public resolver on purpose: this must test what the internet sees, not # what a local cache or the tailnet's MagicDNS happens to answer. dns_resolver: "1.1.1.1" # Expected status per site, checked live before being written down. # 401 is the CORRECT answer for the two behind basic auth - asserting 200 # there would go green precisely when the auth broke. public_sites: - {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401} - {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200} - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200} - {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200} - {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200} - {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200} - {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200} - {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200} - {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401} # 401 is CORRECT: monerod answers an unauthenticated request with an HTTP # Digest challenge. Asserting 200 here would go green precisely when # rpc-login broke and the node started serving the world for free. - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401} - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200} # /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to # the SPA) and only /app/ returns 200 with the real UI. 200 rather than # 401 because Navidrome has no basic auth in front of it - the Subsonic # API the mobile clients speak cannot carry a proxy credential, so its # own login is the gate. See roles/navidrome/README.md. - {name: music, sub: "{{ subdomains.music }}", path: "/app/", status: 200} # /health/ through the whole public path: DNS, vipy's Caddy and cert, the # tailnet hop, gunicorn, and the database and bucket behind it (503 if # either is down). "/" would only prove the login redirect works. - {name: jars-staging, sub: "{{ subdomains.jar_lid_archive_staging }}", path: "/health/", status: 200} # arbret (runs on vipy since 2026-10-17; deployed by the arbret repo, not # this one). /api/health answers 200 + "status":"ok" when healthy and 503 # "degraded" otherwise, DB down included - per the arbret owner. The body # condition guards against a 200 that is not actually healthy. - {name: arbret, fqdn: arbret.com, path: "/", status: 200} - {name: arbret-health, fqdn: arbret.com, path: "/api/health", status: 200, conditions: ['[BODY].status == ok']} # Ports published from the edge host by socket_proxy. public_tcp: - {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"} - {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"} - {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"} tasks: # ── Domain expiry ──────────────────────────────────────────────────────── # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so # whatever the apex serves - a real site, or the registrar's parking page - # is irrelevant. # # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway # because it uses a free whois service that must not be hammered. # 336h = 14 days of runway, because renewal is a manual act at the registrar. - name: Build the domain endpoints ansible.builtin.set_fact: domain_endpoints: "{{ domain_endpoints | default([]) + [{ 'name': item, 'group': 'domain', 'url': 'https://' ~ item, 'interval': '24h', 'conditions': ['[DOMAIN_EXPIRATION] > 336h'], 'alerts': [{'type': 'signal', 'failure-threshold': 1, 'success-threshold': 1, 'send-on-resolved': true, 'minimum-reminder-interval': '168h'}]}] }}" loop: "{{ monitored_domains }}" # ── DNS ────────────────────────────────────────────────────────────────── # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The # domain check stays at 24h because it does a WHOIS/RDAP lookup against a # free service. Alert on the first failure - at a 6h interval, waiting for # three would be nearly a day. - name: Build the DNS endpoints ansible.builtin.set_fact: dns_endpoints: "{{ dns_endpoints | default([]) + [{ 'name': (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain), 'group': 'dns', 'url': dns_resolver, 'interval': item.interval | default('6h'), 'dns': {'query-type': 'A', 'query-name': (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain)}, 'conditions': ['[DNS_RCODE] == NOERROR', '[BODY] == ' ~ hostvars[item.host].ansible_host], 'alerts': [{'type': 'signal', 'failure-threshold': 1, 'success-threshold': 1, 'send-on-resolved': true, 'minimum-reminder-interval': '24h'}]}] }}" loop: "{{ dns_records }}" # ── Public HTTP ────────────────────────────────────────────────────────── # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has # no heartbeat window, so unlike the push checks the tolerance has to live # in the threshold - and one failed poll of a public site is usually a blip. - name: Build the public HTTP endpoints ansible.builtin.set_fact: http_endpoints: "{{ http_endpoints | default([]) + [{ 'name': item.name, 'group': 'public', 'url': 'https://' ~ (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain) ~ item.path, 'interval': item.interval | default('5m'), 'conditions': ['[STATUS] == ' ~ item.status, '[CERTIFICATE_EXPIRATION] > 168h'] + item.conditions | default([]), 'alerts': [{'type': 'signal', 'failure-threshold': item.threshold | default(3), 'success-threshold': 2, 'send-on-resolved': true, 'minimum-reminder-interval': '6h'}]}] }}" loop: "{{ public_sites }}" # ── Public TCP ─────────────────────────────────────────────────────────── - name: Build the public TCP endpoints ansible.builtin.set_fact: tcp_endpoints: "{{ tcp_endpoints | default([]) + [{ 'name': item.name, 'group': 'public', 'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port, 'interval': '5m', 'conditions': ['[CONNECTED] == true'], 'alerts': [{'type': 'signal', 'failure-threshold': 3, 'success-threshold': 2, 'send-on-resolved': true, 'minimum-reminder-interval': '6h'}]}] }}" loop: "{{ public_tcp }}" # ── Provider balance ───────────────────────────────────────────────────── # 99stack is prepaid: when the balance runs out the servers stop. The one # pulled check here that carries a credential - unlike everything above, # the API will not answer anonymously (403 "Authorization token is # missing"). # # The condition is RUNWAY, not balance. billing.secondsdue is the # provider's own "seconds until you run out at current usage", so it # follows the spend as servers come and go; a fixed balance threshold # would be one more number written down once and left behind. 336h of # runway, same as domain expiry, because topping up is a manual act too. # # The token is account-wide - 99stack has no read-only scope - so it can # create and destroy servers. It lands in Gatus's endpoints file (0640 # root:gatus), the same place as the push tokens. - name: Require the 99stack API token ansible.builtin.assert: that: - ninetyninestack_api_token is defined - ninetyninestack_api_token | length > 0 fail_msg: >- ninetyninestack_api_token must be in group_vars/all/vault.yml - an API token from the 99stack control panel. quiet: true - name: Build the provider endpoints ansible.builtin.set_fact: provider_endpoints: - name: 99stack-balance group: provider url: https://api.99stack.com/v1.5/account/info interval: 6h headers: Authorization: "Bearer {{ ninetyninestack_api_token }}" conditions: - "[STATUS] == 200" - "[BODY].billing.secondsdue > {{ 14 * 86400 }}" alerts: - {type: signal, failure-threshold: 1, success-threshold: 1, send-on-resolved: true, minimum-reminder-interval: 24h} - name: Register the public-facing endpoints ansible.builtin.include_role: name: gatus_endpoint vars: gatus_endpoint_name: public gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints + provider_endpoints }}"