personal_infra/ansible/infra/402_public_monitoring.yml

---
# Domain expiry, DNS correctness, and public endpoint reachability.
#
# These are the first checks in the estate that PULL rather than push, and that
# is the right way round for them: all three are about how the outside world
# sees us, so they must be measured from outside. Gatus polls from the
# observability host and needs nothing installed anywhere else - there is no
# script, no timer and no token, because nothing is reporting in.
#
# That also means these have no heartbeat. A heartbeat answers "did the thing
# that was supposed to report in do so"; when Gatus does the checking itself,
# failure is immediate and self-evident.

- name: Register the public-facing checks with Gatus
  hosts: observability
  become: yes

  vars:
    # Expected A records, derived from inventory rather than written down again.
    # The estate's recurring bug is an address recorded in a second place and
    # then left behind when the machine moved, so the check asserts against
    # ansible_host - if a box is renumbered, inventory is the one edit.
    dns_records:
      - {sub: "{{ subdomains.gatus }}",              host: monitoring}
      - {sub: "{{ subdomains.headscale }}",          host: spacey}
      - {sub: "{{ subdomains.vaultwarden }}",        host: vipy}
      - {sub: "{{ subdomains.forgejo }}",            host: vipy}
      - {sub: "{{ subdomains.lnbits }}",             host: vipy}
      - {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy}
      - {sub: "{{ subdomains.personal_blog }}",      host: vipy}
      - {sub: "{{ subdomains.memos }}",              host: vipy}
      - {sub: "{{ subdomains.mempool }}",            host: vipy}
      - {sub: "{{ subdomains.datum_gateway }}",      host: vipy}
      # Daily rather than the 6h default - see the Monero note under
      # public_sites. A record that has not moved in a year does not need
      # four looks a day while the node behind it is mid-sync.
      - {sub: "{{ subdomains.monero }}",             host: vipy, interval: '24h'}
      - {sub: "{{ subdomains.monero_explorer }}",    host: vipy, interval: '24h'}
      - {sub: "{{ subdomains.music }}",              host: vipy}
      - {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy}

    # A public resolver on purpose: this must test what the internet sees, not
    # what a local cache or the tailnet's MagicDNS happens to answer.
    dns_resolver: "1.1.1.1"

    # Expected status per site, checked live before being written down.
    # 401 is the CORRECT answer for the two behind basic auth - asserting 200
    # there would go green precisely when the auth broke.
    public_sites:
      - {name: gatus,       sub: "{{ subdomains.gatus }}",              path: "/",       status: 401}
      - {name: headscale,   sub: "{{ subdomains.headscale }}",          path: "/health", status: 200}
      - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}",        path: "/",       status: 200}
      - {name: forgejo,     sub: "{{ subdomains.forgejo }}",            path: "/",       status: 200}
      - {name: lnbits,      sub: "{{ subdomains.lnbits }}",             path: "/",       status: 200}
      - {name: avisame,     sub: "{{ subdomains.ntfy_emergency_app }}", path: "/",       status: 200}
      - {name: blog,        sub: "{{ subdomains.personal_blog }}",      path: "/",       status: 200}
      - {name: memos,       sub: "{{ subdomains.memos }}",              path: "/",       status: 200}
      - {name: mempool,     sub: "{{ subdomains.mempool }}",            path: "/",       status: 200}
      - {name: datum,       sub: "{{ subdomains.datum_gateway }}",      path: "/",       status: 401}
      # 401 is CORRECT: monerod answers an unauthenticated request with an HTTP
      # Digest challenge. Asserting 200 here would go green precisely when
      # rpc-login broke and the node started serving the world for free.
      #
      # Both Monero sites are polled ONCE A DAY, not every 5 minutes. The node
      # is a multi-week initial sync on a rotating disk and the explorer reads
      # the same LMDB, so a hard poll measures the disk more than the service -
      # and a wedged explorer turned that cadence into 83 Signal messages in
      # six hours. `threshold` drops with it: 3 failures at 5m is 15 minutes of
      # tolerance, but at 24h it would be THREE DAYS before anyone heard. A
      # daily check has no room for a blip, so it alerts on the first one.
      - {name: monero,      sub: "{{ subdomains.monero }}",             path: "/get_info", status: 401, interval: '24h', threshold: 1}
      - {name: exmrplore,   sub: "{{ subdomains.monero_explorer }}",    path: "/",       status: 200, interval: '24h', threshold: 1}
      # /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to
      # the SPA) and only /app/ returns 200 with the real UI. 200 rather than
      # 401 because Navidrome has no basic auth in front of it - the Subsonic
      # API the mobile clients speak cannot carry a proxy credential, so its
      # own login is the gate. See roles/navidrome/README.md.
      - {name: music,       sub: "{{ subdomains.music }}",              path: "/app/",   status: 200}
      # /health/ through the whole public path: DNS, vipy's Caddy and cert, the
      # tailnet hop, gunicorn, and the database and bucket behind it (503 if
      # either is down). "/" would only prove the login redirect works.
      - {name: jars-staging, sub: "{{ subdomains.jar_lid_archive_staging }}", path: "/health/", status: 200}

    # Ports published from the edge host by socket_proxy.
    public_tcp:
      - {name: bitcoin-p2p,    host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"}
      - {name: fulcrum-ssl,    host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"}
      - {name: datum-stratum,  host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"}

  tasks:
    # ── Domain expiry ────────────────────────────────────────────────────────
    # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the
    # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole
    # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so
    # whatever the apex serves - a real site, or the registrar's parking page -
    # is irrelevant.
    #
    # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway
    # because it uses a free whois service that must not be hammered.
    # 336h = 14 days of runway, because renewal is a manual act at the registrar.
    - name: Build the domain endpoints
      ansible.builtin.set_fact:
        domain_endpoints: "{{ domain_endpoints | default([]) + [{
          'name': item,
          'group': 'domain',
          'url': 'https://' ~ item,
          'interval': '24h',
          'conditions': ['[DOMAIN_EXPIRATION] > 336h'],
          'alerts': [{'type': 'signal', 'failure-threshold': 1,
                      'success-threshold': 1, 'send-on-resolved': true,
                      'minimum-reminder-interval': '168h'}]}] }}"
      loop: "{{ monitored_domains }}"

    # ── DNS ──────────────────────────────────────────────────────────────────
    # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The
    # domain check stays at 24h because it does a WHOIS/RDAP lookup against a
    # free service. Alert on the first failure - at a 6h interval, waiting for
    # three would be nearly a day.
    - name: Build the DNS endpoints
      ansible.builtin.set_fact:
        dns_endpoints: "{{ dns_endpoints | default([]) + [{
          'name': item.sub ~ '.' ~ root_domain,
          'group': 'dns',
          'url': dns_resolver,
          'interval': item.interval | default('6h'),
          'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain},
          'conditions': ['[DNS_RCODE] == NOERROR',
                         '[BODY] == ' ~ hostvars[item.host].ansible_host],
          'alerts': [{'type': 'signal', 'failure-threshold': 1,
                      'success-threshold': 1, 'send-on-resolved': true,
                      'minimum-reminder-interval': '24h'}]}] }}"
      loop: "{{ dns_records }}"

    # ── Public HTTP ──────────────────────────────────────────────────────────
    # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has
    # no heartbeat window, so unlike the push checks the tolerance has to live
    # in the threshold - and one failed poll of a public site is usually a blip.
    - name: Build the public HTTP endpoints
      ansible.builtin.set_fact:
        http_endpoints: "{{ http_endpoints | default([]) + [{
          'name': item.name,
          'group': 'public',
          'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path,
          'interval': item.interval | default('5m'),
          'conditions': ['[STATUS] == ' ~ item.status,
                         '[CERTIFICATE_EXPIRATION] > 168h'],
          'alerts': [{'type': 'signal', 'failure-threshold': item.threshold | default(3),
                      'success-threshold': 2, 'send-on-resolved': true,
                      'minimum-reminder-interval': '6h'}]}] }}"
      loop: "{{ public_sites }}"

    # ── Public TCP ───────────────────────────────────────────────────────────
    - name: Build the public TCP endpoints
      ansible.builtin.set_fact:
        tcp_endpoints: "{{ tcp_endpoints | default([]) + [{
          'name': item.name,
          'group': 'public',
          'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port,
          'interval': '5m',
          'conditions': ['[CONNECTED] == true'],
          'alerts': [{'type': 'signal', 'failure-threshold': 3,
                      'success-threshold': 2, 'send-on-resolved': true,
                      'minimum-reminder-interval': '6h'}]}] }}"
      loop: "{{ public_tcp }}"

    - name: Register the public-facing endpoints
      ansible.builtin.include_role:
        name: gatus_endpoint
      vars:
        gatus_endpoint_name: public
        gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}"