personal_infra/ansible/infra/402_public_monitoring.yml

---
# Domain expiry, DNS correctness, and public endpoint reachability.
#
# These are the first checks in the estate that PULL rather than push, and that
# is the right way round for them: all three are about how the outside world
# sees us, so they must be measured from outside. Gatus polls from the
# observability host and needs nothing installed anywhere else - there is no
# script, no timer and no push token, because nothing is reporting in. (The
# provider-balance check does carry an API credential - see below.)
#
# That also means these have no heartbeat. A heartbeat answers "did the thing
# that was supposed to report in do so"; when Gatus does the checking itself,
# failure is immediate and self-evident.

- name: Register the public-facing checks with Gatus
  hosts: observability
  become: yes

  vars:
    # Expected A records, derived from inventory rather than written down again.
    # The estate's recurring bug is an address recorded in a second place and
    # then left behind when the machine moved, so the check asserts against
    # ansible_host - if a box is renumbered, inventory is the one edit.
    dns_records:
      - {sub: "{{ subdomains.gatus }}",              host: spacey}
      - {sub: "{{ subdomains.headscale }}",          host: spacey}
      - {sub: "{{ subdomains.vaultwarden }}",        host: vipy}
      - {sub: "{{ subdomains.forgejo }}",            host: vipy}
      - {sub: "{{ subdomains.lnbits }}",             host: vipy}
      - {sub: "{{ subdomains.personal_blog }}",      host: vipy}
      - {sub: "{{ subdomains.memos }}",              host: vipy}
      - {sub: "{{ subdomains.mempool }}",            host: vipy}
      - {sub: "{{ subdomains.datum_gateway }}",      host: vipy}
      - {sub: "{{ subdomains.monero }}",             host: vipy}
      - {sub: "{{ subdomains.monero_explorer }}",    host: vipy}
      - {sub: "{{ subdomains.music }}",              host: vipy}
      - {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy}
      # Not under root_domain, so named in full. arbret.com's zone is NOT in
      # Porkbun (mynymdns) - this is the check that would have caught it still
      # pointing at prd-arbret's recycled IP after the 2026-10-17 rebuild.
      - {fqdn: arbret.com,     host: vipy}
      - {fqdn: www.arbret.com, host: vipy}

    # A public resolver on purpose: this must test what the internet sees, not
    # what a local cache or the tailnet's MagicDNS happens to answer.
    dns_resolver: "1.1.1.1"

    # Expected status per site, checked live before being written down.
    # 401 is the CORRECT answer for the two behind basic auth - asserting 200
    # there would go green precisely when the auth broke.
    public_sites:
      - {name: gatus,       sub: "{{ subdomains.gatus }}",              path: "/",       status: 401}
      - {name: headscale,   sub: "{{ subdomains.headscale }}",          path: "/health", status: 200}
      - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}",        path: "/",       status: 200}
      - {name: forgejo,     sub: "{{ subdomains.forgejo }}",            path: "/",       status: 200}
      - {name: lnbits,      sub: "{{ subdomains.lnbits }}",             path: "/",       status: 200}
      - {name: blog,        sub: "{{ subdomains.personal_blog }}",      path: "/",       status: 200}
      - {name: memos,       sub: "{{ subdomains.memos }}",              path: "/",       status: 200}
      - {name: mempool,     sub: "{{ subdomains.mempool }}",            path: "/",       status: 200}
      - {name: datum,       sub: "{{ subdomains.datum_gateway }}",      path: "/",       status: 401}
      # 401 is CORRECT: monerod answers an unauthenticated request with an HTTP
      # Digest challenge. Asserting 200 here would go green precisely when
      # rpc-login broke and the node started serving the world for free.
      - {name: monero,      sub: "{{ subdomains.monero }}",             path: "/get_info", status: 401}
      - {name: exmrplore,   sub: "{{ subdomains.monero_explorer }}",    path: "/",       status: 200}
      # /app/, not "/": checked live, "/" answers 302 (Navidrome redirects to
      # the SPA) and only /app/ returns 200 with the real UI. 200 rather than
      # 401 because Navidrome has no basic auth in front of it - the Subsonic
      # API the mobile clients speak cannot carry a proxy credential, so its
      # own login is the gate. See roles/navidrome/README.md.
      - {name: music,       sub: "{{ subdomains.music }}",              path: "/app/",   status: 200}
      # /health/ through the whole public path: DNS, vipy's Caddy and cert, the
      # tailnet hop, gunicorn, and the database and bucket behind it (503 if
      # either is down). "/" would only prove the login redirect works.
      - {name: jars-staging, sub: "{{ subdomains.jar_lid_archive_staging }}", path: "/health/", status: 200}
      # arbret (runs on vipy since 2026-10-17; deployed by the arbret repo, not
      # this one). /api/health answers 200 + "status":"ok" when healthy and 503
      # "degraded" otherwise, DB down included - per the arbret owner. The body
      # condition guards against a 200 that is not actually healthy.
      - {name: arbret,        fqdn: arbret.com, path: "/",           status: 200}
      - {name: arbret-health, fqdn: arbret.com, path: "/api/health", status: 200,
         conditions: ['[BODY].status == ok']}

    # Ports published from the edge host by socket_proxy.
    public_tcp:
      - {name: bitcoin-p2p,    host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"}
      - {name: fulcrum-ssl,    host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"}
      - {name: datum-stratum,  host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"}

  tasks:
    # ── Domain expiry ────────────────────────────────────────────────────────
    # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the
    # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole
    # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so
    # whatever the apex serves - a real site, or the registrar's parking page -
    # is irrelevant.
    #
    # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway
    # because it uses a free whois service that must not be hammered.
    # 336h = 14 days of runway, because renewal is a manual act at the registrar.
    - name: Build the domain endpoints
      ansible.builtin.set_fact:
        domain_endpoints: "{{ domain_endpoints | default([]) + [{
          'name': item,
          'group': 'domain',
          'url': 'https://' ~ item,
          'interval': '24h',
          'conditions': ['[DOMAIN_EXPIRATION] > 336h'],
          'alerts': [{'type': 'signal', 'failure-threshold': 1,
                      'success-threshold': 1, 'send-on-resolved': true,
                      'minimum-reminder-interval': '168h'}]}] }}"
      loop: "{{ monitored_domains }}"

    # ── DNS ──────────────────────────────────────────────────────────────────
    # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The
    # domain check stays at 24h because it does a WHOIS/RDAP lookup against a
    # free service. Alert on the first failure - at a 6h interval, waiting for
    # three would be nearly a day.
    - name: Build the DNS endpoints
      ansible.builtin.set_fact:
        dns_endpoints: "{{ dns_endpoints | default([]) + [{
          'name': (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain),
          'group': 'dns',
          'url': dns_resolver,
          'interval': item.interval | default('6h'),
          'dns': {'query-type': 'A', 'query-name': (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain)},
          'conditions': ['[DNS_RCODE] == NOERROR',
                         '[BODY] == ' ~ hostvars[item.host].ansible_host],
          'alerts': [{'type': 'signal', 'failure-threshold': 1,
                      'success-threshold': 1, 'send-on-resolved': true,
                      'minimum-reminder-interval': '24h'}]}] }}"
      loop: "{{ dns_records }}"

    # ── Public HTTP ──────────────────────────────────────────────────────────
    # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has
    # no heartbeat window, so unlike the push checks the tolerance has to live
    # in the threshold - and one failed poll of a public site is usually a blip.
    - name: Build the public HTTP endpoints
      ansible.builtin.set_fact:
        http_endpoints: "{{ http_endpoints | default([]) + [{
          'name': item.name,
          'group': 'public',
          'url': 'https://' ~ (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain) ~ item.path,
          'interval': item.interval | default('5m'),
          'conditions': ['[STATUS] == ' ~ item.status,
                         '[CERTIFICATE_EXPIRATION] > 168h']
                        + item.conditions | default([]),
          'alerts': [{'type': 'signal', 'failure-threshold': item.threshold | default(3),
                      'success-threshold': 2, 'send-on-resolved': true,
                      'minimum-reminder-interval': '6h'}]}] }}"
      loop: "{{ public_sites }}"

    # ── Public TCP ───────────────────────────────────────────────────────────
    - name: Build the public TCP endpoints
      ansible.builtin.set_fact:
        tcp_endpoints: "{{ tcp_endpoints | default([]) + [{
          'name': item.name,
          'group': 'public',
          'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port,
          'interval': '5m',
          'conditions': ['[CONNECTED] == true'],
          'alerts': [{'type': 'signal', 'failure-threshold': 3,
                      'success-threshold': 2, 'send-on-resolved': true,
                      'minimum-reminder-interval': '6h'}]}] }}"
      loop: "{{ public_tcp }}"

    # ── Provider balance ─────────────────────────────────────────────────────
    # 99stack is prepaid: when the balance runs out the servers stop. The one
    # pulled check here that carries a credential - unlike everything above,
    # the API will not answer anonymously (403 "Authorization token is
    # missing").
    #
    # The condition is RUNWAY, not balance. billing.secondsdue is the
    # provider's own "seconds until you run out at current usage", so it
    # follows the spend as servers come and go; a fixed balance threshold
    # would be one more number written down once and left behind. 336h of
    # runway, same as domain expiry, because topping up is a manual act too.
    #
    # The token is account-wide - 99stack has no read-only scope - so it can
    # create and destroy servers. It lands in Gatus's endpoints file (0640
    # root:gatus), the same place as the push tokens.
    - name: Require the 99stack API token
      ansible.builtin.assert:
        that:
          - ninetyninestack_api_token is defined
          - ninetyninestack_api_token | length > 0
        fail_msg: >-
          ninetyninestack_api_token must be in group_vars/all/vault.yml - an
          API token from the 99stack control panel.
        quiet: true

    - name: Build the provider endpoints
      ansible.builtin.set_fact:
        provider_endpoints:
          - name: 99stack-balance
            group: provider
            url: https://api.99stack.com/v1.5/account/info
            interval: 6h
            headers:
              Authorization: "Bearer {{ ninetyninestack_api_token }}"
            conditions:
              - "[STATUS] == 200"
              - "[BODY].billing.secondsdue > {{ 14 * 86400 }}"
            alerts:
              - {type: signal, failure-threshold: 1, success-threshold: 1,
                 send-on-resolved: true, minimum-reminder-interval: 24h}

    - name: Register the public-facing endpoints
      ansible.builtin.include_role:
        name: gatus_endpoint
      vars:
        gatus_endpoint_name: public
        gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints + provider_endpoints }}"