watchtower is being destroyed. Removed from [vps], with its host_vars, its push token, and the six Gatus endpoints that referenced it (liveness, disk, two systemd services, the ntfy DNS record and the ntfy HTTP check). ntfy went with it - it ran nowhere else - so services/ntfy is deleted, subdomains.ntfy and ntfy_topic are gone from group_vars, and the ntfy playbook is out of site.yml. ntfy_topic already had no readers: the three infra/4xx plays that used it were deleted when their checks were superseded. Two things this exposed. services/ntfy/deploy_ntfy_playbook.yml was pointing at the WRONG MACHINE. It said `hosts: observability`, which resolves to the host `monitoring` (64.226.70.190) - but ntfy ran on watchtower, and ntfy.contrapeso.xyz pointed there. Running it would have installed ntfy on the new VPS. Moot now, but it is the same stale-identity failure as the rest: the group meant watchtower when the play was written, and nobody revisited it when the group changed. Watchtower was in [vps] and NO role group at all, while running caddy, ntfy and Uptime Kuma - nothing in the repo managed any of it. More seriously: ntfy-emergency-app on vipy (avisame.contrapeso.xyz) sends its notifications to https://ntfy.contrapeso.xyz, topic "emergencia". Destroying watchtower breaks it, and it is an EMERGENCY notifier - it would fail silently at exactly the moment it matters. That is NOT resolved here, deliberately: standing ntfy up elsewhere, pointing at ntfy.sh, or retiring the app are all decisions, not cleanups. What this change does is make the break impossible to miss. The URL was derived from subdomains.ntfy, so deleting that would have turned it into an undefined variable buried in a template. It is now an explicit ntfy_service_url in the app's own vars, still holding the old value, with the three options written above it. The ntfy credentials stay in the vault because that app still needs them - the vault was restored from HEAD and only watchtower's push token removed, rather than re-handling the plaintext. Verified: no reference to watchtower or its IP anywhere in the repo; Gatus down from 91 to 85 endpoints, 85 UP, 0 DOWN. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
122 lines
6.8 KiB
YAML
122 lines
6.8 KiB
YAML
---
|
|
# Domain expiry, DNS correctness, and public endpoint reachability.
|
|
#
|
|
# These are the first checks in the estate that PULL rather than push, and that
|
|
# is the right way round for them: all three are about how the outside world
|
|
# sees us, so they must be measured from outside. Gatus polls from the
|
|
# observability host and needs nothing installed anywhere else - there is no
|
|
# script, no timer and no token, because nothing is reporting in.
|
|
#
|
|
# That also means these have no heartbeat. A heartbeat answers "did the thing
|
|
# that was supposed to report in do so"; when Gatus does the checking itself,
|
|
# failure is immediate and self-evident.
|
|
|
|
- name: Register the public-facing checks with Gatus
|
|
hosts: observability
|
|
become: yes
|
|
|
|
vars:
|
|
# Expected A records, derived from inventory rather than written down again.
|
|
# The estate's recurring bug is an address recorded in a second place and
|
|
# then left behind when the machine moved, so the check asserts against
|
|
# ansible_host - if a box is renumbered, inventory is the one edit.
|
|
dns_records:
|
|
- {sub: "{{ subdomains.gatus }}", host: monitoring}
|
|
- {sub: "{{ subdomains.headscale }}", host: spacey}
|
|
- {sub: "{{ subdomains.vaultwarden }}", host: vipy}
|
|
- {sub: "{{ subdomains.forgejo }}", host: vipy}
|
|
- {sub: "{{ subdomains.lnbits }}", host: vipy}
|
|
- {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy}
|
|
- {sub: "{{ subdomains.personal_blog }}", host: vipy}
|
|
- {sub: "{{ subdomains.memos }}", host: vipy}
|
|
- {sub: "{{ subdomains.mempool }}", host: vipy}
|
|
- {sub: "{{ subdomains.datum_gateway }}", host: vipy}
|
|
|
|
# A public resolver on purpose: this must test what the internet sees, not
|
|
# what a local cache or the tailnet's MagicDNS happens to answer.
|
|
dns_resolver: "1.1.1.1"
|
|
|
|
# Expected status per site, checked live before being written down.
|
|
# 401 is the CORRECT answer for the two behind basic auth - asserting 200
|
|
# there would go green precisely when the auth broke.
|
|
public_sites:
|
|
- {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401}
|
|
- {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200}
|
|
- {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200}
|
|
- {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200}
|
|
- {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200}
|
|
- {name: avisame, sub: "{{ subdomains.ntfy_emergency_app }}", path: "/", status: 200}
|
|
- {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200}
|
|
- {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200}
|
|
- {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200}
|
|
- {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401}
|
|
|
|
# Ports published from the edge host by socket_proxy.
|
|
public_tcp:
|
|
- {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"}
|
|
- {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"}
|
|
- {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"}
|
|
|
|
tasks:
|
|
# ── Domain expiry ────────────────────────────────────────────────────────
|
|
# Each domain needs a URL SCHEME: Gatus derives the endpoint type from the
|
|
# prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole
|
|
# config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so
|
|
# whatever the apex serves - a real site, or the registrar's parking page -
|
|
# is irrelevant.
|
|
#
|
|
# 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway
|
|
# because it uses a free whois service that must not be hammered.
|
|
# 336h = 14 days of runway, because renewal is a manual act at the registrar.
|
|
- name: Build the domain endpoints
|
|
ansible.builtin.set_fact:
|
|
domain_endpoints: "{{ domain_endpoints | default([]) + [{
|
|
'name': item,
|
|
'group': 'domain',
|
|
'url': 'https://' ~ item,
|
|
'interval': '24h',
|
|
'conditions': ['[DOMAIN_EXPIRATION] > 336h']}] }}"
|
|
loop: "{{ monitored_domains }}"
|
|
|
|
# ── DNS ──────────────────────────────────────────────────────────────────
|
|
- name: Build the DNS endpoints
|
|
ansible.builtin.set_fact:
|
|
dns_endpoints: "{{ dns_endpoints | default([]) + [{
|
|
'name': item.sub ~ '.' ~ root_domain,
|
|
'group': 'dns',
|
|
'url': dns_resolver,
|
|
'interval': '24h',
|
|
'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain},
|
|
'conditions': ['[DNS_RCODE] == NOERROR',
|
|
'[BODY] == ' ~ hostvars[item.host].ansible_host]}] }}"
|
|
loop: "{{ dns_records }}"
|
|
|
|
# ── Public HTTP ──────────────────────────────────────────────────────────
|
|
- name: Build the public HTTP endpoints
|
|
ansible.builtin.set_fact:
|
|
http_endpoints: "{{ http_endpoints | default([]) + [{
|
|
'name': item.name,
|
|
'group': 'public',
|
|
'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path,
|
|
'interval': '5m',
|
|
'conditions': ['[STATUS] == ' ~ item.status,
|
|
'[CERTIFICATE_EXPIRATION] > 168h']}] }}"
|
|
loop: "{{ public_sites }}"
|
|
|
|
# ── Public TCP ───────────────────────────────────────────────────────────
|
|
- name: Build the public TCP endpoints
|
|
ansible.builtin.set_fact:
|
|
tcp_endpoints: "{{ tcp_endpoints | default([]) + [{
|
|
'name': item.name,
|
|
'group': 'public',
|
|
'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port,
|
|
'interval': '5m',
|
|
'conditions': ['[CONNECTED] == true']}] }}"
|
|
loop: "{{ public_tcp }}"
|
|
|
|
- name: Register the public-facing endpoints
|
|
ansible.builtin.include_role:
|
|
name: gatus_endpoint
|
|
vars:
|
|
gatus_endpoint_name: public
|
|
gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}"
|