gatus: show resolved values on successful conditions, ui as a pass-through

Asked to un-hide endpoint properties; the answer is that nothing was hidden.
All six hide-* options (hide-hostname, hide-url, hide-port, hide-conditions,
hide-errors, and dont-resolve-failed-conditions) already default to false
upstream, so hostname, URL, port, conditions and errors were all being shown.

The one setting that genuinely displays MORE is resolve-successful-conditions.
By default a FAILING check resolves its placeholders - "[STATUS] (502) == 200" -
while a PASSING one drops the value and shows only "[STATUS] == 200". With it
on, a healthy DNS check now reads:

    [DNS_RCODE] (NOERROR) == NOERROR
    [BODY] (64.226.70.190) == 64.226.70.190

which says what it actually resolved to rather than merely that the assertion
held. Applied to all 27 pulled endpoints via a gatus_endpoint_default_ui that
each caller can override.

It applies to pulled endpoints ONLY: an external (push) endpoint has no `ui`
field upstream at all, because it carries no conditions - success comes from the
push. The template was initially emitting the block in both loops; emitting an
unknown key into the external-endpoints list risks a parse rejection, and a
rejected config is exactly what skip-invalid-config-update exists to survive.

Also made the page-level `ui` a pass-through dict, the same shape as
gatus_alerting, so every upstream option (description, dashboard-heading, logo,
link, favicon, buttons, custom-css, dark-mode, default-sort-by,
default-filter-by) is reachable without a variable per key. Replaces the two
one-off gatus_ui_title / gatus_ui_header variables.

Set default-sort-by: group, because the dashboard's own grouping toggle starts
OFF and remembers per browser in localStorage - without it the ten groups render
as one flat list of 86 rows for anyone who has not clicked it.

Note the limit of all this: it is configuration. Layout, card design and group
rendering come from the Vue app compiled into the binary (//go:embed static), so
changing those means forking and rebuilding the image - which would discard the
pinned-digest property the deployment relies on.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.gitignore

@@ -1,13 +1,16 @@
 # OpenTofu / Terraform
 .terraform/
 .tofu/
-.terraform.lock.hcl
-.tofu.lock.hcl
 terraform.tfstate
 terraform.tfstate.*
 crash.log
-*.tfvars
+
-*.tfvars.json
+# Provider credentials only. Non-secret infra config (the vms map) is committed
+# as *.auto.tfvars, and *.lock.hcl is committed on purpose so provider hashes
+# are pinned.
+terraform.tfvars
+terraform.tfvars.json
+*secrets.auto.tfvars
 
 venv/*
 .env

ansible/group_vars/all/main.yml

@@ -6,7 +6,6 @@ root_domain: contrapeso.xyz
 # Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the
 # playbooks are kept deliberately — the check logic is meant to be rewired to
 # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
-uptime_kuma_enabled: false
 
 # age recipient for all backup artefacts
 age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza"
@@ -14,3 +13,54 @@ age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr
 # Public key small-backups-box pulls with
 # Authorised on each source host for an unprivileged, dedicated user only
 backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Subdomains. Global because the edge host proxies for services that live on
+# other machines, so no single inventory group covers the readers. Combine with
+# root_domain above to build an FQDN.
+#
+# Moved here from services_config.yml, which 30 plays had to remember to name in
+# vars_files: - a file everyone must opt into is a file someone will forget.
+# ─────────────────────────────────────────────────────────────────────────────
+subdomains:
+  # Monitoring
+  gatus: status
+
+  # VPN infrastructure (spacey)
+  headscale: headscale
+
+  # Core services (vipy)
+  vaultwarden: vault
+  forgejo: forgejo
+  lnbits: wallet
+
+  # Secondary services (vipy)
+  ntfy_emergency_app: avisame
+  personal_blog: pablohere
+
+  # Memos (memos-box)
+  memos: memos
+
+  # Mempool block explorer (mempool-box, proxied via vipy)
+  mempool: mempool
+
+  # DATUM Gateway dashboard (knots-box, proxied via vipy)
+  datum_gateway: datum
+
+# Read by plays across several groups, so global rather than group_vars/<group>.
+headscale_namespace: counter-net
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Domains whose registration expiry is monitored (infra/402_public_monitoring).
+#
+# Registration renewal is a manual act at the registrar, and losing a domain is
+# not recoverable in the way losing a host is - so these are checked daily and
+# alarm with two weeks of runway.
+#
+# root_domain is the estate's own domain; the rest are domains we own that are
+# served from it or from a host in the inventory.
+# ─────────────────────────────────────────────────────────────────────────────
+monitored_domains:
+  - "{{ root_domain }}"
+  - arbret.com

ansible/group_vars/all/vault.yml

@@ -1,60 +1,164 @@
 $ANSIBLE_VAULT;1.1;AES256
-61356165613635386631393135656434646436303665313031346566323336313138353433316463
+34383033613438623432656438313763613639313139626432303637383363643339613166626665
-3363323534613064643132663335623238366431393062340a346538396662306537663163623366
+6537663336353661373437303636613364643736663031300a623461353135623537343763383333
-38626166383933616331623231373137306562623637313263333237633661663436666266616433
+37323731646262613436646231656263396532616132316237353762376363383062646132303435
-3862346438643638650a306634333535653633613534646630386131333236366538333765323333
+3435316666316235660a323163613932616661303935336633656332353766393536393736356539
-38656163303837303732663561373232393132343331663164656262393730326434373731333636
+61396164386266303464373262373862363361363365363235383363613335663933313062663565
-36613538646431396536363936336562616431656665653965373864633366663836353434626434
+39613466353161356462656539613335613536363532333431393935303430373435373737363537
-34383932373461333564303439623565383661646365386665393831383463663662356536356236
+62383266323339373062313134316464303263313830646161333530383736336637333865373764
-36373964666236626465366161636135393734356536633466383262326537343833636630343738
+31313165396366653763343466643561383236643836616662383439666165343139326232326364
-62353066316131613737373162643363653662656261363465386364323962656537373061373032
+32633133373863616163343365656231353939366435346534383462396134653064663566343566
-33343763353464383438363438343965653532393831343930393562633630383932653862623637
+66343464373865383864313264646635316233336133346365396634626261613561373965643339
-38386239353237356631646436356166373961333464396639383538383662326534343339313330
+30353831393037316334313632376537393361336366326561343832373531303139323937613531
-65373339356364636634616532363832386631323062363530313861336238353261353334306235
+34386164303430316162363536386137316330336638303365393462386465633862663932376563
-62616338396431316537346638656365356564346666366366343638356261623664393263323937
+34323862636435306563333065383134343362333733623066656439613339376666353331373665
-30343261363562383332323462336435376664386134646562643836363834313237373631353731
+32353938663364313036373237323062393430306661633732356434333232376338313835663330
-35653535663864643266313332356635363262363533663232656531373130336539633066376139
+32306536393666323464663238306562613936376236613935616664653865373330626365626330
-32316665393831663035623962656364363831333563366135636164346335383738363336663566
+64613066653632646638306338393331376430383165316363323437373362613866313962663432
-63356532643563393939383635386462663561386434323939303431653438653131363538383034
+62356663636630356239613831353066303530363966396135376134633936313530643262353965
-33623933333464363032656636643033326162626163353633343062633966343332383138363963
+38383330613962313935366633353635363638303937316362303866343433353437363334396431
-36383831663562616533316436366566323061386535343538393861383462333166343562316633
+32393134626133353564643233616632376633303934653065613262353436333533376137303862
-30623665623035393537393965626363323132656433313339396233356666346634316332616336
+35303861383833326537376433336634633730373833313036643365373431666438653033653930
-37363635363330323230373332326565343530653335383437373230366563366237633665626331
+33346230396661636333613031353133353631306331666362386366633062666534303539316238
-66623336626230663361636439316337393865383035326136653264666438666566646132353036
+39366634656333646634643032393738333330636230663366623265666533356534623465316665
-38616264313833316536623238633339373466613866626366383835656638623863323838653030
+35336535326237343231653661613736626239316262386363666264613636326332366232333964
-63303938376164653966356435386333363731656666313234663535666165646233313137343563
+38353635616662663133376534306366643037613732343233373336653166316330643638353539
-36386437393139656438333262383437656666343831313239323961373637653163643664356565
+35353532643534613864333839393365313439343663333337373639633832383630316164306132
-62346431393133656530316262303763646165643336396661666431383436323562336137653031
+63636162373034363839656638353534613733663737626432636436373164303239316436313566
-62626538373839613734396366653065306534636630346338316237616161613037616364356431
+66623466396663353136323565643262383865323830616232373466373431373261306537646463
-38643965376133616161336633383664326230383435363334353137303162663738313331346238
+61353966623062356438626635663461396165343366393132393965383734633632666335313030
-39356161616533616134356231323530306338333162343363353531303263636632613036386638
+37333631373239303231396264393031373162373936303462363736313538343562343737356332
-63633136386232306234323936303563646466313935326631396565383432386130656638616266
+63663762633032356130303964386664366137373432353533386566613037613163383963623732
-32383334363237336539396665336366643764633131643663643137376438323666326435626461
+32343765376535343630656633333362393765666131653632613361363163383038323138333537
-64346633636431393137633537306431646564386565303933636434386462346630626537346438
+37373333316466316562613439356365316163336366623435666166653835616531653563343664
-32623333666133303061646564366366326665363163396262633164323631636337346130303239
+30303464663838646333373837363961356465613234636138633162366636633030383337643131
-38373936663337356134666132303165393365663763396362623434633737373538653566646134
+62636236663066316630396132333934353139383465363034363033656161383663656462313939
-62316164396438303532616266313062326666633130656338653139376634306664633031333037
+62353738396132343734343363636164323163616237613861643862303433633934656165343263
-31636166306565353334633435656233336233363664306264626237623366336161303134353433
+35313737373333386335663731643664393630653839396464376639306231653934386336353630
-63383739666462623336386537346662633666626466393039653439346436653937633537396436
+61393735336662663435613431386361373561363531643232303831363163636139653538393931
-35393339383066326630353066623132333034656539363561346462626265363263303535343961
+66313965646562616237383439343637623835303065333730613865666638383131616261346463
-39316461616630326539613731303039613736393633373338646266323938326162373831346336
+31396464306537613135653830653138323539393731626264326335336432666333333735646534
-30656130343463366534323030646238313465306266383034623065636665623366333063383736
+30616439393464326631633635363466303336613135346231626232313361303666323661616638
-38373063393837306462303564643962373334343139626338623935336435643730646532633630
+64363937343231363536353034363966326132333734386638613737636130646232363666643633
-66363730386636633639346463363365343239373265303738353732653633653437636130636664
+61353563636466356630346232613761636432303430336461663434333636623962643764336639
-39386365353334653765343335303263363461313965383664326563333734626533376436626530
+35303565613861323431393036303534303061666437366538306236373930313439353330313632
-63353163323637303730353564383733653365613635353764333266393532653663326533646132
+61363363303562316664303065613863613339626632643931386438326330373938613762303334
-39313763373735323835626437306435373238653432393936643165663663656665316132653330
+33316430363931393262373661623137633835656136313235613666613932313236343966306331
-35383038353532656434366336346235363563636264303734633138323963396562306232646236
+36343031303632323432366337633637336335343638393564313738386162386164613339343662
-38366536306561653937336333373434336164663336613839353439356435333833396363636437
+62383939663265373265613932633265626263623939303638383838646531343433393864363235
-38323934613735643363656233333037336465336564313966623063376566663030303230323262
+37313761396436623833376435393537376136373162393465343764326533666533393061643965
-65643564666534326234306164343365383632333061316238623565353538313538396364313337
+61306365346562333737313962633764313232323161623861336566343735353737633539656162
-35336230313339643736653238386231343661623337306236383665356632366236356335323530
+34323939306661373662663331653333346264323930663636633134666532623438323537383165
-66373831616239376231636361333430343433303233393066323865663434643433303832373262
+66343033363962633766303331623866306462623235373838616565653066646132633034363535
-36366534646161323130393931626362626139663139643263366639656531613436313533363130
+31613462356361613535383963643662373162363363653334393937313039666266366363653537
-65643736363833333939613566663339623964333262643863333030623138633464386238613934
+64356166396139393262313565303731346534646462333638316661383139346362333364613466
-31323639333234336264313663376465323737353766643839303665313737336534386665363034
+38303561316263636136303431393366343936653161336238613439366266346136636465303337
-30326439666433373232306136306365343764643434306561343339353132346430646436343362
+61373634376464333037303862616335623031656133636165636264386265643261373735663266
-32333765616262363930353435616563333736313533653339656231316230346166363335356638
+39396265366138363430343035356337303438346165366361316230656239326633653537626339
-37333561366339613136306438306130343230663732333862663838396463623661303961336433
+39613334316563376133343336343563363564653237633764393532336437303334363830396635
-65646332373139303462303633346432366530643130366133363937653739653036366136373434
+31333963666462313830333163646465356337643263636462356363613266323630393866626535
-343864373734666431326239373866313734
+35336436663063333364353663616564626535396133386461326433323936323161613633353539
+66323935626433353162666335613839346561343264323763303034613366663233613037343538
+66376236373462383430313030373235373337653338333033306431616562353435656433356365
+37653063363435366532306334373438306632306435653334396630343863623938666335656164
+37386634366430383762316435396237313362313965353634613266616532633465646464313662
+35303433363236393465396161346431306231666432666531626136623966663864376663373562
+37653239633732346536323265643434616136326561666330633435383264333937353366653736
+61353430633363323263323163646639663330663632343038623964396433626337353232373334
+32343938356539303938666137323831646562393233323733336464316165623232326132316362
+61313631626234336164656533616231393663363364623130313439623466396231383537656562
+35393163323131383838356130643863633736613635306635306135666261623731613239303962
+61373432336638353339646665653561653738353731373032366361313562306466373365653333
+35306236336331333961613864346165613763383266613236343066623432303766333434333331
+66376263363132616132366165393235626336323665373135653462653533376138346632393363
+35396165383966326635333134663138366136336430393532373935373264616530613762386430
+37643862316232613261343430616438333831623835663239656531356666613032653837373531
+30386232643537306530323936386561646265616339376265353763663833623539653831656634
+63626564633964343663656435636635656562666236386639353332356566316631643431353330
+32643238623932643732353332633535643738663066383830376432346463393464333236386163
+64376165333266323166613564313061643832656239333165393035343836376434626162643061
+31373861393534636236383765313834356638613332626666653762343537363339383939666263
+61366230623930643534353565316165656132396664393138326139613536343738396330653735
+37623463343834626133313231393130646134363331393238663065643930383237326439343831
+32393536613361616637373761393361303936363863333838666633343063386235316237323531
+62633362316436623761616366383336613765373362393161303765613464643632666462343565
+63616430396164346263666436336463623638313130656231643131336663336565616135373433
+39326533643733333564373434333738343634383033353036646534646632383730613161393130
+62653836376435663435353739373536366466646332653030653332353761626466353631663163
+38613734383634663662376464653733306564313761336566336336656366653536643834653631
+35333963306562393234623039613338643266333762333236366331333431343830393864623830
+34323439386133613035363861623332663933313734313637623739613163663730316134303434
+35363239363836666463366334356435666632396132306163303037663430383535393934323537
+30353266373864346662333035393761646632313032383738346439343232663666343833333565
+35363564623038376666363436663830356165323234653230636264303066666230643632356237
+66633734346633366533623966393462313438333965623033343337323135353730613561363236
+30636333613034626162393137323762306432663761393766303163393536656338366664643432
+34636630323965396166356538623165663335333739323034333237306531393734306138343436
+36613937373932633234303336373530663533646635396664343935326334333239646534343336
+35396666353938326337646431353965653836323731313861613031356561376262363965623934
+65336164623134366365303062393161353965363937646363396564313138323938656330333964
+64663862363766613630333531623463616331363533343962653164376665303463336134346662
+66376238663632663932656230323638323135653666363736643065336330636236323064386361
+37646631303138616565383665663634333338613336346438366333636165366635313337373635
+62656236653961393061373262326534393339363236353336613339656462336661663739396261
+35363636653237316436656662633462306333313336313865333037636661383731336466666536
+36646537653431666261353232653233653930346538383438633466346636656432383963333765
+62346363633265306665366630383831303434336139343837616361666539656262633735363064
+30376138386531336430613563366666333964663231363465326230316233323532366632663930
+63653164373931353861653833326130643365373530663039363166366331303134366336613438
+63633336363539636265396331653630633162316366316130326531363463616464376635376538
+32376638353337343639633434613663666437313564333264383766623561393933333561663235
+37666163626538353736383435373530663438363636643665626266316338336532623235353337
+61653762383630616266353438386137653137616532633066623165383663646437343565366237
+33333463613539346136613135386466656135616136663036313661323738363066633038656139
+38353230313861666463383737303038613731316334333031643438343463666164633536313031
+39613663376436643165333362636462303230656336333234363336633262336363643736336231
+66316630613762383832613836333263303038306235663662323162633932653733626130356665
+65316630346534356261373961353130393334396365376237353439326261336339393336303030
+63323439666439313831626639323735653730356565643938376564316337613333616664643865
+30633530383865663731383635366566613565393632336337306334663937396135303539363536
+65656331386532306438383031313164626237393762303636663831666239336637666465633630
+63383961336333663537666561376532376631343132363638633364616531306363623432396332
+31343864626335613131653762363338393738343531626330663663376531336230656365363661
+64363530333965386439376161313534333435393966656231613161633034303964653564363436
+63396131623562303962383137666439373132633330363237353031306561306362666561376435
+66383535333233663736363762323662363264393365643135323834366565653932313863353437
+38363536323739336165356561613335653464363637636234313637383066633865353239393961
+31613532373361383062393238303430353662383930373764373237383035633364323661623661
+62653661623234366666616532663563316234663137616338393036333937633963323635366165
+61613032363531666338353132346365323661626265626461396362646338373536306362303338
+31646564343434633433663039366533656665643235336136333134366231376336343465336539
+37363139346239666232313366613432646265626564313838346364306133306434313337626562
+32346430316532623037633838303363346130623636386134313566383333613565316135373264
+38393261313435323433326130303032393538333963646430646366653363613830613534353832
+32643434613964613239363966376138376661636335373130356430336631333461623734383535
+35356461326136636234663738343738303064386536336462303632303461303733333666383036
+38383563393739343134363038623638393766316533373163336439336238613734656132623163
+32626564666430663339316163393934323133306238353562323866633738363737643937376435
+66306430623334643766623564363239346361393666663766306637313265313833396435626234
+35366239616132623863343365663833363934316362636638303536656631633364646235366566
+39386637613735373761643339396132323031323438316633363464336636316534643435323861
+62663732366430646538626663313035616235643537643234356434323635373962343633336266
+30323031306438656537626430356231393466313334633934623632663661306361313330633339
+33396234386536336331616161363132343765306562303932313963623037366633383765663134
+62636466636433666530363930633765626531613539363832313361666565323635326334326533
+36336439363836663033643262336139613437623633616138656564393032393263393132303031
+31626336663039633362366663306662333432643261613464303939326562653562346239383261
+37373966383635626433363831653936333964626262363839383936356634633233336365623765
+62313031353538636464303234383865613932323164386336316362323731303263346637346534
+32353734343130343364646161643431353230336364366330363261326334613936633234646264
+39623163346663613431323630303034383761663835373565663166366239303130386139396236
+62393565353761636636366462353038366664663430616332326465373364396264323064666536
+61653035633562316634336263636332363733663666626131383161656535383133613939623034
+66363632613733306336323165336534346337386163656631343332323636353539353737316131
+34616336343865656565646338653037333838663736333330376330663834373138363739633064
+61646439636365323838376131663266636235333062616532613936616339633661303634323531
+63356335386534353639323566376134373565353137333134363761666532366431633634333430
+39326662633765626230326130386333393463323433366162363432613234336634663439313339
+33653737333830336264343839396563316462613032376335356634383436353962613636333866
+32333032623635323430356636623138366635386533646133626164316438393937626462313239
+31636635363064363763396566306234363965346438653738333961623435303233396634643763
+65343763386464366636383734336335646464306262623363303934393734333666636635356365
+65353931303862343765343665373139376530656263356365366333323135383838383030613166
+30303963383833643937636539656164336537383337346436303534303035313935323633663765
+333464336235373230316337333632326138

ansible/host_vars/forgejo_runner_local/main.yml

@@ -0,0 +1,12 @@
+---
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - forgejo-runner

ansible/host_vars/fulcrum_box_local/main.yml

@@ -0,0 +1,17 @@
+# fulcrum-box: the Electrum server.
+#
+# Read by the fulcrum role here and by the socket-proxy play on the edge host,
+# which publishes the port. See host_vars/knots_box_local/main.yml for why this
+# lives in host_vars rather than in the role's defaults.
+fulcrum_ssl_port: 50002
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - fulcrum

ansible/host_vars/knots_box_local/main.yml

@@ -0,0 +1,26 @@
+# knots-box: Bitcoin Knots and the DATUM Gateway.
+#
+# These ports are read twice: by the role that deploys the service here, and by
+# the socket-proxy / Caddy plays that run on the EDGE host and publish them.
+# A role default is invisible to that second play, which is why these live in
+# host_vars rather than roles/<svc>/defaults/ - the edge play reads them as
+# hostvars['knots_box_local'].<name>, and the role picks them up automatically
+# because host_vars outranks role defaults.
+#
+# They used to live in services_config.yml, a file 30 plays had to remember to
+# name in vars_files: and that four role defaults silently depended on.
+bitcoin_p2p_port: 8333
+datum_gateway_api_port: 7152
+datum_gateway_stratum_port: 23334
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - bitcoind
+  - datum-gateway

ansible/host_vars/memos_box_local/main.yml

@@ -0,0 +1,12 @@
+---
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - memos

ansible/host_vars/mempool_box_local/main.yml

@@ -0,0 +1,6 @@
+# mempool-box: the Mempool block explorer.
+#
+# Read by the mempool role here and by the Caddy play on the edge host, which
+# proxies to it. See host_vars/knots_box_local/main.yml for why this lives in
+# host_vars rather than in the role's defaults.
+mempool_frontend_port: 8080

ansible/host_vars/monitoring/main.yml

@@ -0,0 +1,12 @@
+---
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - caddy

ansible/host_vars/nodito/main.yml

@@ -14,7 +14,12 @@ systemd_service_name: nodito-cpu-temp-monitor
 
 # ZFS Pool Configuration
 zfs_pool_name: "proxmox-tank-1"
-zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z"  # First disk for RAID 1 mirror
+# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the
+# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently
+# replaced and the repo never caught up. Pool creation is guarded by
+# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been
+# wrong on any disaster-recovery run.
+zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ"  # First disk for RAID 1 mirror
 zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P"  # Second disk for RAID 1 mirror
 zfs_pool_mountpoint: "/var/lib/vz"
 
@@ -26,3 +31,15 @@ ups_port: auto
 ups_user: counterweight
 ups_offdelay: 120  # Seconds after shutdown before UPS cuts outlet power
 ups_ondelay: 30    # Seconds after mains returns before UPS restores outlet power
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - nut-server
+  - nut-monitor

ansible/host_vars/nodito/vault.yml

@@ -0,0 +1,11 @@
+$ANSIBLE_VAULT;1.1;AES256
+30333035323663393939343061323234336164396465623665346165393534646366333332376463
+3364373463333664363334373964323838336531353364310a636636373539623464336630666164
+61376532616339376562373238383436306664313564663266303534346461666466383965323538
+3163313239626663310a613033336332653165333537313366636361663036383031376561613761
+31313563373062333033323037653939663762343161656264633436343361663737626366663732
+39366666626338323436383134646263643538333564313566346336323563663534653161396136
+39333565393538366238643563323630346166643461643063393631643665363566623631373762
+36643866646637306231653837363838656163613766636265383139333838396535626335343163
+32613935353330636263616333666230323436663935326133636362343836323535623237646235
+6266316363366335323162663039366137633865396237373632

ansible/host_vars/spacey/main.yml

@@ -0,0 +1,13 @@
+---
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - headscale
+  - caddy

ansible/host_vars/vipy/main.yml

@@ -0,0 +1,15 @@
+---
+
+# Systemd services deployed on this host, monitored every 5 minutes.
+#
+# The fact lives with the machine rather than in a central map, for the same
+# reason the cross-host ports do: "what runs here" is a property of the host,
+# and a central list is one more thing to forget to update when a service moves.
+#
+# Only units WE deploy belong here. Distro units (ssh, cron) have their own
+# supervision and would be noise.
+monitored_services:
+  - forgejo
+  - lnbits
+  - caddy
+  - phoenixd

ansible/infra/01_user_and_access_setup_playbook.yml

@@ -1,7 +1,5 @@
 - name: Secure Debian
   hosts: managed
-  vars_files:
-    - ../infra_vars.yml
   become: true
 
   tasks:

ansible/infra/02_firewall_and_fail2ban_playbook.yml

@@ -1,7 +1,5 @@
 - name: Secure Debian
   hosts: managed
-  vars_files:
-    - ../infra_vars.yml
   become: true
 
   tasks:

ansible/infra/400_host_monitoring.yml

@@ -0,0 +1,216 @@
+---
+# Host-level monitoring for the whole estate, reported to Gatus.
+#
+# Every check here PUSHES. Gatus never reaches out, which matters because nodito
+# and its VMs sit behind NAT, and because four of the five checks are internal
+# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool
+# health and UPS mains status cannot be observed from outside the machine.
+#
+# Liveness is a push too, and that is a choice rather than a limitation. A
+# heartbeat proves the host is running AND can reach the internet; an ICMP probe
+# from one vantage point only proves it answers pings from there. And because
+# Gatus alerts when a heartbeat window expires, a check that stops running
+# raises the alarm by itself - a dead timer looks exactly like a dead host,
+# which is the correct reading.
+#
+# Each host has ONE bearer token, shared across its own checks: a token can only
+# write results for that host's endpoints, so a compromised host can lie about
+# itself, which it could do anyway.
+#
+# The push URL must use Gatus's own key format (config/key/key.go):
+#     key = sanitize(group) + "_" + sanitize(name)
+# where sanitize lowercases and replaces / _ . , space # + & with "-". So
+# knots_box_local becomes knots-box-local in the URL but stays readable in the
+# name. host_key below is the Jinja equivalent; do not hand-write these.
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Register everything with Gatus.
+#
+# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the
+# host plays below take minutes, so every endpoint exists before its first push
+# arrives. Registering afterwards would 404 every first report.
+#
+# Heartbeat windows are several times the check interval, so one missed run - a
+# slow apt run, a reboot - does not raise an alarm, but a check that has
+# genuinely stopped does.
+# ─────────────────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────────
+# Alerting thresholds, and why they differ by check type.
+#
+# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
+# different amount of wall-clock time per check:
+#
+#   push/heartbeat endpoints   a failure is produced once per heartbeat window
+#   pulled endpoints           a failure is produced once per interval
+#
+# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
+# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
+# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
+# threshold of 3 on top triples a tolerance that was already chosen.
+#
+# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
+# have no built-in tolerance, so the threshold is where it belongs for them.
+# ─────────────────────────────────────────────────────────────────────────────
+- name: Register the host checks with Gatus
+  hosts: observability
+  become: yes
+  vars:
+    monitored: "{{ groups['managed'] | sort }}"
+
+  tasks:
+    - name: Build the liveness endpoint list
+      ansible.builtin.set_fact:
+        liveness_endpoints: "{{ liveness_endpoints | default([]) + [{
+          'name': item,
+          'group': 'liveness',
+          'token': gatus_push_tokens[item],
+          'heartbeat': '11m'}] }}"
+      loop: "{{ monitored }}"
+
+    - name: Build the disk endpoint list
+      ansible.builtin.set_fact:
+        disk_endpoints: "{{ disk_endpoints | default([]) + [{
+          'name': item,
+          'group': 'disk',
+          'token': gatus_push_tokens[item],
+          'heartbeat': '7h'}] }}"
+      loop: "{{ monitored }}"
+
+    - name: Register liveness endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        gatus_endpoint_default_alerts:
+          - type: signal
+            # 1, not 3: the heartbeat window is the tolerance. See the note above.
+            failure-threshold: 1
+            success-threshold: 2
+            send-on-resolved: true
+            minimum-reminder-interval: 6h
+        gatus_endpoint_name: liveness
+        gatus_endpoint_external: "{{ liveness_endpoints }}"
+
+    - name: Register disk endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        gatus_endpoint_default_alerts:
+          - type: signal
+            # 1, not 3: the heartbeat window is the tolerance. See the note above.
+            failure-threshold: 1
+            success-threshold: 2
+            send-on-resolved: true
+            minimum-reminder-interval: 6h
+        gatus_endpoint_name: disk
+        gatus_endpoint_external: "{{ disk_endpoints }}"
+
+    - name: Register the hypervisor endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        gatus_endpoint_default_alerts:
+          - type: signal
+            # 1, not 3: the heartbeat window is the tolerance. See the note above.
+            failure-threshold: 1
+            success-threshold: 2
+            send-on-resolved: true
+            minimum-reminder-interval: 6h
+        gatus_endpoint_name: hypervisor
+        gatus_endpoint_external:
+          - name: cpu
+            group: hypervisor
+            token: "{{ gatus_push_tokens['nodito'] }}"
+            heartbeat: "11m"
+          - name: zfs
+            group: hypervisor
+            token: "{{ gatus_push_tokens['nodito'] }}"
+            heartbeat: "7h"
+          - name: ups
+            group: hypervisor
+            token: "{{ gatus_push_tokens['nodito'] }}"
+            heartbeat: "11m"
+
+- name: Deploy host liveness and disk checks
+  hosts: managed
+  become: yes
+  vars:
+    gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
+    host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}"
+    host_token: "{{ gatus_push_tokens[inventory_hostname] }}"
+
+  tasks:
+    - name: Is the host up?
+      ansible.builtin.include_role:
+        name: healthcheck
+      vars:
+        healthcheck_name: liveness
+        healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}"
+        healthcheck_check: liveness
+        healthcheck_interval: "5min"
+        healthcheck_boot_delay: "1min"
+        healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external"
+        healthcheck_push_token: "{{ host_token }}"
+
+    - name: Is the disk packed?
+      ansible.builtin.include_role:
+        name: healthcheck
+      vars:
+        healthcheck_name: disk-usage
+        healthcheck_description: "Disk usage for {{ inventory_hostname }}"
+        healthcheck_check: disk-usage
+        # Every 6h rather than daily. Disk usage itself moves slowly, but the
+        # heartbeat can only be as tight as the push frequency - a daily push
+        # forces a >24h window, and a stuck check then hides for a day and a
+        # half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the
+        # hosts so twelve boxes do not all report in the same second.
+        healthcheck_on_calendar: "*-*-* 00/6:00:00"
+        healthcheck_randomized_delay: "900"
+        healthcheck_boot_delay: "5min"
+        healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external"
+        healthcheck_push_token: "{{ host_token }}"
+
+- name: Deploy the hypervisor-only checks
+  hosts: hypervisor
+  become: yes
+  vars:
+    gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
+    host_token: "{{ gatus_push_tokens[inventory_hostname] }}"
+
+  tasks:
+    - name: Is the CPU hot?
+      ansible.builtin.include_role:
+        name: healthcheck
+      vars:
+        healthcheck_name: cpu-temp
+        healthcheck_description: "CPU temperature for {{ inventory_hostname }}"
+        healthcheck_check: cpu-temp
+        healthcheck_packages: [curl, lm-sensors]
+        healthcheck_interval: "5min"
+        healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external"
+        healthcheck_push_token: "{{ host_token }}"
+
+    - name: Is ZFS broken?
+      ansible.builtin.include_role:
+        name: healthcheck
+      vars:
+        healthcheck_name: zfs-health
+        healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}"
+        healthcheck_check: zfs-health
+        healthcheck_packages: [curl, jq]
+        healthcheck_zfs_pool: "{{ zfs_pool_name }}"
+        healthcheck_on_calendar: "*-*-* 00/6:20:00"
+        healthcheck_boot_delay: "10min"
+        healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external"
+        healthcheck_push_token: "{{ host_token }}"
+
+    - name: Is the UPS online?
+      ansible.builtin.include_role:
+        name: healthcheck
+      vars:
+        healthcheck_name: ups-status
+        healthcheck_description: "UPS mains status for {{ ups_name }}"
+        healthcheck_check: ups-status
+        healthcheck_ups_name: "{{ ups_name }}"
+        healthcheck_interval: "5min"
+        healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external"
+        healthcheck_push_token: "{{ host_token }}"

ansible/infra/401_service_monitoring.yml

@@ -0,0 +1,107 @@
+---
+# Is each systemd-deployed service actually running?
+#
+# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before
+# it alarms, so a reboot or a slow check does not page anyone, but a host that
+# stops reporting does.
+#
+# This closes the gap that let a real bug run unnoticed: a backup script left
+# forgejo, lnbits, headscale and memos stopped, and NOTHING caught it. The dumps
+# exited 0, the artefacts were correct, the deploy said failed=0, and liveness
+# only proves the HOST is up - not that anything on it is serving.
+#
+# One endpoint PER UNIT, not per host. A host running four services needs four
+# endpoints, or a single red light says "something on vipy is down" without
+# saying which - and that is the question you actually have at 3am. But only ONE
+# timer per host: the check iterates that host's units and pushes a result for
+# each, the same way check-backups.sh reports per source. Four units on vipy
+# would otherwise mean four scripts, four services and four timers.
+#
+# Which units each host runs is in host_vars/<host>/main.yml as
+# monitored_services, because "what runs here" is a property of the machine.
+#
+# Keys are host-qualified because unit names collide - caddy runs on four
+# machines. Gatus computes sanitize(group)_sanitize(name), so group "services"
+# and name "vipy/caddy" give services_vipy-caddy.
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the
+# host play above takes minutes, so every endpoint exists before its first push.
+# ─────────────────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────────
+# Alerting thresholds, and why they differ by check type.
+#
+# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
+# different amount of wall-clock time per check:
+#
+#   push/heartbeat endpoints   a failure is produced once per heartbeat window
+#   pulled endpoints           a failure is produced once per interval
+#
+# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
+# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
+# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
+# threshold of 3 on top triples a tolerance that was already chosen.
+#
+# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
+# have no built-in tolerance, so the threshold is where it belongs for them.
+# ─────────────────────────────────────────────────────────────────────────────
+- name: Register the service checks with Gatus
+  hosts: observability
+  become: yes
+
+  tasks:
+    # Two plain steps rather than one clever expression: first collect which
+    # units each host declares, then flatten that into endpoints.
+    - name: Collect the units each host declares
+      ansible.builtin.set_fact:
+        host_units: "{{ host_units | default([]) + [{'host': item, 'units': hostvars[item].monitored_services}] }}"
+      loop: "{{ groups['managed'] | sort }}"
+      when: hostvars[item].monitored_services | default([]) | length > 0
+
+    - name: Build one endpoint per unit
+      ansible.builtin.set_fact:
+        service_endpoints: "{{ service_endpoints | default([]) + [{
+          'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1,
+          'group': 'services',
+          'token': gatus_push_tokens[item.0.host],
+          'heartbeat': '11m'}] }}"
+      loop: "{{ host_units | subelements('units') }}"
+
+    - name: Register the service endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        gatus_endpoint_default_alerts:
+          - type: signal
+            # 1, not 3: the heartbeat window is the tolerance. See the note above.
+            failure-threshold: 1
+            success-threshold: 2
+            send-on-resolved: true
+            minimum-reminder-interval: 6h
+        gatus_endpoint_name: services
+        gatus_endpoint_external: "{{ service_endpoints }}"
+
+- name: Monitor systemd services on every host that has them
+  hosts: managed
+  become: yes
+  vars:
+    gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
+    host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}"
+
+  tasks:
+    - name: Is every deployed service running?
+      ansible.builtin.include_role:
+        name: healthcheck
+      vars:
+        healthcheck_name: service-health
+        healthcheck_description: "systemd services on {{ inventory_hostname }}"
+        healthcheck_check: systemd-units
+        healthcheck_units: "{{ monitored_services }}"
+        healthcheck_units_key_prefix: "services_{{ host_key }}"
+        healthcheck_interval: "5min"
+        healthcheck_boot_delay: "2min"
+        # The per-unit results go to keys under this collection; the role's own
+        # single-result push is unused here, so only the base is set.
+        healthcheck_push_base: "{{ gatus_api }}"
+        healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"
+      when: monitored_services | default([]) | length > 0

ansible/infra/402_public_monitoring.yml

@@ -0,0 +1,141 @@
+---
+# Domain expiry, DNS correctness, and public endpoint reachability.
+#
+# These are the first checks in the estate that PULL rather than push, and that
+# is the right way round for them: all three are about how the outside world
+# sees us, so they must be measured from outside. Gatus polls from the
+# observability host and needs nothing installed anywhere else - there is no
+# script, no timer and no token, because nothing is reporting in.
+#
+# That also means these have no heartbeat. A heartbeat answers "did the thing
+# that was supposed to report in do so"; when Gatus does the checking itself,
+# failure is immediate and self-evident.
+
+- name: Register the public-facing checks with Gatus
+  hosts: observability
+  become: yes
+
+  vars:
+    # Expected A records, derived from inventory rather than written down again.
+    # The estate's recurring bug is an address recorded in a second place and
+    # then left behind when the machine moved, so the check asserts against
+    # ansible_host - if a box is renumbered, inventory is the one edit.
+    dns_records:
+      - {sub: "{{ subdomains.gatus }}",              host: monitoring}
+      - {sub: "{{ subdomains.headscale }}",          host: spacey}
+      - {sub: "{{ subdomains.vaultwarden }}",        host: vipy}
+      - {sub: "{{ subdomains.forgejo }}",            host: vipy}
+      - {sub: "{{ subdomains.lnbits }}",             host: vipy}
+      - {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy}
+      - {sub: "{{ subdomains.personal_blog }}",      host: vipy}
+      - {sub: "{{ subdomains.memos }}",              host: vipy}
+      - {sub: "{{ subdomains.mempool }}",            host: vipy}
+      - {sub: "{{ subdomains.datum_gateway }}",      host: vipy}
+
+    # A public resolver on purpose: this must test what the internet sees, not
+    # what a local cache or the tailnet's MagicDNS happens to answer.
+    dns_resolver: "1.1.1.1"
+
+    # Expected status per site, checked live before being written down.
+    # 401 is the CORRECT answer for the two behind basic auth - asserting 200
+    # there would go green precisely when the auth broke.
+    public_sites:
+      - {name: gatus,       sub: "{{ subdomains.gatus }}",              path: "/",       status: 401}
+      - {name: headscale,   sub: "{{ subdomains.headscale }}",          path: "/health", status: 200}
+      - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}",        path: "/",       status: 200}
+      - {name: forgejo,     sub: "{{ subdomains.forgejo }}",            path: "/",       status: 200}
+      - {name: lnbits,      sub: "{{ subdomains.lnbits }}",             path: "/",       status: 200}
+      - {name: avisame,     sub: "{{ subdomains.ntfy_emergency_app }}", path: "/",       status: 200}
+      - {name: blog,        sub: "{{ subdomains.personal_blog }}",      path: "/",       status: 200}
+      - {name: memos,       sub: "{{ subdomains.memos }}",              path: "/",       status: 200}
+      - {name: mempool,     sub: "{{ subdomains.mempool }}",            path: "/",       status: 200}
+      - {name: datum,       sub: "{{ subdomains.datum_gateway }}",      path: "/",       status: 401}
+
+    # Ports published from the edge host by socket_proxy.
+    public_tcp:
+      - {name: bitcoin-p2p,    host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"}
+      - {name: fulcrum-ssl,    host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"}
+      - {name: datum-stratum,  host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"}
+
+  tasks:
+    # ── Domain expiry ────────────────────────────────────────────────────────
+    # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the
+    # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole
+    # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so
+    # whatever the apex serves - a real site, or the registrar's parking page -
+    # is irrelevant.
+    #
+    # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway
+    # because it uses a free whois service that must not be hammered.
+    # 336h = 14 days of runway, because renewal is a manual act at the registrar.
+    - name: Build the domain endpoints
+      ansible.builtin.set_fact:
+        domain_endpoints: "{{ domain_endpoints | default([]) + [{
+          'name': item,
+          'group': 'domain',
+          'url': 'https://' ~ item,
+          'interval': '24h',
+          'conditions': ['[DOMAIN_EXPIRATION] > 336h'],
+          'alerts': [{'type': 'signal', 'failure-threshold': 1,
+                      'success-threshold': 1, 'send-on-resolved': true,
+                      'minimum-reminder-interval': '168h'}]}] }}"
+      loop: "{{ monitored_domains }}"
+
+    # ── DNS ──────────────────────────────────────────────────────────────────
+    # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The
+    # domain check stays at 24h because it does a WHOIS/RDAP lookup against a
+    # free service. Alert on the first failure - at a 6h interval, waiting for
+    # three would be nearly a day.
+    - name: Build the DNS endpoints
+      ansible.builtin.set_fact:
+        dns_endpoints: "{{ dns_endpoints | default([]) + [{
+          'name': item.sub ~ '.' ~ root_domain,
+          'group': 'dns',
+          'url': dns_resolver,
+          'interval': '6h',
+          'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain},
+          'conditions': ['[DNS_RCODE] == NOERROR',
+                         '[BODY] == ' ~ hostvars[item.host].ansible_host],
+          'alerts': [{'type': 'signal', 'failure-threshold': 1,
+                      'success-threshold': 1, 'send-on-resolved': true,
+                      'minimum-reminder-interval': '24h'}]}] }}"
+      loop: "{{ dns_records }}"
+
+    # ── Public HTTP ──────────────────────────────────────────────────────────
+    # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has
+    # no heartbeat window, so unlike the push checks the tolerance has to live
+    # in the threshold - and one failed poll of a public site is usually a blip.
+    - name: Build the public HTTP endpoints
+      ansible.builtin.set_fact:
+        http_endpoints: "{{ http_endpoints | default([]) + [{
+          'name': item.name,
+          'group': 'public',
+          'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path,
+          'interval': '5m',
+          'conditions': ['[STATUS] == ' ~ item.status,
+                         '[CERTIFICATE_EXPIRATION] > 168h'],
+          'alerts': [{'type': 'signal', 'failure-threshold': 3,
+                      'success-threshold': 2, 'send-on-resolved': true,
+                      'minimum-reminder-interval': '6h'}]}] }}"
+      loop: "{{ public_sites }}"
+
+    # ── Public TCP ───────────────────────────────────────────────────────────
+    - name: Build the public TCP endpoints
+      ansible.builtin.set_fact:
+        tcp_endpoints: "{{ tcp_endpoints | default([]) + [{
+          'name': item.name,
+          'group': 'public',
+          'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port,
+          'interval': '5m',
+          'conditions': ['[CONNECTED] == true'],
+          'alerts': [{'type': 'signal', 'failure-threshold': 3,
+                      'success-threshold': 2, 'send-on-resolved': true,
+                      'minimum-reminder-interval': '6h'}]}] }}"
+      loop: "{{ public_tcp }}"
+
+    - name: Register the public-facing endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        gatus_endpoint_name: public
+        gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}"

ansible/infra/403_service_probe_registration.yml

@@ -0,0 +1,71 @@
+---
+# The per-service health probes.
+#
+# These are NOT the same thing as the systemd checks in infra/401. Those answer
+# "is the unit running"; these answer "does the service actually respond" - an
+# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch
+# from the Mempool backend. A process can be perfectly `active` and useless,
+# which is precisely the gap these close.
+#
+# The checks themselves live in each service's own role, deployed by that
+# service's playbook. This play only registers where they report, because the
+# endpoints must exist in Gatus before the first push arrives.
+#
+# They used to push to Uptime Kuma. The scripts now POST with a bearer token
+# instead of GETting ?status=up, and each host uses its own token.
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Alerting thresholds, and why they differ by check type.
+#
+# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
+# different amount of wall-clock time per check:
+#
+#   push/heartbeat endpoints   a failure is produced once per heartbeat window
+#   pulled endpoints           a failure is produced once per interval
+#
+# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
+# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
+# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
+# threshold of 3 on top triples a tolerance that was already chosen.
+#
+# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
+# have no built-in tolerance, so the threshold is where it belongs for them.
+# ─────────────────────────────────────────────────────────────────────────────
+- name: Register the per-service probes with Gatus
+  hosts: observability
+  become: yes
+
+  vars:
+    probes:
+      - {name: bitcoin-knots,    host: knots_box_local}
+      - {name: datum-gateway,    host: knots_box_local}
+      - {name: fulcrum,          host: fulcrum_box_local}
+      - {name: phoenixd,         host: vipy}
+      - {name: forgejo-runner,   host: forgejo_runner_local}
+      - {name: mempool-mariadb,  host: mempool_box_local}
+      - {name: mempool-backend,  host: mempool_box_local}
+      - {name: mempool-frontend, host: mempool_box_local}
+
+  tasks:
+    - name: Build the probe endpoint list
+      ansible.builtin.set_fact:
+        probe_endpoints: "{{ probe_endpoints | default([]) + [{
+          'name': item.name,
+          'group': 'probe',
+          'token': gatus_push_tokens[item.host],
+          'heartbeat': '11m'}] }}"
+      loop: "{{ probes }}"
+
+    - name: Register the probe endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        gatus_endpoint_default_alerts:
+          - type: signal
+            # 1, not 3: the heartbeat window is the tolerance. See the note above.
+            failure-threshold: 1
+            success-threshold: 2
+            send-on-resolved: true
+            minimum-reminder-interval: 6h
+        gatus_endpoint_name: probes
+        gatus_endpoint_external: "{{ probe_endpoints }}"

ansible/infra/409_remove_legacy_monitoring.yml

@@ -0,0 +1,109 @@
+---
+# Remove the Uptime-Kuma-era monitoring that 400/401/402 replaced.
+#
+# Deleting the playbooks that installed these is NOT enough: the units are on
+# the hosts, enabled, and keep firing regardless of what the repo says. Two of
+# them still push to https://uptime.contrapeso.xyz every 15 minutes. A playbook
+# that is deleted without a cleanup leaves its output running forever, with
+# nothing in the repo left to explain it.
+#
+# What replaced what, all verified against the deployed scripts before removal:
+#
+#   disk-usage-monitor       -> disk-usage-healthcheck   (infra/400)
+#       The old one checked ONLY "/" at 80%. The replacement walks every real
+#       filesystem, excluding tmpfs/devtmpfs/squashfs/overlay, at 85%. Strictly
+#       more coverage, so nothing is lost.
+#
+#   system-healthcheck       -> liveness-healthcheck     (infra/400)
+#       The old script computed uptime and pushed. That is exactly a liveness
+#       heartbeat and nothing more.
+#
+#   nodito-cpu-temp-monitor  -> cpu-temp-healthcheck     (infra/400)
+#   zfs-health-monitor       -> zfs-health-healthcheck   (infra/400)
+#       The ZFS check logic was ported verbatim - same five conditions - so only
+#       the reporting transport changed.
+#
+# NOT removed, because they are not monitoring:
+#   zfs-monthly-scrub.{timer,service}  the actual scrub (infra/nodito/32)
+#   pull-backups, check-backups        the backup machinery (playbooks/backups)
+#
+# This play is idempotent and kept permanently rather than run once and deleted:
+# on a host that never had these it does nothing, and it guarantees a rebuilt or
+# restored machine cannot quietly bring them back.
+
+- name: Remove the legacy Uptime Kuma monitoring units
+  hosts: managed
+  become: yes
+
+  vars:
+    legacy_units:
+      - disk-usage-monitor
+      - system-healthcheck
+      - nodito-cpu-temp-monitor
+      - zfs-health-monitor
+    legacy_dirs:
+      - /opt/disk-monitoring
+      - /opt/system-healthcheck
+      - /opt/nodito-monitoring
+      - /opt/zfs-monitoring
+
+  tasks:
+    - name: Find which legacy units exist here
+      ansible.builtin.stat:
+        path: "/etc/systemd/system/{{ item.0 }}.{{ item.1 }}"
+      register: legacy_unit_files
+      loop: "{{ legacy_units | product(['timer', 'service']) | list }}"
+
+    # Stop and disable BEFORE deleting the unit file: systemd cannot disable a
+    # unit whose file has already gone, which would leave a dangling symlink in
+    # multi-user.target.wants and a warning on every daemon-reload.
+    - name: Stop and disable the legacy units
+      ansible.builtin.systemd:
+        name: "{{ item.item.0 }}.{{ item.item.1 }}"
+        state: stopped
+        enabled: no
+      loop: "{{ legacy_unit_files.results }}"
+      loop_control:
+        label: "{{ item.item.0 }}.{{ item.item.1 }}"
+      when: item.stat.exists
+      failed_when: false
+
+    - name: Remove the legacy unit files
+      ansible.builtin.file:
+        path: "/etc/systemd/system/{{ item.item.0 }}.{{ item.item.1 }}"
+        state: absent
+      loop: "{{ legacy_unit_files.results }}"
+      loop_control:
+        label: "{{ item.item.0 }}.{{ item.item.1 }}"
+      when: item.stat.exists
+
+    - name: Reload systemd
+      ansible.builtin.systemd:
+        daemon_reload: yes
+
+    - name: Remove the legacy monitoring scripts and their logs
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop: "{{ legacy_dirs }}"
+
+    # An orphan predating all of this: mode 0644, not executable, referenced by
+    # no unit and no cron entry, pushing to a Kuma monitor. Superseded by
+    # ups-status-healthcheck.
+    - name: Remove the orphaned hand-written UPS heartbeat
+      ansible.builtin.file:
+        path: /usr/local/bin/ups-heartbeat.sh
+        state: absent
+
+    - name: Confirm nothing still pushes to Uptime Kuma
+      ansible.builtin.shell: >-
+        grep -rl "uptime.contrapeso.xyz" /etc/systemd/system /usr/local/bin /opt 2>/dev/null || true
+      register: kuma_refs
+      changed_when: false
+
+    - name: Report any remaining references
+      ansible.builtin.debug:
+        msg: >-
+          {{ 'clean - nothing references Uptime Kuma'
+             if kuma_refs.stdout | trim | length == 0
+             else 'STILL REFERENCING KUMA: ' ~ kuma_refs.stdout_lines | join(', ') }}

ansible/infra/410_disk_usage_alerts.yml

@@ -1,343 +0,0 @@
-# ═════════════════════════════════════════════════════════════════════════════
-# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
-#
-# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
-# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
-# Kuma configuration" assert stops it before anything is installed or changed.
-#
-# It is kept because the CHECK LOGIC is the durable part — what gets measured,
-# the thresholds, and the systemd timer plumbing. When something replaces Uptime
-# Kuma, only the push transport needs rewriting; the rest still applies.
-#
-# What was being monitored: archive/uptime_kuma/MONITORS.md
-# ═════════════════════════════════════════════════════════════════════════════
-- name: Deploy Disk Usage Monitoring
-  hosts: managed
-  become: yes
-  vars_files:
-    - ../infra_vars.yml
-    - ../services_config.yml
-    - ../infra_secrets.yml
-
-  vars:
-    disk_usage_threshold_percent: 80
-    disk_check_interval_minutes: 15
-    monitored_mount_point: "/"
-    monitoring_script_dir: /opt/disk-monitoring
-    monitoring_script_path: "{{ monitoring_script_dir }}/disk_usage_monitor.sh"
-    log_file: "{{ monitoring_script_dir }}/disk_usage_monitor.log"
-    systemd_service_name: disk-usage-monitor
-    # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml)
-    uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
-    ntfy_topic: "{{ service_settings.ntfy.topic }}"
-
-  tasks:
-    - name: Validate Uptime Kuma configuration
-      assert:
-        that:
-          - uptime_kuma_api_url is defined
-          - uptime_kuma_api_url != ""
-          - uptime_kuma_username is defined
-          - uptime_kuma_username != ""
-          - uptime_kuma_password is defined
-          - uptime_kuma_password != ""
-        fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
-
-    - name: Get hostname for monitor identification
-      command: hostname
-      register: host_name
-      changed_when: false
-
-    - name: Set monitor name and group based on hostname and mount point
-      set_fact:
-        monitor_name: "disk-usage-{{ host_name.stdout }}-{{ monitored_mount_point | replace('/', 'root') }}"
-        monitor_friendly_name: "Disk Usage: {{ host_name.stdout }} ({{ monitored_mount_point }})"
-        uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
-
-    - name: Create Uptime Kuma monitor setup script
-      copy:
-        dest: /tmp/setup_uptime_kuma_monitor.py
-        content: |
-          #!/usr/bin/env python3
-          import sys
-          import json
-          from uptime_kuma_api import UptimeKumaApi
-          
-          def main():
-              api_url = sys.argv[1]
-              username = sys.argv[2]
-              password = sys.argv[3]
-              group_name = sys.argv[4]
-              monitor_name = sys.argv[5]
-              monitor_description = sys.argv[6]
-              interval = int(sys.argv[7])
-              ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts"
-              
-              api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0)
-              api.login(username, password)
-              
-              # Get all monitors
-              monitors = api.get_monitors()
-              
-              # Get all notifications and find ntfy notification
-              notifications = api.get_notifications()
-              ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
-              notification_id_list = {}
-              if ntfy_notification:
-                  notification_id_list[ntfy_notification['id']] = True
-              
-              # Find or create group
-              group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              if not group:
-                  group_result = api.add_monitor(type='group', name=group_name)
-                  # Refresh to get the full group object with id
-                  monitors = api.get_monitors()
-                  group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              
-              # Find or create/update push monitor
-              existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              
-              monitor_data = {
-                  'type': 'push',
-                  'name': monitor_name,
-                  'parent': group['id'],
-                  'interval': interval,
-                  'upsideDown': True,
-                  'description': monitor_description,
-                  'notificationIDList': notification_id_list
-              }
-              
-              if existing_monitor:
-                  monitor = api.edit_monitor(existing_monitor['id'], **monitor_data)
-                  # Refresh to get the full monitor object with pushToken
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              else:
-                  monitor_result = api.add_monitor(**monitor_data)
-                  # Refresh to get the full monitor object with pushToken
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              
-              # Output result as JSON
-              result = {
-                  'monitor_id': monitor['id'],
-                  'push_token': monitor['pushToken'],
-                  'group_name': group_name,
-                  'group_id': group['id'],
-                  'monitor_name': monitor_name
-              }
-              print(json.dumps(result))
-              
-              api.disconnect()
-          
-          if __name__ == '__main__':
-              main()
-        mode: '0755'
-      delegate_to: localhost
-      become: no
-
-    - name: Run Uptime Kuma monitor setup script
-      command: >
-        {{ ansible_playbook_python }}
-        /tmp/setup_uptime_kuma_monitor.py
-        "{{ uptime_kuma_api_url }}"
-        "{{ uptime_kuma_username }}"
-        "{{ uptime_kuma_password }}"
-        "{{ uptime_kuma_monitor_group }}"
-        "{{ monitor_name }}"
-        "{{ monitor_friendly_name }} - Alerts when usage exceeds {{ disk_usage_threshold_percent }}%"
-        "{{ (disk_check_interval_minutes * 60) + 60 }}"
-        "{{ ntfy_topic }}"
-      register: monitor_setup_result
-      delegate_to: localhost
-      become: no
-      changed_when: false
-
-    - name: Parse monitor setup result
-      set_fact:
-        monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
-
-    - name: Set push URL and monitor ID as facts
-      set_fact:
-        uptime_kuma_disk_usage_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
-        uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
-
-    - name: Install required packages for disk monitoring
-      package:
-        name:
-          - curl
-        state: present
-
-    - name: Create monitoring script directory
-      file:
-        path: "{{ monitoring_script_dir }}"
-        state: directory
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create disk usage monitoring script
-      copy:
-        dest: "{{ monitoring_script_path }}"
-        content: |
-          #!/bin/bash
-          
-          # Disk Usage Monitoring Script
-          # Monitors disk usage and sends alerts to Uptime Kuma
-          # Mode: "No news is good news" - only sends alerts when disk usage is HIGH
-          
-          LOG_FILE="{{ log_file }}"
-          USAGE_THRESHOLD="{{ disk_usage_threshold_percent }}"
-          UPTIME_KUMA_URL="{{ uptime_kuma_disk_usage_push_url }}"
-          MOUNT_POINT="{{ monitored_mount_point }}"
-          
-          # Function to log messages
-          log_message() {
-              echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
-          }
-          
-          # Function to get disk usage percentage
-          get_disk_usage() {
-              local mount_point="$1"
-              local usage=""
-              
-              # Get disk usage percentage (without % sign)
-              usage=$(df -h "$mount_point" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
-              
-              if [ -z "$usage" ]; then
-                  log_message "ERROR: Could not read disk usage for $mount_point"
-                  return 1
-              fi
-              
-              echo "$usage"
-          }
-          
-          # Function to get disk usage details
-          get_disk_details() {
-              local mount_point="$1"
-              df -h "$mount_point" 2>/dev/null | awk 'NR==2 {print "Used: "$3" / Total: "$2" ("$5" full)"}'
-          }
-          
-          # Function to send alert to Uptime Kuma when disk usage exceeds threshold
-          # With upside-down mode enabled, sending status=up will trigger an alert
-          send_uptime_kuma_alert() {
-              local usage="$1"
-              local details="$2"
-              local message="DISK FULL WARNING: ${MOUNT_POINT} is ${usage}% full (Threshold: ${USAGE_THRESHOLD}%) - ${details}"
-              
-              log_message "ALERT: $message"
-              
-              # Send push notification to Uptime Kuma with status=up
-              # In upside-down mode, status=up is treated as down/alert
-              response=$(curl -s -w "\n%{http_code}" -G \
-                  --data-urlencode "status=up" \
-                  --data-urlencode "msg=$message" \
-                  "$UPTIME_KUMA_URL" 2>&1)
-              http_code=$(echo "$response" | tail -n1)
-              
-              if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
-                  log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)"
-              else
-                  log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)"
-              fi
-          }
-          
-          # Main monitoring logic
-          main() {
-              log_message "Starting disk usage check for $MOUNT_POINT"
-              
-              # Get current disk usage
-              current_usage=$(get_disk_usage "$MOUNT_POINT")
-              
-              if [ $? -ne 0 ] || [ -z "$current_usage" ]; then
-                  log_message "ERROR: Could not read disk usage"
-                  exit 1
-              fi
-              
-              # Get disk details
-              disk_details=$(get_disk_details "$MOUNT_POINT")
-              
-              log_message "Current disk usage: ${current_usage}% - $disk_details"
-              
-              # Check if usage exceeds threshold
-              if [ "$current_usage" -gt "$USAGE_THRESHOLD" ]; then
-                  log_message "WARNING: Disk usage ${current_usage}% exceeds threshold ${USAGE_THRESHOLD}%"
-                  send_uptime_kuma_alert "$current_usage" "$disk_details"
-              else
-                  log_message "Disk usage is within normal range - no alert needed (no news is good news)"
-              fi
-          }
-          
-          # Run main function
-          main
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create systemd service for disk usage monitoring
-      copy:
-        dest: "/etc/systemd/system/{{ systemd_service_name }}.service"
-        content: |
-          [Unit]
-          Description=Disk Usage Monitor
-          After=network.target
-          
-          [Service]
-          Type=oneshot
-          ExecStart={{ monitoring_script_path }}
-          User=root
-          StandardOutput=journal
-          StandardError=journal
-          
-          [Install]
-          WantedBy=multi-user.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Create systemd timer for disk usage monitoring
-      copy:
-        dest: "/etc/systemd/system/{{ systemd_service_name }}.timer"
-        content: |
-          [Unit]
-          Description=Run Disk Usage Monitor every {{ disk_check_interval_minutes }} minute(s)
-          Requires={{ systemd_service_name }}.service
-          
-          [Timer]
-          OnBootSec={{ disk_check_interval_minutes }}min
-          OnUnitActiveSec={{ disk_check_interval_minutes }}min
-          Persistent=true
-          
-          [Install]
-          WantedBy=timers.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Reload systemd daemon
-      systemd:
-        daemon_reload: yes
-
-    - name: Enable and start disk usage monitoring timer
-      systemd:
-        name: "{{ systemd_service_name }}.timer"
-        enabled: yes
-        state: started
-
-    - name: Test disk usage monitoring script
-      command: "{{ monitoring_script_path }}"
-      register: script_test
-      changed_when: false
-
-    - name: Verify script execution
-      assert:
-        that:
-          - script_test.rc == 0
-        fail_msg: "Disk usage monitoring script failed to execute properly"
-
-    - name: Clean up temporary Uptime Kuma setup script
-      file:
-        path: /tmp/setup_uptime_kuma_monitor.py
-        state: absent
-      delegate_to: localhost
-      become: no

ansible/infra/420_system_healthcheck.yml

@@ -1,325 +0,0 @@
-# ═════════════════════════════════════════════════════════════════════════════
-# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
-#
-# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
-# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
-# Kuma configuration" assert stops it before anything is installed or changed.
-#
-# It is kept because the CHECK LOGIC is the durable part — what gets measured,
-# the thresholds, and the systemd timer plumbing. When something replaces Uptime
-# Kuma, only the push transport needs rewriting; the rest still applies.
-#
-# What was being monitored: archive/uptime_kuma/MONITORS.md
-# ═════════════════════════════════════════════════════════════════════════════
-- name: Deploy System Healthcheck Monitoring
-  hosts: managed
-  become: yes
-  vars_files:
-    - ../infra_vars.yml
-    - ../services_config.yml
-    - ../infra_secrets.yml
-
-  vars:
-    healthcheck_interval_seconds: 60  # Send healthcheck every 60 seconds (1 minute)
-    healthcheck_timeout_seconds: 90   # Uptime Kuma should alert if no ping received within 90s
-    healthcheck_retries: 1            # Number of retries before alerting
-    monitoring_script_dir: /opt/system-healthcheck
-    monitoring_script_path: "{{ monitoring_script_dir }}/system_healthcheck.sh"
-    log_file: "{{ monitoring_script_dir }}/system_healthcheck.log"
-    systemd_service_name: system-healthcheck
-    # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml)
-    uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
-    ntfy_topic: "{{ service_settings.ntfy.topic }}"
-
-  tasks:
-    - name: Validate Uptime Kuma configuration
-      assert:
-        that:
-          - uptime_kuma_api_url is defined
-          - uptime_kuma_api_url != ""
-          - uptime_kuma_username is defined
-          - uptime_kuma_username != ""
-          - uptime_kuma_password is defined
-          - uptime_kuma_password != ""
-        fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
-
-    - name: Get hostname for monitor identification
-      command: hostname
-      register: host_name
-      changed_when: false
-
-    - name: Set monitor name and group based on hostname
-      set_fact:
-        monitor_name: "system-healthcheck-{{ host_name.stdout }}"
-        monitor_friendly_name: "System Healthcheck: {{ host_name.stdout }}"
-        uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
-
-    - name: Create Uptime Kuma monitor setup script
-      copy:
-        dest: /tmp/setup_uptime_kuma_healthcheck_monitor.py
-        content: |
-          #!/usr/bin/env python3
-          import sys
-          import json
-          from uptime_kuma_api import UptimeKumaApi
-          
-          def main():
-              api_url = sys.argv[1]
-              username = sys.argv[2]
-              password = sys.argv[3]
-              group_name = sys.argv[4]
-              monitor_name = sys.argv[5]
-              monitor_description = sys.argv[6]
-              interval = int(sys.argv[7])
-              retries = int(sys.argv[8])
-              ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts"
-              
-              api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0)
-              api.login(username, password)
-              
-              # Get all monitors
-              monitors = api.get_monitors()
-              
-              # Get all notifications and find ntfy notification
-              notifications = api.get_notifications()
-              ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
-              notification_id_list = {}
-              if ntfy_notification:
-                  notification_id_list[ntfy_notification['id']] = True
-              
-              # Find or create group
-              group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              if not group:
-                  group_result = api.add_monitor(type='group', name=group_name)
-                  # Refresh to get the full group object with id
-                  monitors = api.get_monitors()
-                  group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              
-              # Find or create/update push monitor
-              existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              
-              monitor_data = {
-                  'type': 'push',
-                  'name': monitor_name,
-                  'parent': group['id'],
-                  'interval': interval,
-                  'upsideDown': False,  # Normal mode: receiving pings = healthy
-                  'maxretries': retries,
-                  'description': monitor_description,
-                  'notificationIDList': notification_id_list
-              }
-              
-              if existing_monitor:
-                  monitor = api.edit_monitor(existing_monitor['id'], **monitor_data)
-                  # Refresh to get the full monitor object with pushToken
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              else:
-                  monitor_result = api.add_monitor(**monitor_data)
-                  # Refresh to get the full monitor object with pushToken
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              
-              # Output result as JSON
-              result = {
-                  'monitor_id': monitor['id'],
-                  'push_token': monitor['pushToken'],
-                  'group_name': group_name,
-                  'group_id': group['id'],
-                  'monitor_name': monitor_name
-              }
-              print(json.dumps(result))
-              
-              api.disconnect()
-          
-          if __name__ == '__main__':
-              main()
-        mode: '0755'
-      delegate_to: localhost
-      become: no
-
-    - name: Run Uptime Kuma monitor setup script
-      command: >
-        {{ ansible_playbook_python }}
-        /tmp/setup_uptime_kuma_healthcheck_monitor.py
-        "{{ uptime_kuma_api_url }}"
-        "{{ uptime_kuma_username }}"
-        "{{ uptime_kuma_password }}"
-        "{{ uptime_kuma_monitor_group }}"
-        "{{ monitor_name }}"
-        "{{ monitor_friendly_name }} - Regular healthcheck ping every {{ healthcheck_interval_seconds }}s"
-        "{{ healthcheck_timeout_seconds }}"
-        "{{ healthcheck_retries }}"
-        "{{ ntfy_topic }}"
-      register: monitor_setup_result
-      delegate_to: localhost
-      become: no
-      changed_when: false
-
-    - name: Parse monitor setup result
-      set_fact:
-        monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
-
-    - name: Set push URL and monitor ID as facts
-      set_fact:
-        uptime_kuma_healthcheck_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
-        uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
-
-    - name: Install required packages for healthcheck monitoring
-      package:
-        name:
-          - curl
-        state: present
-
-    - name: Create monitoring script directory
-      file:
-        path: "{{ monitoring_script_dir }}"
-        state: directory
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create system healthcheck script
-      copy:
-        dest: "{{ monitoring_script_path }}"
-        content: |
-          #!/bin/bash
-          
-          # System Healthcheck Script
-          # Sends regular heartbeat pings to Uptime Kuma
-          # This ensures the system is running and able to communicate
-          
-          LOG_FILE="{{ log_file }}"
-          UPTIME_KUMA_URL="{{ uptime_kuma_healthcheck_push_url }}"
-          HOSTNAME=$(hostname)
-          
-          # Function to log messages
-          log_message() {
-              echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
-          }
-          
-          # Function to send healthcheck ping to Uptime Kuma
-          send_healthcheck() {
-              local uptime_seconds=$(awk '{print int($1)}' /proc/uptime)
-              local uptime_days=$((uptime_seconds / 86400))
-              local uptime_hours=$(((uptime_seconds % 86400) / 3600))
-              local uptime_minutes=$(((uptime_seconds % 3600) / 60))
-              
-              local message="System healthy - Uptime: ${uptime_days}d ${uptime_hours}h ${uptime_minutes}m"
-              
-              log_message "Sending healthcheck ping: $message"
-              
-              # Send push notification to Uptime Kuma with status=up
-              encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g')
-              response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
-              http_code=$(echo "$response" | tail -n1)
-              
-              if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
-                  log_message "Healthcheck ping sent successfully (HTTP $http_code)"
-              else
-                  log_message "ERROR: Failed to send healthcheck ping (HTTP $http_code)"
-                  return 1
-              fi
-          }
-          
-          # Main healthcheck logic
-          main() {
-              log_message "Starting system healthcheck for $HOSTNAME"
-              
-              # Send healthcheck ping
-              if send_healthcheck; then
-                  log_message "Healthcheck completed successfully"
-              else
-                  log_message "ERROR: Healthcheck failed"
-                  exit 1
-              fi
-          }
-          
-          # Run main function
-          main
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create systemd service for system healthcheck
-      copy:
-        dest: "/etc/systemd/system/{{ systemd_service_name }}.service"
-        content: |
-          [Unit]
-          Description=System Healthcheck Monitor
-          After=network.target
-          
-          [Service]
-          Type=oneshot
-          ExecStart={{ monitoring_script_path }}
-          User=root
-          StandardOutput=journal
-          StandardError=journal
-          
-          [Install]
-          WantedBy=multi-user.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Create systemd timer for system healthcheck
-      copy:
-        dest: "/etc/systemd/system/{{ systemd_service_name }}.timer"
-        content: |
-          [Unit]
-          Description=Run System Healthcheck every minute
-          Requires={{ systemd_service_name }}.service
-          
-          [Timer]
-          OnBootSec=30sec
-          OnUnitActiveSec={{ healthcheck_interval_seconds }}sec
-          Persistent=true
-          
-          [Install]
-          WantedBy=timers.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Reload systemd daemon
-      systemd:
-        daemon_reload: yes
-
-    - name: Enable and start system healthcheck timer
-      systemd:
-        name: "{{ systemd_service_name }}.timer"
-        enabled: yes
-        state: started
-
-    - name: Test system healthcheck script
-      command: "{{ monitoring_script_path }}"
-      register: script_test
-      changed_when: false
-
-    - name: Verify script execution
-      assert:
-        that:
-          - script_test.rc == 0
-        fail_msg: "System healthcheck script failed to execute properly"
-
-    - name: Display monitor information
-      debug:
-        msg: |
-          ✓ System healthcheck monitoring deployed successfully!
-          
-          Monitor Name: {{ monitor_friendly_name }}
-          Monitor Group: {{ uptime_kuma_monitor_group }}
-          Healthcheck Interval: Every {{ healthcheck_interval_seconds }} seconds (1 minute)
-          Timeout: {{ healthcheck_timeout_seconds }} seconds (90s)
-          Retries: {{ healthcheck_retries }}
-          
-          The system will send a heartbeat ping every minute.
-          Uptime Kuma will alert if no ping is received within 90 seconds (with 1 retry).
-
-    - name: Clean up temporary Uptime Kuma setup script
-      file:
-        path: /tmp/setup_uptime_kuma_healthcheck_monitor.py
-        state: absent
-      delegate_to: localhost
-      become: no
-

ansible/infra/430_cpu_temp_alerts.yml

@@ -1,329 +0,0 @@
-# ═════════════════════════════════════════════════════════════════════════════
-# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
-#
-# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
-# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
-# Kuma configuration" assert stops it before anything is installed or changed.
-#
-# It is kept because the CHECK LOGIC is the durable part — what gets measured,
-# the thresholds, and the systemd timer plumbing. When something replaces Uptime
-# Kuma, only the push transport needs rewriting; the rest still applies.
-#
-# What was being monitored: archive/uptime_kuma/MONITORS.md
-# ═════════════════════════════════════════════════════════════════════════════
-- name: Deploy CPU Temperature Monitoring
-  hosts: hypervisor
-  become: yes
-  vars_files:
-    - ../infra_vars.yml
-    - ../services_config.yml
-    - ../infra_secrets.yml
-
-  vars:
-    temp_threshold_celsius: 80
-    temp_check_interval_minutes: 1
-    monitoring_script_dir: /opt/nodito-monitoring
-    monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh"
-    log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log"
-    systemd_service_name: nodito-cpu-temp-monitor
-    uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
-    ntfy_topic: "{{ service_settings.ntfy.topic }}"
-
-  tasks:
-    - name: Validate Uptime Kuma configuration
-      assert:
-        that:
-          - uptime_kuma_api_url is defined
-          - uptime_kuma_api_url != ""
-          - uptime_kuma_username is defined
-          - uptime_kuma_username != ""
-          - uptime_kuma_password is defined
-          - uptime_kuma_password != ""
-        fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
-
-    - name: Get hostname for monitor identification
-      command: hostname
-      register: host_name
-      changed_when: false
-
-    - name: Set monitor name and group based on hostname
-      set_fact:
-        monitor_name: "cpu-temp-{{ host_name.stdout }}"
-        monitor_friendly_name: "CPU Temperature: {{ host_name.stdout }}"
-        uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
-
-    - name: Create Uptime Kuma CPU temperature monitor setup script
-      copy:
-        dest: /tmp/setup_uptime_kuma_cpu_temp_monitor.py
-        content: |
-          #!/usr/bin/env python3
-          import sys
-          import json
-          from uptime_kuma_api import UptimeKumaApi
-          
-          def main():
-              api_url = sys.argv[1]
-              username = sys.argv[2]
-              password = sys.argv[3]
-              group_name = sys.argv[4]
-              monitor_name = sys.argv[5]
-              monitor_description = sys.argv[6]
-              interval = int(sys.argv[7])
-              ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts"
-          
-              api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0)
-              api.login(username, password)
-          
-              monitors = api.get_monitors()
-              notifications = api.get_notifications()
-          
-              ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
-              notification_id_list = {}
-              if ntfy_notification:
-                  notification_id_list[ntfy_notification['id']] = True
-          
-              group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              if not group:
-                  api.add_monitor(type='group', name=group_name)
-                  monitors = api.get_monitors()
-                  group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-          
-              existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-          
-              monitor_data = {
-                  'type': 'push',
-                  'name': monitor_name,
-                  'parent': group['id'],
-                  'interval': interval,
-                  'upsideDown': True,
-                  'description': monitor_description,
-                  'notificationIDList': notification_id_list
-              }
-          
-              if existing_monitor:
-                  api.edit_monitor(existing_monitor['id'], **monitor_data)
-              else:
-                  api.add_monitor(**monitor_data)
-          
-              monitors = api.get_monitors()
-              monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-          
-              result = {
-                  'monitor_id': monitor['id'],
-                  'push_token': monitor['pushToken'],
-                  'group_name': group_name,
-                  'group_id': group['id'],
-                  'monitor_name': monitor_name
-              }
-              print(json.dumps(result))
-          
-              api.disconnect()
-          
-          if __name__ == '__main__':
-              main()
-        mode: '0755'
-      delegate_to: localhost
-      become: no
-
-    - name: Run Uptime Kuma monitor setup script
-      command: >
-        {{ ansible_playbook_python }}
-        /tmp/setup_uptime_kuma_cpu_temp_monitor.py
-        "{{ uptime_kuma_api_url }}"
-        "{{ uptime_kuma_username }}"
-        "{{ uptime_kuma_password }}"
-        "{{ uptime_kuma_monitor_group }}"
-        "{{ monitor_name }}"
-        "{{ monitor_friendly_name }} - Alerts when temperature exceeds {{ temp_threshold_celsius }}°C"
-        "{{ (temp_check_interval_minutes * 60) + 60 }}"
-        "{{ ntfy_topic }}"
-      register: monitor_setup_result
-      delegate_to: localhost
-      become: no
-      changed_when: false
-
-    - name: Parse monitor setup result
-      set_fact:
-        monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
-
-    - name: Set push URL and monitor ID as facts
-      set_fact:
-        uptime_kuma_cpu_temp_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
-        uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
-
-    - name: Install required packages for temperature monitoring
-      package:
-        name:
-          - lm-sensors
-          - curl
-          - jq
-          - bc
-        state: present
-
-    - name: Create monitoring script directory
-      file:
-        path: "{{ monitoring_script_dir }}"
-        state: directory
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create CPU temperature monitoring script
-      copy:
-        dest: "{{ monitoring_script_path }}"
-        content: |
-          #!/bin/bash
-          
-          # CPU Temperature Monitoring Script
-          # Monitors CPU temperature and sends alerts to Uptime Kuma
-          
-          LOG_FILE="{{ log_file }}"
-          TEMP_THRESHOLD="{{ temp_threshold_celsius }}"
-          UPTIME_KUMA_URL="{{ uptime_kuma_cpu_temp_push_url }}"
-          
-          log_message() {
-              echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
-          }
-          
-          get_cpu_temp() {
-              local temp=""
-          
-              if command -v sensors >/dev/null 2>&1; then
-                  temp=$(sensors 2>/dev/null | grep -E "Core 0|Package id 0|Tdie|Tctl" | head -1 | grep -oE '[0-9]+\.[0-9]+°C' | grep -oE '[0-9]+\.[0-9]+')
-              fi
-          
-              if [ -z "$temp" ] && [ -f /sys/class/thermal/thermal_zone0/temp ]; then
-                  temp=$(cat /sys/class/thermal/thermal_zone0/temp)
-                  temp=$(echo "scale=1; $temp/1000" | bc -l 2>/dev/null || echo "$temp")
-              fi
-          
-              if [ -z "$temp" ] && command -v acpi >/dev/null 2>&1; then
-                  temp=$(acpi -t 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1)
-              fi
-          
-              echo "$temp"
-          }
-          
-          send_uptime_kuma_alert() {
-              local temp="$1"
-              local message="CPU Temperature Alert: ${temp}°C (Threshold: ${TEMP_THRESHOLD}°C)"
-          
-              log_message "ALERT: $message"
-          
-              encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/°/%C2%B0/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g')
-              response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
-              http_code=$(echo "$response" | tail -n1)
-          
-              if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
-                  log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)"
-              else
-                  log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)"
-              fi
-          }
-          
-          main() {
-              log_message "Starting CPU temperature check"
-          
-              current_temp=$(get_cpu_temp)
-          
-              if [ -z "$current_temp" ]; then
-                  log_message "ERROR: Could not read CPU temperature"
-                  exit 1
-              fi
-          
-              log_message "Current CPU temperature: ${current_temp}°C"
-          
-              if (( $(echo "$current_temp > $TEMP_THRESHOLD" | bc -l) )); then
-                  log_message "WARNING: CPU temperature ${current_temp}°C exceeds threshold ${TEMP_THRESHOLD}°C"
-                  send_uptime_kuma_alert "$current_temp"
-              else
-                  log_message "CPU temperature is within normal range"
-              fi
-          }
-          
-          main
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create systemd service for CPU temperature monitoring
-      copy:
-        dest: "/etc/systemd/system/{{ systemd_service_name }}.service"
-        content: |
-          [Unit]
-          Description=CPU Temperature Monitor
-          After=network.target
-          
-          [Service]
-          Type=oneshot
-          ExecStart={{ monitoring_script_path }}
-          User=root
-          StandardOutput=journal
-          StandardError=journal
-          
-          [Install]
-          WantedBy=multi-user.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Create systemd timer for CPU temperature monitoring
-      copy:
-        dest: "/etc/systemd/system/{{ systemd_service_name }}.timer"
-        content: |
-          [Unit]
-          Description=Run CPU Temperature Monitor every {{ temp_check_interval_minutes }} minute(s)
-          Requires={{ systemd_service_name }}.service
-          
-          [Timer]
-          OnBootSec={{ temp_check_interval_minutes }}min
-          OnUnitActiveSec={{ temp_check_interval_minutes }}min
-          Persistent=true
-          
-          [Install]
-          WantedBy=timers.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Reload systemd daemon
-      systemd:
-        daemon_reload: yes
-
-    - name: Enable and start CPU temperature monitoring timer
-      systemd:
-        name: "{{ systemd_service_name }}.timer"
-        enabled: yes
-        state: started
-
-    - name: Test CPU temperature monitoring script
-      command: "{{ monitoring_script_path }}"
-      register: script_test
-      changed_when: false
-
-    - name: Verify script execution
-      assert:
-        that:
-          - script_test.rc == 0
-        fail_msg: "CPU temperature monitoring script failed to execute properly"
-
-    - name: Display monitoring configuration
-      debug:
-        msg:
-          - "CPU Temperature Monitoring configured successfully"
-          - "Temperature threshold: {{ temp_threshold_celsius }}°C"
-          - "Check interval: {{ temp_check_interval_minutes }} minute(s)"
-          - "Monitor Name: {{ monitor_friendly_name }}"
-          - "Monitor Group: {{ uptime_kuma_monitor_group }}"
-          - "Uptime Kuma Push URL: {{ uptime_kuma_cpu_temp_push_url }}"
-          - "Monitoring script: {{ monitoring_script_path }}"
-          - "Systemd Service: {{ systemd_service_name }}.service"
-          - "Systemd Timer: {{ systemd_service_name }}.timer"
-
-    - name: Clean up temporary Uptime Kuma setup script
-      file:
-        path: /tmp/setup_uptime_kuma_cpu_temp_monitor.py
-        state: absent
-      delegate_to: localhost
-      become: no
-

ansible/infra/900_install_rsync.yml

@@ -1,7 +1,5 @@
 - name: Install rsync
   hosts: managed
-  vars_files:
-    - ../infra_vars.yml
   become: true
 
   tasks:

ansible/infra/920_join_headscale_mesh.yml

@@ -1,14 +1,10 @@
 - name: Join machine to headscale mesh network
   hosts: managed
   become: yes
-  vars_files:
-    - ../infra_vars.yml
-    - ../services_config.yml
   vars:
     headscale_host_name: "spacey"
     headscale_subdomain: "{{ subdomains.headscale }}"
     headscale_domain: "https://{{ headscale_subdomain }}.{{ root_domain }}"
-    headscale_namespace: "{{ service_settings.headscale.namespace }}"
 
   tasks:
     - name: Set facts for headscale server connection

ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml

@@ -1,8 +1,6 @@
 - name: Bootstrap Nodito SSH Key Access
   hosts: hypervisor
   become: true
-  vars_files:
-    - ../infra_vars.yml
 
   tasks:
     - name: Install sudo package

ansible/infra/nodito/31_proxmox_community_repos_playbook.yml

@@ -1,8 +1,6 @@
 - name: Switch Proxmox VE from Enterprise to Community Repositories
   hosts: hypervisor
   become: true
-  vars_files:
-    - ../infra_vars.yml
 
   tasks:
     - name: Check for deb822 sources format

ansible/infra/nodito/32_zfs_pool_setup_playbook.yml

@@ -1,9 +1,6 @@
 - name: Setup ZFS RAID 1 Pool for Proxmox Storage
   hosts: hypervisor
   become: true
-  vars_files:
-    - ../infra_vars.yml
-    - nodito_vars.yml
 
   tasks:
     - name: Verify Proxmox VE is running
@@ -139,17 +136,18 @@
           Config file exists: {{ storage_cfg_file.stat.exists }}
           Storage check result: {{ storage_exists_check.rc }}
           Pool exists: {{ zfs_pool_exists.rc == 0 }}
-          Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }}
           Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }}
 
-    - name: Remove existing storage if it exists
+    # Registration is add-only on purpose. There used to be a "Remove existing
-      command: pvesm remove {{ zfs_pool_name }}
+    # storage if it exists" task here that ran `pvesm remove` whenever the
-      register: pvesm_remove_result
+    # storage WAS present, paired with an add that only ran when it was ABSENT.
-      failed_when: false
+    # The two conditions are mutually exclusive, so a real run against a
-      when: 
+    # correctly-configured hypervisor removed the storage entry backing every VM
-        - zfs_pool_exists.rc == 0
+    # and never put it back. It would also have dropped `mountpoint /var/lib/vz`,
-        - storage_exists_check.rc == 0
+    # which the live entry has and which `pvesm add` below does not set.
-
+    #
+    # If the storage entry ever needs its options changed, edit
+    # /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here.
     - name: Add ZFS pool storage to Proxmox using pvesm
       command: >
         pvesm add zfspool {{ zfs_pool_name }}
@@ -171,511 +169,56 @@
         msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state"
       when: "'ONLINE' not in final_zfs_status.stdout"
 
-# ═════════════════════════════════════════════════════════════════════════════
+# ─────────────────────────────────────────────────────────────────────────────
-# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
+# The monthly scrub.
 #
-# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
+# The ZFS HEALTH CHECK that used to share this play is gone: it is now the
-# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
+# zfs-health check in infra/400_host_monitoring.yml, which carries the same five
-# Kuma configuration" assert stops it before anything is installed or changed.
+# conditions - pool state, device states, resilver in progress, read/write/
+# checksum errors, and errors from the last scan - but reports to Gatus like
+# every other host check instead of owning its own push plumbing.
 #
-# It is kept because the CHECK LOGIC is the durable part — what gets measured,
+# The scrub itself stays here, because it is not monitoring: it is the
-# the thresholds, and the systemd timer plumbing. When something replaces Uptime
+# maintenance that gives the health check something true to report. A pool that
-# Kuma, only the push transport needs rewriting; the rest still applies.
+# is never scrubbed has no idea whether it is healthy.
-#
+# ─────────────────────────────────────────────────────────────────────────────
-# What was being monitored: archive/uptime_kuma/MONITORS.md
+- name: Schedule the monthly ZFS scrub
-# ═════════════════════════════════════════════════════════════════════════════
-- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs
   hosts: hypervisor
   become: true
   vars_files:
     - ../../infra_vars.yml
-    - ../../services_config.yml
-    - ../../infra_secrets.yml
-    - nodito_vars.yml
 
   vars:
-    zfs_check_interval_seconds: 86400  # 24 hours
-    zfs_check_timeout_seconds: 90000   # ~25 hours (interval + buffer)
-    zfs_check_retries: 1
-    zfs_monitoring_script_dir: /opt/zfs-monitoring
-    zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh"
-    zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log"
-    zfs_systemd_health_service_name: zfs-health-monitor
     zfs_systemd_scrub_service_name: zfs-monthly-scrub
-    uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
-    ntfy_topic: "{{ service_settings.ntfy.topic }}"
 
   tasks:
-    - name: Validate Uptime Kuma configuration
-      assert:
-        that:
-          - uptime_kuma_api_url is defined
-          - uptime_kuma_api_url != ""
-          - uptime_kuma_username is defined
-          - uptime_kuma_username != ""
-          - uptime_kuma_password is defined
-          - uptime_kuma_password != ""
-        fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
-
-    - name: Get hostname for monitor identification
-      command: hostname
-      register: host_name
-      changed_when: false
-
-    - name: Set monitor name and group based on hostname
-      set_fact:
-        monitor_name: "zfs-health-{{ host_name.stdout }}"
-        monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}"
-        uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
-
-    - name: Create Uptime Kuma ZFS health monitor setup script
-      copy:
-        dest: /tmp/setup_uptime_kuma_zfs_monitor.py
-        content: |
-          #!/usr/bin/env python3
-          import sys
-          import json
-          from uptime_kuma_api import UptimeKumaApi
-          
-          def main():
-              api_url = sys.argv[1]
-              username = sys.argv[2]
-              password = sys.argv[3]
-              group_name = sys.argv[4]
-              monitor_name = sys.argv[5]
-              monitor_description = sys.argv[6]
-              interval = int(sys.argv[7])
-              retries = int(sys.argv[8])
-              ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts"
-              
-              api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0)
-              api.login(username, password)
-              
-              # Get all monitors
-              monitors = api.get_monitors()
-              
-              # Get all notifications and find ntfy notification
-              notifications = api.get_notifications()
-              ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
-              notification_id_list = {}
-              if ntfy_notification:
-                  notification_id_list[ntfy_notification['id']] = True
-              
-              # Find or create group
-              group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              if not group:
-                  group_result = api.add_monitor(type='group', name=group_name)
-                  # Refresh to get the full group object with id
-                  monitors = api.get_monitors()
-                  group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              
-              # Find or create/update push monitor
-              existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              
-              monitor_data = {
-                  'type': 'push',
-                  'name': monitor_name,
-                  'parent': group['id'],
-                  'interval': interval,
-                  'upsideDown': False,  # Normal heartbeat mode: receiving pings = healthy
-                  'maxretries': retries,
-                  'description': monitor_description,
-                  'notificationIDList': notification_id_list
-              }
-              
-              if existing_monitor:
-                  monitor = api.edit_monitor(existing_monitor['id'], **monitor_data)
-                  # Refresh to get the full monitor object with pushToken
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              else:
-                  monitor_result = api.add_monitor(**monitor_data)
-                  # Refresh to get the full monitor object with pushToken
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              
-              # Output result as JSON
-              result = {
-                  'monitor_id': monitor['id'],
-                  'push_token': monitor['pushToken'],
-                  'group_name': group_name,
-                  'group_id': group['id'],
-                  'monitor_name': monitor_name
-              }
-              print(json.dumps(result))
-              
-              api.disconnect()
-          
-          if __name__ == '__main__':
-              main()
-        mode: '0755'
-      delegate_to: localhost
-      become: no
-
-    - name: Run Uptime Kuma ZFS monitor setup script
-      command: >
-        {{ ansible_playbook_python }}
-        /tmp/setup_uptime_kuma_zfs_monitor.py
-        "{{ uptime_kuma_api_url }}"
-        "{{ uptime_kuma_username }}"
-        "{{ uptime_kuma_password }}"
-        "{{ uptime_kuma_monitor_group }}"
-        "{{ monitor_name }}"
-        "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}"
-        "{{ zfs_check_timeout_seconds }}"
-        "{{ zfs_check_retries }}"
-        "{{ ntfy_topic }}"
-      register: monitor_setup_result
-      delegate_to: localhost
-      become: no
-      changed_when: false
-
-    - name: Parse monitor setup result
-      set_fact:
-        monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
-
-    - name: Set push URL and monitor ID as facts
-      set_fact:
-        uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
-        uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
-
-    - name: Install required packages for ZFS monitoring
-      package:
-        name:
-          - curl
-          - jq
-        state: present
-
-    - name: Create monitoring script directory
-      file:
-        path: "{{ zfs_monitoring_script_dir }}"
-        state: directory
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create ZFS health monitoring script
-      copy:
-        dest: "{{ zfs_monitoring_script_path }}"
-        content: |
-          #!/bin/bash
-          
-          # ZFS Pool Health Monitoring Script
-          # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy
-          # If any issues detected, does NOT send heartbeat (triggers timeout alert)
-          
-          LOG_FILE="{{ zfs_log_file }}"
-          UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}"
-          POOL_NAME="{{ zfs_pool_name }}"
-          HOSTNAME=$(hostname)
-          
-          # Function to log messages
-          log_message() {
-              echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
-          }
-          
-          # Function to check pool health using JSON output
-          check_pool_health() {
-              local pool="$1"
-              local issues_found=0
-              
-              # Get pool status as JSON
-              local pool_json
-              pool_json=$(zpool status -j "$pool" 2>&1)
-              
-              if [ $? -ne 0 ]; then
-                  log_message "ERROR: Failed to get pool status for $pool"
-                  log_message "  -> $pool_json"
-                  return 1
-              fi
-              
-              # Check 1: Pool state must be ONLINE
-              local pool_state
-              pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state')
-              
-              if [ "$pool_state" != "ONLINE" ]; then
-                  log_message "ISSUE: Pool state is $pool_state (expected ONLINE)"
-                  issues_found=1
-              else
-                  log_message "OK: Pool state is ONLINE"
-              fi
-              
-              # Check 2: Check all vdevs and devices for non-ONLINE states
-              local bad_states
-              bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" '
-                  .pools[$pool].vdevs[] | 
-                  .. | objects | 
-                  select(.state? and .state != "ONLINE") | 
-                  "\(.name // "unknown"): \(.state)"
-              ' 2>/dev/null)
-              
-              if [ -n "$bad_states" ]; then
-                  log_message "ISSUE: Found devices not in ONLINE state:"
-                  echo "$bad_states" | while read -r line; do
-                      log_message "  -> $line"
-                  done
-                  issues_found=1
-              else
-                  log_message "OK: All devices are ONLINE"
-              fi
-              
-              # Check 3: Check for resilvering in progress
-              local scan_function scan_state
-              scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"')
-              scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"')
-              
-              if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then
-                  local resilver_progress
-                  resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"')
-                  log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed"
-                  issues_found=1
-              fi
-              
-              # Check 4: Check for read/write/checksum errors on all devices
-              # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison
-              local devices_with_errors
-              devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '
-                  .pools[$pool].vdevs[] | 
-                  .. | objects | 
-                  select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | 
-                  "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)"
-              ' 2>/dev/null)
-              
-              if [ -n "$devices_with_errors" ]; then
-                  log_message "ISSUE: Found devices with I/O errors:"
-                  echo "$devices_with_errors" | while read -r line; do
-                      log_message "  -> $line"
-                  done
-                  issues_found=1
-              else
-                  log_message "OK: No read/write/checksum errors detected"
-              fi
-              
-              # Check 5: Check for scan errors (from last scrub/resilver)
-              local scan_errors
-              scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"')
-              
-              if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then
-                  log_message "ISSUE: Last scan reported $scan_errors errors"
-                  issues_found=1
-              else
-                  log_message "OK: No scan errors"
-              fi
-              
-              return $issues_found
-          }
-          
-          # Function to get last scrub info for status message
-          get_scrub_info() {
-              local pool="$1"
-              local pool_json
-              pool_json=$(zpool status -j "$pool" 2>/dev/null)
-              
-              local scan_func scan_state scan_start
-              scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"')
-              scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"')
-              scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""')
-              
-              if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then
-                  echo "scrub in progress (started $scan_start)"
-              elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then
-                  echo "last scrub: $scan_start"
-              else
-                  echo "no scrub history"
-              fi
-          }
-          
-          # Function to send heartbeat to Uptime Kuma
-          send_heartbeat() {
-              local message="$1"
-              
-              log_message "Sending heartbeat to Uptime Kuma: $message"
-              
-              # URL encode the message
-              local encoded_message
-              encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g')
-              
-              local response http_code
-              response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
-              http_code=$(echo "$response" | tail -n1)
-              
-              if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
-                  log_message "Heartbeat sent successfully (HTTP $http_code)"
-                  return 0
-              else
-                  log_message "ERROR: Failed to send heartbeat (HTTP $http_code)"
-                  return 1
-              fi
-          }
-          
-          # Main health check logic
-          main() {
-              log_message "=========================================="
-              log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME"
-              
-              # Run all health checks
-              if check_pool_health "$POOL_NAME"; then
-                  # All checks passed - send heartbeat
-                  local scrub_info
-                  scrub_info=$(get_scrub_info "$POOL_NAME")
-                  
-                  local message="Pool $POOL_NAME healthy ($scrub_info)"
-                  send_heartbeat "$message"
-                  
-                  log_message "Health check completed: ALL OK"
-                  exit 0
-              else
-                  # Issues found - do NOT send heartbeat (will trigger timeout alert)
-                  log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat"
-                  log_message "Uptime Kuma will alert after timeout due to missing heartbeat"
-                  exit 1
-              fi
-          }
-          
-          # Run main function
-          main
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create systemd service for ZFS health monitoring
-      copy:
-        dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service"
-        content: |
-          [Unit]
-          Description=ZFS Pool Health Monitor
-          After=zfs.target network.target
-          
-          [Service]
-          Type=oneshot
-          ExecStart={{ zfs_monitoring_script_path }}
-          User=root
-          StandardOutput=journal
-          StandardError=journal
-          
-          [Install]
-          WantedBy=multi-user.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Create systemd timer for daily ZFS health monitoring
-      copy:
-        dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer"
-        content: |
-          [Unit]
-          Description=Run ZFS Pool Health Monitor daily
-          Requires={{ zfs_systemd_health_service_name }}.service
-          
-          [Timer]
-          OnBootSec=5min
-          OnUnitActiveSec={{ zfs_check_interval_seconds }}sec
-          Persistent=true
-          
-          [Install]
-          WantedBy=timers.target
-        owner: root
-        group: root
-        mode: '0644'
-
     - name: Create systemd service for ZFS monthly scrub
-      copy:
+      template:
+        src: templates/zfs-monthly-scrub.service.j2
         dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service"
-        content: |
-          [Unit]
-          Description=ZFS Monthly Scrub for {{ zfs_pool_name }}
-          After=zfs.target
-          
-          [Service]
-          Type=oneshot
-          ExecStart=/sbin/zpool scrub {{ zfs_pool_name }}
-          User=root
-          StandardOutput=journal
-          StandardError=journal
-          
-          [Install]
-          WantedBy=multi-user.target
         owner: root
         group: root
         mode: '0644'
 
     - name: Create systemd timer for monthly ZFS scrub
-      copy:
+      template:
+        src: templates/zfs-monthly-scrub.timer.j2
         dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer"
-        content: |
-          [Unit]
-          Description=Run ZFS Scrub on last day of every month at 4:00 AM
-          Requires={{ zfs_systemd_scrub_service_name }}.service
-          
-          [Timer]
-          OnCalendar=*-*~01 04:00:00
-          Persistent=true
-          
-          [Install]
-          WantedBy=timers.target
         owner: root
         group: root
         mode: '0644'
 
-    - name: Reload systemd daemon
+    - name: Enable and start the monthly scrub timer
-      systemd:
-        daemon_reload: yes
-
-    - name: Enable and start ZFS health monitoring timer
-      systemd:
-        name: "{{ zfs_systemd_health_service_name }}.timer"
-        enabled: yes
-        state: started
-
-    - name: Enable and start ZFS monthly scrub timer
       systemd:
         name: "{{ zfs_systemd_scrub_service_name }}.timer"
         enabled: yes
         state: started
+        daemon_reload: yes
 
-    - name: Test ZFS health monitoring script
+    - name: Report the scrub schedule
-      command: "{{ zfs_monitoring_script_path }}"
-      register: script_test
-      changed_when: false
-
-    - name: Verify script execution
-      assert:
-        that:
-          - script_test.rc == 0
-        fail_msg: "ZFS health monitoring script failed - check pool health"
-
-    - name: Display monitoring configuration
       debug:
-        msg: |
+        msg: >-
-          ✓ ZFS Pool Health Monitoring deployed successfully!
+          Monthly scrub of {{ zfs_pool_name }}:
-          
+          last day of each month at 04:00.
-          Monitor Name: {{ monitor_friendly_name }}
+          Health is reported separately by the zfs-health check
-          Monitor Group: {{ uptime_kuma_monitor_group }}
+          (infra/400_host_monitoring.yml).
-          Pool Name: {{ zfs_pool_name }}
-          
-          Health Check:
-            - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours)
-            - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours)
-            - Script: {{ zfs_monitoring_script_path }}
-            - Log: {{ zfs_log_file }}
-            - Service: {{ zfs_systemd_health_service_name }}.service
-            - Timer: {{ zfs_systemd_health_service_name }}.timer
-          
-          Monthly Scrub:
-            - Schedule: Last day of month at 4:00 AM
-            - Service: {{ zfs_systemd_scrub_service_name }}.service
-            - Timer: {{ zfs_systemd_scrub_service_name }}.timer
-          
-          Conditions monitored:
-            - Pool state (must be ONLINE)
-            - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL)
-            - Resilver status (alerts if resilvering)
-            - Read/Write/Checksum errors
-            - Scrub errors
-
-    - name: Clean up temporary Uptime Kuma setup script
-      file:
-        path: /tmp/setup_uptime_kuma_zfs_monitor.py
-        state: absent
-      delegate_to: localhost
-      become: no

ansible/infra/nodito/33_proxmox_debian_cloud_template.yml

@@ -1,9 +1,6 @@
 - name: Create Proxmox template from Debian cloud image (no VM clone)
   hosts: hypervisor
   become: true
-  vars_files:
-    - ../../infra_vars.yml
-    - nodito_vars.yml
 
   vars:
     # Defaults (override via vars_files or --extra-vars as needed)

ansible/infra/nodito/34_nut_ups_setup_playbook.yml

@@ -1,12 +1,33 @@
 - name: Setup NUT (Network UPS Tools) for CyberPower UPS
   hosts: hypervisor
   become: true
-  vars_files:
-    - ../../infra_vars.yml
-    - nodito_vars.yml
-    - nodito_secrets.yml
 
   tasks:
+    # ------------------------------------------------------------------
+    # Safety catch
+    #
+    # /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by
+    # hand in January 2026 and carry a working password. host_vars/nodito/vault.yml
+    # (formerly infra/nodito/nodito_secrets.yml) still holds the literal string
+    # CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that
+    # working pair with a placeholder and restart NUT - leaving the hypervisor's
+    # UPS unmonitored and unable to trigger a clean shutdown on mains loss.
+    #
+    # Until the real password is put in the vault, stop here.
+    #   ansible-vault edit host_vars/nodito/vault.yml
+    # ------------------------------------------------------------------
+    - name: Refuse to run with a placeholder UPS password
+      assert:
+        that:
+          - ups_password is defined
+          - ups_password | length > 0
+          - ups_password != "CHANGE_ME_TO_SECURE_PASSWORD"
+        fail_msg: >-
+          ups_password is unset or still the placeholder. Applying this play would
+          overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on
+          nodito and restart NUT. Put the real password in the vault first:
+          ansible-vault edit host_vars/nodito/vault.yml
+
     # ------------------------------------------------------------------
     # Installation
     # ------------------------------------------------------------------
@@ -75,90 +96,45 @@
     # Configuration files
     # ------------------------------------------------------------------
     - name: Configure NUT mode (standalone)
-      copy:
+      template:
         dest: /etc/nut/nut.conf
-        content: |
+        src: templates/nut.conf.j2
-          # Managed by Ansible
-          MODE=standalone
         owner: root
         group: nut
         mode: "0640"
       notify: Restart NUT services
 
     - name: Configure UPS device
-      copy:
+      template:
         dest: /etc/nut/ups.conf
-        content: |
+        src: templates/ups.conf.j2
-          # Managed by Ansible
-          [{{ ups_name }}]
-              driver = {{ ups_driver }}
-              port = {{ ups_port }}
-              desc = "{{ ups_desc }}"
-              offdelay = {{ ups_offdelay }}
-              ondelay = {{ ups_ondelay }}
         owner: root
         group: nut
         mode: "0640"
       notify: Restart NUT services
 
     - name: Configure upsd to listen on localhost
-      copy:
+      template:
         dest: /etc/nut/upsd.conf
-        content: |
+        src: templates/upsd.conf.j2
-          # Managed by Ansible
-          LISTEN 127.0.0.1 3493
         owner: root
         group: nut
         mode: "0640"
       notify: Restart NUT services
 
     - name: Configure upsd users
-      copy:
+      template:
         dest: /etc/nut/upsd.users
-        content: |
+        src: templates/upsd.users.j2
-          # Managed by Ansible
-          [{{ ups_user }}]
-              password = {{ ups_password }}
-              upsmon master
         owner: root
         group: nut
         mode: "0640"
       notify: Restart NUT services
 
     - name: Configure upsmon
-      copy:
+      template:
         dest: /etc/nut/upsmon.conf
-        content: |
+        src: templates/upsmon.conf.j2
-          # Managed by Ansible
-          MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master
-
-          MINSUPPLIES 1
-          SHUTDOWNCMD "/sbin/shutdown -h +0"
-          POLLFREQ 5
-          POLLFREQALERT 5
-          HOSTSYNC 15
-          DEADTIME 15
-          POWERDOWNFLAG /etc/killpower
-
-          # Notifications
-          NOTIFYMSG ONLINE    "UPS %s on line power"
-          NOTIFYMSG ONBATT    "UPS %s on battery"
-          NOTIFYMSG LOWBATT   "UPS %s battery is low"
-          NOTIFYMSG FSD       "UPS %s: forced shutdown in progress"
-          NOTIFYMSG COMMOK    "Communications with UPS %s established"
-          NOTIFYMSG COMMBAD   "Communications with UPS %s lost"
-          NOTIFYMSG SHUTDOWN  "Auto logout and shutdown proceeding"
-          NOTIFYMSG REPLBATT  "UPS %s battery needs replacing"
-
-          # Log all events to syslog
-          NOTIFYFLAG ONLINE   SYSLOG
-          NOTIFYFLAG ONBATT   SYSLOG
-          NOTIFYFLAG LOWBATT  SYSLOG
-          NOTIFYFLAG FSD      SYSLOG
-          NOTIFYFLAG COMMOK   SYSLOG
-          NOTIFYFLAG COMMBAD  SYSLOG
-          NOTIFYFLAG SHUTDOWN SYSLOG
-          NOTIFYFLAG REPLBATT SYSLOG
         owner: root
         group: nut
         mode: "0640"
@@ -249,334 +225,11 @@
         - nut-server
         - nut-monitor
 
-
+# The UPS heartbeat play that used to live here is gone. What it deployed -
-# ═════════════════════════════════════════════════════════════════════════════
+# /opt/ups-monitoring plus a ups-heartbeat timer - is now the ups-status check
-# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
+# in infra/400_host_monitoring.yml, which reports to Gatus like every other
+# host check instead of carrying its own push plumbing.
 #
-# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
+# This playbook is now purely NUT setup: the driver, upsd, upsmon and the
-# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
+# shutdown behaviour. Monitoring whether the UPS is on mains is a separate
-# Kuma configuration" assert stops it before anything is installed or changed.
+# concern and belongs with the other host checks.
-#
-# It is kept because the CHECK LOGIC is the durable part — what gets measured,
-# the thresholds, and the systemd timer plumbing. When something replaces Uptime
-# Kuma, only the push transport needs rewriting; the rest still applies.
-#
-# What was being monitored: archive/uptime_kuma/MONITORS.md
-# ═════════════════════════════════════════════════════════════════════════════
-- name: Setup UPS Heartbeat Monitoring with Uptime Kuma
-  hosts: hypervisor
-  become: true
-  vars_files:
-    - ../../infra_vars.yml
-    - ../../services_config.yml
-    - ../../infra_secrets.yml
-    - nodito_vars.yml
-    - nodito_secrets.yml
-
-  vars:
-    ups_heartbeat_interval_seconds: 60
-    ups_heartbeat_timeout_seconds: 120
-    ups_heartbeat_retries: 1
-    ups_monitoring_script_dir: /opt/ups-monitoring
-    ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh"
-    ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log"
-    ups_systemd_service_name: ups-heartbeat
-    uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
-    ntfy_topic: "{{ service_settings.ntfy.topic }}"
-
-  tasks:
-    - name: Validate Uptime Kuma configuration
-      assert:
-        that:
-          - uptime_kuma_api_url is defined
-          - uptime_kuma_api_url != ""
-          - uptime_kuma_username is defined
-          - uptime_kuma_username != ""
-          - uptime_kuma_password is defined
-          - uptime_kuma_password != ""
-        fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
-
-    - name: Get hostname for monitor identification
-      command: hostname
-      register: host_name
-      changed_when: false
-
-    - name: Set monitor name and group based on hostname
-      set_fact:
-        monitor_name: "ups-{{ host_name.stdout }}"
-        monitor_friendly_name: "UPS Status: {{ host_name.stdout }}"
-        uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
-
-    - name: Create Uptime Kuma UPS monitor setup script
-      copy:
-        dest: /tmp/setup_uptime_kuma_ups_monitor.py
-        content: |
-          #!/usr/bin/env python3
-          import sys
-          import json
-          from uptime_kuma_api import UptimeKumaApi
-
-          def main():
-              api_url = sys.argv[1]
-              username = sys.argv[2]
-              password = sys.argv[3]
-              group_name = sys.argv[4]
-              monitor_name = sys.argv[5]
-              monitor_description = sys.argv[6]
-              interval = int(sys.argv[7])
-              retries = int(sys.argv[8])
-              ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts"
-
-              api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0)
-              api.login(username, password)
-
-              monitors = api.get_monitors()
-              notifications = api.get_notifications()
-
-              ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
-              notification_id_list = {}
-              if ntfy_notification:
-                  notification_id_list[ntfy_notification['id']] = True
-
-              group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-              if not group:
-                  api.add_monitor(type='group', name=group_name)
-                  monitors = api.get_monitors()
-                  group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
-
-              existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-
-              monitor_data = {
-                  'type': 'push',
-                  'name': monitor_name,
-                  'parent': group['id'],
-                  'interval': interval,
-                  'upsideDown': False,  # Normal heartbeat mode: receiving pings = healthy
-                  'maxretries': retries,
-                  'description': monitor_description,
-                  'notificationIDList': notification_id_list
-              }
-
-              if existing_monitor:
-                  api.edit_monitor(existing_monitor['id'], **monitor_data)
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-              else:
-                  api.add_monitor(**monitor_data)
-                  monitors = api.get_monitors()
-                  monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
-
-              result = {
-                  'monitor_id': monitor['id'],
-                  'push_token': monitor['pushToken'],
-                  'group_name': group_name,
-                  'group_id': group['id'],
-                  'monitor_name': monitor_name
-              }
-              print(json.dumps(result))
-
-              api.disconnect()
-
-          if __name__ == '__main__':
-              main()
-        mode: '0755'
-      delegate_to: localhost
-      become: no
-
-    - name: Run Uptime Kuma UPS monitor setup script
-      command: >
-        {{ ansible_playbook_python }}
-        /tmp/setup_uptime_kuma_ups_monitor.py
-        "{{ uptime_kuma_api_url }}"
-        "{{ uptime_kuma_username }}"
-        "{{ uptime_kuma_password }}"
-        "{{ uptime_kuma_monitor_group }}"
-        "{{ monitor_name }}"
-        "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication"
-        "{{ ups_heartbeat_timeout_seconds }}"
-        "{{ ups_heartbeat_retries }}"
-        "{{ ntfy_topic }}"
-      register: monitor_setup_result
-      delegate_to: localhost
-      become: no
-      changed_when: false
-
-    - name: Parse monitor setup result
-      set_fact:
-        monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
-
-    - name: Set push URL as fact
-      set_fact:
-        uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
-
-    - name: Install required packages for UPS monitoring
-      package:
-        name:
-          - curl
-        state: present
-
-    - name: Create monitoring script directory
-      file:
-        path: "{{ ups_monitoring_script_dir }}"
-        state: directory
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create UPS heartbeat monitoring script
-      copy:
-        dest: "{{ ups_monitoring_script_path }}"
-        content: |
-          #!/bin/bash
-
-          # UPS Heartbeat Monitoring Script
-          # Sends heartbeat to Uptime Kuma only when UPS is on mains power
-          # When on battery or communication lost, no heartbeat is sent (triggers timeout alert)
-
-          LOG_FILE="{{ ups_log_file }}"
-          UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}"
-          UPS_NAME="{{ ups_name }}"
-
-          log_message() {
-              echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
-          }
-
-          send_heartbeat() {
-              local message="$1"
-
-              local encoded_message
-              encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g')
-
-              local response http_code
-              response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
-              http_code=$(echo "$response" | tail -n1)
-
-              if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
-                  log_message "Heartbeat sent: $message (HTTP $http_code)"
-                  return 0
-              else
-                  log_message "ERROR: Failed to send heartbeat (HTTP $http_code)"
-                  return 1
-              fi
-          }
-
-          main() {
-              local status charge runtime load
-
-              status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null)
-
-              if [ -z "$status" ]; then
-                  log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat"
-                  exit 1
-              fi
-
-              charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null)
-              runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null)
-              load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null)
-
-              if [[ "$status" == *"OL"* ]]; then
-                  local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)"
-                  send_heartbeat "$message"
-                  exit 0
-              else
-                  log_message "UPS not on mains power (status=$status) - NOT sending heartbeat"
-                  exit 1
-              fi
-          }
-
-          main
-        owner: root
-        group: root
-        mode: '0755'
-
-    - name: Create systemd service for UPS heartbeat
-      copy:
-        dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service"
-        content: |
-          [Unit]
-          Description=UPS Heartbeat Monitor
-          After=network.target nut-monitor.service
-
-          [Service]
-          Type=oneshot
-          ExecStart={{ ups_monitoring_script_path }}
-          User=root
-          StandardOutput=journal
-          StandardError=journal
-
-          [Install]
-          WantedBy=multi-user.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Create systemd timer for UPS heartbeat
-      copy:
-        dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer"
-        content: |
-          [Unit]
-          Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds
-          Requires={{ ups_systemd_service_name }}.service
-
-          [Timer]
-          OnBootSec=1min
-          OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec
-          Persistent=true
-
-          [Install]
-          WantedBy=timers.target
-        owner: root
-        group: root
-        mode: '0644'
-
-    - name: Reload systemd daemon
-      systemd:
-        daemon_reload: yes
-
-    - name: Enable and start UPS heartbeat timer
-      systemd:
-        name: "{{ ups_systemd_service_name }}.timer"
-        enabled: yes
-        state: started
-
-    - name: Test UPS heartbeat script
-      command: "{{ ups_monitoring_script_path }}"
-      register: script_test
-      changed_when: false
-
-    - name: Verify script execution
-      assert:
-        that:
-          - script_test.rc == 0
-        fail_msg: "UPS heartbeat script failed - check UPS status and communication"
-
-    - name: Display monitoring configuration
-      debug:
-        msg:
-          - "UPS Monitoring configured successfully"
-          - ""
-          - "NUT Configuration:"
-          - "  UPS Name: {{ ups_name }}"
-          - "  UPS Description: {{ ups_desc }}"
-          - "  Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)"
-          - "  On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)"
-          - ""
-          - "Uptime Kuma Monitoring:"
-          - "  Monitor Name: {{ monitor_friendly_name }}"
-          - "  Monitor Group: {{ uptime_kuma_monitor_group }}"
-          - "  Push URL: {{ uptime_kuma_ups_push_url }}"
-          - "  Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s"
-          - "  Timeout: {{ ups_heartbeat_timeout_seconds }}s"
-          - ""
-          - "Scripts and Services:"
-          - "  Script: {{ ups_monitoring_script_path }}"
-          - "  Log: {{ ups_log_file }}"
-          - "  Service: {{ ups_systemd_service_name }}.service"
-          - "  Timer: {{ ups_systemd_service_name }}.timer"
-
-    - name: Clean up temporary Uptime Kuma setup script
-      file:
-        path: /tmp/setup_uptime_kuma_ups_monitor.py
-        state: absent
-      delegate_to: localhost
-      become: no

ansible/infra/nodito/nodito_secrets.yml

@@ -1,12 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-39386134336137333139343861353965353665626161303634323563386563356364373265616163
-3237623438636161323366313363356335613566303338630a663732316663333136653737663935
-34373466316538663962636134303530323837316563336136653636643430363566376363323666
-3531613636393331310a306633613030343138656237663061653438336562323135633732356563
-33396136656564353465636361613161326465303166373131343562363338303966666362643532
-33383561366539393930633766363533313363653733393263376634316362643863376635366638
-36396333633231616662323465323932656565396138313264613832346261616231333265636439
-61666334613662373839613833613663333436373365376534643662656335316536303739616437
-65333635613330353536353162646234323266316338643435653864666364313734386665303830
-34383961633734646134373866623330663038366130306265656466653562643764346162313333
-353664363335303433393330306332666336

ansible/infra/nodito/nodito_vars.yml

@@ -1,28 +0,0 @@
-# Nodito CPU Temperature Monitoring Configuration
-
-# Temperature Monitoring Configuration
-temp_threshold_celsius: 80
-temp_check_interval_minutes: 1
-
-# Script Configuration
-monitoring_script_dir: /opt/nodito-monitoring
-monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh"
-log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log"
-
-# System Configuration
-systemd_service_name: nodito-cpu-temp-monitor
-
-# ZFS Pool Configuration
-zfs_pool_name: "proxmox-tank-1"
-zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z"  # First disk for RAID 1 mirror
-zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P"  # Second disk for RAID 1 mirror
-zfs_pool_mountpoint: "/var/lib/vz"
-
-# UPS Configuration (CyberPower CP900EPFCLCD via USB)
-ups_name: cyberpower
-ups_desc: "CyberPower CP900EPFCLCD"
-ups_driver: usbhid-ups
-ups_port: auto
-ups_user: counterweight
-ups_offdelay: 120  # Seconds after shutdown before UPS cuts outlet power
-ups_ondelay: 30    # Seconds after mains returns before UPS restores outlet power

ansible/infra/nodito/templates/nut.conf.j2

@@ -0,0 +1,2 @@
+# Managed by Ansible
+MODE=standalone

ansible/infra/nodito/templates/ups.conf.j2

@@ -0,0 +1,9 @@
+# Managed by Ansible
+maxretry = 3
+
+[{{ ups_name }}]
+    driver = {{ ups_driver }}
+    port = {{ ups_port }}
+    desc = "{{ ups_desc }}"
+    offdelay = {{ ups_offdelay }}
+    ondelay = {{ ups_ondelay }}

ansible/infra/nodito/templates/upsd.conf.j2

@@ -0,0 +1,2 @@
+# Managed by Ansible
+LISTEN 127.0.0.1 3493

ansible/infra/nodito/templates/upsd.users.j2

@@ -0,0 +1,4 @@
+# Managed by Ansible
+[{{ ups_user }}]
+    password = {{ ups_password }}
+    upsmon master

ansible/infra/nodito/templates/upsmon.conf.j2

@@ -0,0 +1,34 @@
+# Managed by Ansible
+MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master
+
+MINSUPPLIES 1
+SHUTDOWNCMD "/sbin/shutdown -h +0"
+POLLFREQ 5
+POLLFREQALERT 5
+HOSTSYNC 15
+DEADTIME 15
+POWERDOWNFLAG "/etc/killpower"
+OFFDURATION 30
+RBWARNTIME 43200
+NOCOMMWARNTIME 300
+FINALDELAY 5
+
+# Notifications
+NOTIFYMSG ONLINE    "UPS %s on line power"
+NOTIFYMSG ONBATT    "UPS %s on battery"
+NOTIFYMSG LOWBATT   "UPS %s battery is low"
+NOTIFYMSG FSD       "UPS %s: forced shutdown in progress"
+NOTIFYMSG COMMOK    "Communications with UPS %s established"
+NOTIFYMSG COMMBAD   "Communications with UPS %s lost"
+NOTIFYMSG SHUTDOWN  "Auto logout and shutdown proceeding"
+NOTIFYMSG REPLBATT  "UPS %s battery needs replacing"
+
+# Log all events to syslog
+NOTIFYFLAG ONLINE   SYSLOG
+NOTIFYFLAG ONBATT   SYSLOG
+NOTIFYFLAG LOWBATT  SYSLOG
+NOTIFYFLAG FSD      SYSLOG
+NOTIFYFLAG COMMOK   SYSLOG
+NOTIFYFLAG COMMBAD  SYSLOG
+NOTIFYFLAG SHUTDOWN SYSLOG
+NOTIFYFLAG REPLBATT SYSLOG

ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=ZFS Monthly Scrub for {{ zfs_pool_name }}
+After=zfs.target
+
+[Service]
+Type=oneshot
+ExecStart=/sbin/zpool scrub {{ zfs_pool_name }}
+User=root
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run ZFS Scrub on last day of every month at 4:00 AM
+Requires={{ zfs_systemd_scrub_service_name }}.service
+
+[Timer]
+OnCalendar=*-*~01 04:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/infra_secrets.yml

@@ -1,60 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-34616531626537376463326332306434383163393761363536633133363161373631376437653234
-3831656536313531343433376264643261396634633965370a303931336236323766313065636535
-31343338616364386132623232373266663665353563666535383232666262663062323864303932
-3439373734316639650a633732333064633335383833663261326464336538636666323063646331
-66653464636534663935616461636132613162303530663237376661343338396133323431623561
-30303563303465633231306339326561313533333536376433393130376634303932643339656336
-30616130343733626162366561393939393766623138616537393032376665396139333561623830
-33623130336235633034666561383165616439316334323664623661343438663733373833306335
-30396165313731636435636263353133326135346236323030653734353831626238666663656364
-62386164363361373464383730333733626333333736306535336635613634646535383237326332
-37636532663933663739303634333235343137333363316430643263326231613635643633636437
-30336462633233396430633939616661643261666136656363363461343230653738613166346434
-63376363666462346539343063343562623962303130616535306439653134633630353963373337
-64313239353162363862643432666438383330636638333464323731643163643236313535313030
-65656231383738356537326661663163376634613031396436646633376139313237343534653065
-64323532346133313265346334353530393633396339316330366536646565353836396662303866
-65373030393463306664623235626437613965303837313365373632643935656630303936373837
-64663539623764656636613263346638376665626262663430333231633735653563643864343835
-62303637383332653366383066326536316336306539623230353066343739316430356437313365
-33323031613130643832636133636263303565623330306135333762363036633737343933623437
-33633436363062643436376632336235666330656265316333323361316430656631383734343238
-65616366666536363866343361666563643336306532666332643230656264303565353032393932
-36613738326239376161383064646634623639356439663965323237303361343838373737636235
-39656465333063633136643463356564636537326339653033633063366265636138656466646264
-32643961343863656464353439346531633036613138353333353631626665396239326465303632
-39336664333038636435366333623336353963323535316436646136313739373537376537353735
-66333265633232346536313933333836643439656666636236626266323333383935383565616139
-38626133356364613562393664643563346539366364636332613162663337356232333735393431
-30343635393462356431646336636663333735343164616261343836363366616162613563613066
-66333034613234626431356536323832393466363135653231336238356334353638613838626430
-63636630313533363563376161663637353362383431663130646666646433353864373736613662
-62343666633862643839666532656132653939663437373164393536653739376265303130353636
-62616538326330333136626238306133363634326365366637363961333635356133613564373131
-64626139656633663065353862343135393233333231386166346532656534623361623061336434
-37616565626134333863376532393861376132323434656433613731613834633963386635353336
-39653263346637303836663763653264396139383363616461393333333362616332663863363834
-61613832326466646133626337663364333566326633313664613236346135653332373035313034
-62356530313065393737393634313166313039616635363365633031336434396463326462316236
-62353838636533303664663132353838636632353733316639643964666139383539333761663531
-37366530373165653063303032383461326535613336343635626538373635356266396666346133
-61383434323565306438356634646338666232363238393932353365363461376130323363363236
-34643862653937353461383265663933646264373365623465313666643662633334366437646434
-34343332613832366235663563353435323738356339383338333361383561336366376238393237
-38656265663164376666616366623062393530366261623361383365376235643035353838313235
-32396430376139313430316439333539343266393965353030636638656661316137313437663832
-64366661323865383262323036613230316233666566386133363633303931323464663238323861
-62333132346165353830653062353763306232366563323430383933653163373562326536393362
-66353931376262383931613931303034356537363137366235643430323465623162363031373034
-64653563666533363062333831336463376530306639616631303266386136666233653638653732
-31313537393838333730303235363661326333646531653935633935373362303732653565656438
-36626330623063373832333739643666396135653838643164376264393863646433353036643731
-34623636623366353466333237373265323936643134613230326565303663306431373634356334
-37613732393262626135306632623336613364323632316636623037656538313236666562633438
-37306361333130376538366232393637646263656239303431623963626265386634313735373162
-64393733303132383464386330626135336237303563326238633437346164333738646431333730
-62616533363638643261306565373835303832313539656564386132393863373039653938616261
-36343031356361336563316230343433633033353130373933313361326164633765316433616162
-62356338393634653330623933313433343630386264396337373532376335316362333164363963
-363663666365336564663232363462613162

ansible/infra_secrets.yml.example

@@ -1,38 +0,0 @@
-# Uptime Kuma login credentials
-# Used by the disk monitoring playbook to create monitors automatically
-
-
-# ntfy credentials
-# Used for notification channel setup in Uptime Kuma
-
-ntfy_username: "your_ntfy_username"
-ntfy_password: "your_ntfy_password"
-
-# headscale-ui credentials
-# Used for HTTP basic authentication via Caddy
-# Provide either:
-#   - headscale_ui_password: plain text password (will be hashed automatically)
-#   - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate)
-
-headscale_ui_username: "admin"
-headscale_ui_password: "your_secure_password_here"
-# headscale_ui_password_hash: "$2a$14$..."  # Optional: pre-hashed password
-
-bitcoin_rpc_user: "bitcoinrpc"
-bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD"
-
-# Mempool MariaDB credentials
-# Used by: services/mempool/deploy_mempool_playbook.yml
-mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD"
-
-# Forgejo Runner registration token
-# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml
-# See: services/forgejo-runner/SETUP.md for how to obtain this token
-forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE"
-
-# DATUM Gateway secrets
-# Used by: services/datum-gateway/deploy_datum_gateway_playbook.yml
-datum_mining_address: "YOUR_BITCOIN_ADDRESS_FOR_BLOCK_REWARDS"
-datum_gateway_admin_password: "CHANGE_ME_TO_SECURE_PASSWORD"
-datum_dashboard_username: "admin"
-datum_dashboard_password_hash: "$2a$14$..."  # Generate with: caddy hash-password

ansible/infra_vars.yml

@@ -1,9 +0,0 @@
-new_user: counterweight
-ssh_port: 22
-allow_ssh_from: "any"
-root_domain: contrapeso.xyz
-
-# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the
-# playbooks are kept deliberately — the check logic is meant to be rewired to
-# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
-uptime_kuma_enabled: false

ansible/inventory.ini

@@ -1,7 +1,7 @@
 [vps]
 vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
-watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
 spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
+monitoring ansible_host=64.226.70.190 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
 
 [nodito_host]
 nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
@@ -15,11 +15,10 @@ memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counter
 forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
 arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
 small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
-nonkeiwaisi_local ansible_host=nonkeiwaisi-box lan_ip=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
 
 # Local connection to laptop: this assumes you're running ansible commands from your personal laptop
 [lapy]
-localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C
+localhost ansible_connection=local ansible_user=counterweight
 
 [arbret]
 prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
@@ -27,8 +26,12 @@ prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22
 [edge]
 vipy
 
-[monitoring]
+# The group is `observability`, NOT `monitoring` — there is a HOST named
-watchtower
+# `monitoring` on line 5, and a group with the same name makes `hosts: monitoring`
+# ambiguous. Ansible resolved it to the host and warned:
+#   [WARNING]: Found both group and host with same name: monitoring
+[observability]
+monitoring
 
 [vpn_control]
 spacey
@@ -64,7 +67,7 @@ nodito_vms
 # Hosts that run Caddy and therefore have /etc/caddy/sites-enabled.
 [caddy:children]
 edge
-monitoring
+observability
 vpn_control
 
 [backup_store]

ansible/playbooks/backups.yml

@@ -7,6 +7,10 @@
       ansible.builtin.include_role:
         name: backup_store
       vars:
+        # check-backups.sh reports one result per source plus one for the store
+        # itself, so it needs the collection URL and appends each key.
+        backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
+        backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"
         backup_store_sources:
           - name: arbret
             source: "arbret@prd-arbret:/opt/arbret/backups/"
@@ -26,3 +30,78 @@
           - name: forgejo
             source: "backup-pull@prd-vipy:/opt/backups/forgejo/"
             retention_days: 14
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Register the backup checks with Gatus.
+#
+# Two groups on purpose, because they answer different questions and fail for
+# different reasons:
+#
+#   backup-dump   did the SOURCE produce an artefact? Pushed by each dump right
+#                 after it runs, so a broken dump is visible within minutes.
+#   backup-store  did it ARRIVE, is it fresh, non-zero, plausibly sized, and is
+#                 retention pruning? Pushed by check-backups.sh at 05:30.
+#
+# The store alone could catch almost everything, because the artefact filename
+# carries the source's dump timestamp - a source whose timer died still pulls
+# "ok" forever, but the timestamp gives it away. What the source side adds is
+# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it
+# cannot tell you whether the dump broke or the pull did.
+#
+# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed`
+# deliberately excludes, so nothing of ours runs there. It is store-checked only.
+# ─────────────────────────────────────────────────────────────────────────────
+- name: Register the backup checks with Gatus
+  hosts: observability
+  become: yes
+  vars:
+    # Sources we deploy the dump for, and the host each one runs on.
+    dump_sources:
+      - {name: headscale,   host: spacey}
+      - {name: memos,       host: memos_box_local}
+      - {name: vaultwarden, host: vipy}
+      - {name: lnbits,      host: vipy}
+      - {name: forgejo,     host: vipy}
+    store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo]
+
+  tasks:
+    # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter
+    # than a day plus slack. The store-side check catches the same fault within
+    # 6h by reading the artefact's dump timestamp out of the filename, so this is
+    # the slow backstop rather than the primary signal.
+    - name: Build the dump endpoint list
+      ansible.builtin.set_fact:
+        dump_endpoints: "{{ dump_endpoints | default([]) + [{
+          'name': item.name,
+          'group': 'backup-dump',
+          'token': gatus_push_tokens[item.host],
+          'heartbeat': '26h'}] }}"
+      loop: "{{ dump_sources }}"
+
+    - name: Build the store endpoint list
+      ansible.builtin.set_fact:
+        store_endpoints: "{{ store_endpoints | default([]) + [{
+          'name': item,
+          'group': 'backup-store',
+          'token': gatus_push_tokens['small_backups_local'],
+          'heartbeat': '7h'}] }}"
+      loop: "{{ store_sources }}"
+
+    - name: Register the backup endpoints
+      ansible.builtin.include_role:
+        name: gatus_endpoint
+      vars:
+        # Push endpoints: the heartbeat window is the tolerance, so alert on
+        # the first failure rather than waiting for three 7h windows to pass.
+        gatus_endpoint_default_alerts:
+          - type: signal
+            failure-threshold: 1
+            success-threshold: 2
+            send-on-resolved: true
+            minimum-reminder-interval: 12h
+        gatus_endpoint_name: backups
+        gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{
+          'name': 'pull job',
+          'group': 'backup-store',
+          'token': gatus_push_tokens['small_backups_local'],
+          'heartbeat': '7h'}] }}"

ansible/roles/backup_source/defaults/main.yml

@@ -29,3 +29,16 @@ backup_source_retention_days: 7
 
 # Schedule. The box pulls at 04:00, so dumps must land before that.
 backup_source_on_calendar: "*-*-* 02:00:00"
+
+# ── Reporting ────────────────────────────────────────────────────────────────
+# Where to report that this dump ran and produced a plausible artefact.
+# Gatus external endpoint:
+#     POST {url}?success=true|false&error=...
+#     Authorization: Bearer {token}
+# Empty is valid and is not an error: the unit's exit code is still the answer,
+# and the STORE will independently notice a stale dump within ~26h because the
+# artefact filename carries this dump's timestamp. Reporting here only buys
+# earlier detection and tells you it was the DUMP that broke rather than the
+# pull.
+backup_source_push_url: ""
+backup_source_push_token: ""

ansible/roles/backup_source/tasks/main.yml

@@ -16,12 +16,27 @@
 
 # Declared here rather than assumed. Stage 1 installed it by hand; this is what
 # makes a rebuilt host get it too.
-- name: Ensure age is installed
+# Cache refresh is best-effort on purpose. An unrelated third-party repo with a
+# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource
+# key) makes `apt-get update` return warnings, which the apt module treats as a
+# hard failure — and that must not stop backups being configured. Installing the
+# package is NOT best-effort: if age is genuinely unavailable, the next task fails.
+- name: Refresh the apt cache (best effort)
   ansible.builtin.apt:
-    name: age
-    state: present
     update_cache: yes
     cache_valid_time: 3600
+  failed_when: false
+  changed_when: false
+
+- name: Ensure age is installed
+  ansible.builtin.apt:
+    name:
+      - age
+      # curl is needed only when backup_source_push_url is set, but installing it
+      # unconditionally keeps the task idempotent and it is present on every
+      # Debian host here anyway.
+      - curl
+    state: present
 
 # The pull account: unprivileged, no sudo, exists only so small-backups-box can
 # read the dump directory. Trust points one way — the box can read backups, and
@@ -73,14 +88,18 @@
     mode: '0750'
     validate: "bash -n %s"
 
+# The .service carries the push token in an Environment= line, so it is 0600.
+# The .timer holds nothing secret and stays world-readable.
 - name: "Install the {{ backup_source_name }}-backup systemd units"
   ansible.builtin.template:
-    src: "backup.{{ item }}.j2"
+    src: "backup.{{ item.unit }}.j2"
-    dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}"
+    dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}"
     owner: root
     group: root
-    mode: '0644'
+    mode: "{{ item.mode }}"
-  loop: [service, timer]
+  loop:
+    - {unit: service, mode: "0600"}
+    - {unit: timer,   mode: "0644"}
   notify: Reload systemd for backup units
 
 - name: "Enable the {{ backup_source_name }}-backup timer"

ansible/roles/backup_source/templates/backup.service.j2

@@ -9,6 +9,8 @@ After={{ backup_source_stop_service if '.' in backup_source_stop_service else ba
 [Service]
 Type=oneshot
 ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh
+Environment=BACKUP_PUSH_URL={{ backup_source_push_url }}
+Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }}
 StandardOutput=journal
 StandardError=journal
 SyslogIdentifier={{ backup_source_name }}-backup

ansible/roles/backup_source/templates/backup.sh.j2

@@ -41,18 +41,76 @@ chmod 700 "$BACKUP_DIR"
 # here or they accumulate forever.
 rm -f "${BACKUP_DIR}/${NAME}_"*.partial
 
+# --- Reporting -------------------------------------------------------------
+# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed
+# backup even though the script "finished". Both are reported as failures.
+PUSH_URL="${BACKUP_PUSH_URL:-}"
+PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}"
+
+report() {
+    local success="$1" message="$2"
+    [ -n "$PUSH_URL" ] || return 0
+    local encoded
+    encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g')
+    curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \
+        -H "Authorization: Bearer ${PUSH_TOKEN}" \
+        "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true
+}
+
+# Reports on ANY exit path, so a dump that dies halfway still reports rather
+# than going quiet. The size of the FINISHED artefact decides success, not
+# merely reaching the end of the script.
+#
+# This is called FROM the single EXIT trap below - it must never register an
+# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather
+# than adding to it, so a second trap here silently discards the one that
+# restarts the service, and a backup run leaves the service stopped. That is
+# precisely the failure the restart trap exists to prevent.
+report_outcome() {
+    local rc="$1"
+    if [ "$rc" -ne 0 ]; then
+        report "false" "${NAME} dump exited ${rc}"
+    elif [ ! -s "$ARTIFACT" ]; then
+        report "false" "${NAME} produced no artefact at ${ARTIFACT}"
+    else
+        report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)"
+    fi
+}
+
+# --- One EXIT handler, doing both jobs -------------------------------------
+# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous
+# handler rather than adding to it. Registering a second one here would
+# silently discard the service restart and leave the service stopped after
+# every backup - which is the exact bug the restart exists to prevent, and it
+# is invisible until someone notices the service is down.
+on_exit() {
+    local rc=$?
 {% if backup_source_stop_service or backup_source_stop_command %}
-# --- Stop the service, and guarantee it comes back ---
+    log "Restarting ${SERVICE}..."
+    eval "$START_CMD" || true
+{% endif %}
+    report_outcome "$rc"
+}
+trap on_exit EXIT
+
+{% if backup_source_stop_service or backup_source_stop_command %}
+# --- Stop the service; the trap above guarantees it comes back -------------
 # The trap is the point: without it a failed dump leaves the service down until
 # the next timer fires. Every hand-written script this replaced had that bug.
+# It is armed BEFORE the stop, so even a failure during the stop restarts.
 log "Stopping ${SERVICE}..."
 eval "$STOP_CMD"
-trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT
 {% endif %}
 
 # --- Dump straight into age; plaintext never touches the disk ---
 log "Writing ${ARTIFACT}..."
 {{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial"
+{% if backup_source_pull_user %}
+# Match the final ownership immediately, so even a partial left by a later
+# failure is not an unreadable obstacle to the pull.
+chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial"
+chmod 640 "${ARTIFACT}.partial"
+{% endif %}
 mv "${ARTIFACT}.partial" "$ARTIFACT"
 {% if backup_source_pull_user %}
 # Readable by the pull account and nobody else. The contents are age-encrypted

ansible/roles/backup_store/defaults/main.yml

@@ -9,3 +9,21 @@ backup_store_on_calendar: "*-*-* 04:00:00"
 #     source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
 #     retention_days: 90
 backup_store_sources: []
+
+# ── Reporting ────────────────────────────────────────────────────────────────
+# check-backups.sh reports one result PER SOURCE plus one for the store itself,
+# so the base URL is the endpoints collection and the script appends each key.
+# Empty is valid: the script still prints its report and exits 0/1.
+backup_store_check_push_base: ""
+backup_store_check_push_token: ""
+
+# Every six hours, offset past the 04:00 pull so the first run of the day sees a
+# finished pull. The BACKUPS are daily, but this check is not - it reads the
+# source's dump timestamp out of the artefact filename, so running it more often
+# catches "the source stopped dumping" within hours rather than a day, and lets
+# the Gatus heartbeat be 7h instead of 30h.
+backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00"
+
+# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the
+# pull is at 04:00, so 26h tolerates exactly one missed night before alarming.
+backup_store_check_max_age_hours: 26

ansible/roles/backup_store/tasks/main.yml

@@ -33,6 +33,42 @@
     validate: "bash -n %s"
   become: yes
 
+# An assertion that last night actually worked. Generated from the same source
+# list as the puller, so it can never drift out of sync with what is supposed to
+# be arriving. Runs on a timer AND is useful by hand.
+- name: Install the backup check script
+  ansible.builtin.template:
+    src: check-backups.sh.j2
+    dest: /usr/local/bin/check-backups.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+  become: yes
+
+# The .service carries the push token, so it is 0600; the .timer is not secret.
+- name: Install the check-backups systemd units
+  ansible.builtin.template:
+    src: "check-backups.{{ item.unit }}.j2"
+    dest: "/etc/systemd/system/check-backups.{{ item.unit }}"
+    owner: root
+    group: root
+    mode: "{{ item.mode }}"
+  loop:
+    - {unit: service, mode: "0600"}
+    - {unit: timer,   mode: "0644"}
+  become: yes
+
+# restarted, not started: `started` is a no-op on an already-active timer, so a
+# changed schedule would never be picked up.
+- name: Enable the check-backups timer
+  ansible.builtin.systemd:
+    name: check-backups.timer
+    enabled: yes
+    state: restarted
+    daemon_reload: yes
+  become: yes
+
 - name: Install the pull-backups systemd units
   ansible.builtin.template:
     src: "pull-backups.{{ item }}.j2"

ansible/roles/backup_store/templates/check-backups.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Verify the nightly backup pull actually worked
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User={{ ansible_user_id }}
+ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }}
+Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }}
+Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }}
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/backup_store/templates/check-backups.sh.j2

@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+# Assert the nightly backups actually worked.
+#
+# Run as {{ ansible_user_id }} on this host. Needs no sudo.
+#
+# What it CANNOT do: verify contents. The age identity lives only on lapy, so
+# this host cannot decrypt anything it holds — by design. These are freshness,
+# completeness and integrity checks. To verify content, decrypt on lapy:
+#     ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups/<svc>/<artefact>" \
+#       | age -d -i ~/.age/counterweight_age | tar -tzf - | head
+#
+# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL.
+set -uo pipefail
+
+STORE="{{ backup_store_dir }}"
+MAX_AGE_H="${1:-26}"          # an artefact older than this is stale
+NOW=$(date +%s)
+fails=0; warns=0
+
+# Colour only when attached to a terminal: this gets piped into files and, later,
+# probably into a notification.
+if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m'
+else R=''; Y=''; G=''; N=''; fi
+
+# Per-source verdicts, so each source can be reported independently. A single
+# aggregate red light tells you backups are broken; it does not tell you which
+# one, which is the thing you need at 3am.
+declare -A SRC_FAIL SRC_MSG
+CURRENT=""
+
+red()  { printf '  %sFAIL%s  %s\n' "$R" "$N" "$*"; fails=$((fails+1));
+         [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; }
+yell() { printf '  %sWARN%s  %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); }
+ok()   { printf '  %sok%s    %s\n' "$G" "$N" "$*";
+         [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }
+
+# --- Reporting -------------------------------------------------------------
+# Each source gets its own Gatus external endpoint, plus one for the store
+# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means
+# report nowhere, which is valid: the exit code is still the whole answer.
+PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}"
+PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}"
+
+report() {
+    local key="$1" success="$2" message="$3"
+    [ -n "$PUSH_BASE" ] || return 0
+    local encoded
+    encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g')
+    curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \
+        -H "Authorization: Bearer ${PUSH_TOKEN}" \
+        "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true
+}
+
+hours_since() { echo $(( (NOW - $1) / 3600 )); }
+
+# Pull the dump timestamp out of <name>_YYYYmmdd_HHMMSS.<suffix>. This is when
+# the SOURCE produced it, which is the thing that actually matters: a source
+# whose timer died still pulls "ok" forever, because yesterday's artefact is
+# still sitting there. Checking only the pull would miss exactly that.
+dump_epoch() {
+  local base ts
+  base=$(basename "$1")
+  ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1
+  [ -n "$ts" ] || return 1
+  date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null
+}
+
+check_source() {
+  local name="$1" keep="$2" dir="$STORE/$1"
+  printf '\n%s\n' "== $name"
+  CURRENT="$name"
+  SRC_FAIL[$name]=0
+  SRC_MSG[$name]=""
+
+  [ -d "$dir" ] || { red "$name: no directory $dir"; return; }
+
+  local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l)
+  [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; }
+
+  local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l)
+  [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these"
+
+  local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1)
+  local prev;   prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p)
+
+  # 1. Did the SOURCE dump recently?
+  local de; de=$(dump_epoch "$newest")
+  if [ -z "${de:-}" ]; then
+    yell "$name: cannot parse a dump timestamp from $(basename "$newest")"
+  else
+    local dh; dh=$(hours_since "$de")
+    if [ "$dh" -lt 0 ]; then
+      # A future-dated artefact would otherwise stay "fresh" forever and the
+      # staleness check would never fire again — the exact silent failure this
+      # script exists to catch.
+      red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?"
+    elif [ "$dh" -gt "$MAX_AGE_H" ]; then
+      red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run"
+    else
+      ok "$name: dumped ${dh}h ago"
+    fi
+  fi
+
+  # 2. Did the PULL bring it over recently?
+  local ph; ph=$(hours_since "$(stat -c %Y "$newest")")
+  if [ "$ph" -gt "$MAX_AGE_H" ]; then
+    red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)"
+  else
+    ok "$name: pulled ${ph}h ago"
+  fi
+
+  # 3. Is it plausibly a real backup?
+  local sz; sz=$(stat -c %s "$newest")
+  if [ "$sz" -eq 0 ]; then
+    red "$name: newest artefact is ZERO bytes"
+  elif [ -n "$prev" ]; then
+    local psz; psz=$(stat -c %s "$prev")
+    if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then
+      # Not automatically wrong: headscale legitimately shrank 297K -> 20K when
+      # a clean stop checkpointed its write-ahead log into the database.
+      yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect"
+    else
+      ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)"
+    fi
+  else
+    ok "$name: $(numfmt --to=iec "$sz") (first artefact)"
+  fi
+
+  # 4. Is retention pruning? Allow generous slack for multiple dumps per day.
+  if [ "$n" -gt $(( keep * 3 + 10 )) ]; then
+    yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working"
+  fi
+}
+
+echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')"
+echo "Artefacts older than ${MAX_AGE_H}h are treated as stale."
+
+# --- the pull job itself ---
+# Reserved key, reported as backup-store_pull-job. The store's own machinery is
+# a different alarm from any one source being stale, and it is a LEADING
+# indicator where the per-source checks are lagging ones: those only fire once
+# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer,
+# a failed pull job or a filling disk are all visible here immediately, and they
+# name the cause instead of showing six stale sources with no explanation.
+CURRENT="__store"
+SRC_FAIL[__store]=0
+SRC_MSG[__store]=""
+printf '\n%s\n' "== pull-backups.service"
+result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null)
+status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null)
+when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null)
+[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})"
+if [ -n "$when" ]; then
+  wh=$(hours_since "$(date -d "$when" +%s)")
+  [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?"
+fi
+systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \
+  && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \
+  || red "pull-backups.timer is NOT enabled"
+
+# --- each source ---
+{% for src in backup_store_sources %}
+check_source "{{ src.name }}" {{ src.retention_days }}
+CURRENT=""
+# The store key is what Gatus computes from group+name: sanitize("backup-store")
+# + "_" + sanitize("{{ src.name }}").
+report "backup-store_{{ src.name }}" \
+       "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \
+       "${SRC_MSG[{{ src.name }}]:-no result}"
+{% endfor %}
+
+# --- capacity ---
+CURRENT="__store"
+printf '\n%s\n' "== disk"
+use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9')
+avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ')
+if   [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free"
+elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free"
+else ok "store is ${use}% full, ${avail} free"; fi
+
+CURRENT=""
+report "backup-store_pull-job" \
+       "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \
+       "${SRC_MSG[__store]:-no result}"
+
+printf '\n%s\n' "-----"
+if [ "$fails" -gt 0 ]; then
+  echo "RESULT: $fails failure(s), $warns warning(s)"
+  echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager"
+  exit 1
+fi
+if [ "$warns" -gt 0 ]; then
+  echo "RESULT: all checks passed, $warns warning(s)"
+else
+  echo "RESULT: all checks passed"
+fi
+exit 0

ansible/roles/backup_store/templates/check-backups.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run the backup verification after the nightly pull
+Requires=check-backups.service
+
+[Timer]
+OnCalendar={{ backup_store_check_on_calendar }}
+# Run a missed occurrence on the next boot rather than skipping the day.
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/backup_store/templates/pull-backups.sh.j2

@@ -22,7 +22,12 @@ pull_{{ src.name | replace('-', '_') }}() {
   local dir="${STORE}/{{ src.name }}"
   mkdir -p "$dir"
   log "Pulling {{ src.name }} from {{ src.source }}..."
-  if rsync -az --timeout=120 \
+  # --exclude '*.partial': a dump that died mid-write leaves one behind, owned
+  # root:root 0600 because the chown only happens after a successful mv. Without
+  # this exclude the pull account cannot read it and rsync fails for the WHOLE
+  # source — so one failed dump would silently block every subsequent pull of
+  # that service. An incomplete artefact is never worth transferring anyway.
+  if rsync -az --timeout=120 --exclude '*.partial' \
        -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
        "{{ src.source }}" "$dir/"; then
     log "  {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"

ansible/roles/bitcoin_knots/README.md

@@ -0,0 +1,85 @@
+# `bitcoin_knots`
+
+Builds Bitcoin Knots from source with PGP + SHA256 verification of the release
+tarball, runs it as a full node on `knots-box`, and keeps a health check on a
+systemd timer. The second play in the calling playbook publishes the P2P port
+from the edge host via `socket_proxy`.
+
+Converted from `deploy_bitcoin_knots_playbook.yml` (892 lines) under Plan 6. The
+playbook is now 40 lines.
+
+## The build is guarded; the chain is never touched
+
+`build.yml` is 32 tasks, every one carrying
+`when: not bitcoind_binary_exists.stat.exists`. On a host that already has the
+binary the whole download / verify / 30-60 minute compile skips — **including the
+two `state: absent` deletions**, which target `/opt/bitcoin-knots/source` and the
+extracted build directory.
+
+The chain lives elsewhere and nothing here touches it:
+
+| | |
+|---|---|
+| `bitcoin_knots_dir` | `/opt/bitcoin-knots` — build tree, safe to delete |
+| `bitcoin_data_dir` | `/var/lib/bitcoin` — config, logs, wallets |
+| `bitcoin_large_data_dir` | `/mnt/knots_data` — **~875 GB of blockchain** |
+
+The signature-verification tasks are the security control of this role. They are
+copied verbatim; do not "simplify" them.
+
+## ⚠ This node is half of the mining setup
+
+`bitcoin.conf` carries a DATUM Gateway section that was hand-added on the node
+and was **missing from the playbook's template**:
+
+```ini
+blockmaxsize=3985000
+blockmaxweight=3985000
+blocknotify=killall -USR1 datum_gateway
+maxmempool=1000
+blockreconstructionextratxn=1000000
+```
+
+`blocknotify` is how `datum_gateway` learns a new block landed. Running the old
+playbook would have deleted all of it, and solo mining would have carried on
+grinding against a stale template — a silent failure that costs money rather
+than raising an error. The template now carries it behind
+`bitcoin_datum_gateway_enabled`.
+
+**bitcoin-knots and datum-gateway are one system, not two services.** Changing
+either config means thinking about both.
+
+## The restart handler, and why exactness matters now
+
+The hand-written `Restart bitcoind` handler carried
+`when: uptime_kuma_enabled | default(false)`, so it had been inert since the
+decommissioning: `bitcoin.conf` and the systemd unit both notify it and neither
+could restart anything. A config change applied to disk, reported success, and
+never took effect.
+
+It is ungated here — which raises the bar for the template. **Any** residual
+difference between the template and the live file, down to a trailing newline,
+means the task reports `changed` and restarts a Bitcoin node on every run. It
+took four rounds of `--check --diff` to reach `changed=0`: the DATUM section, an
+explanatory comment that was rendering into the deployed file (now a `{# #}`
+Jinja comment), a `# Pruning (optional)` comment the live file had, and one
+trailing blank line.
+
+## `dbcache`
+
+Computed as 90% of RAM unless `bitcoin_dbcache_mb_override` is set. The live node
+was hand-tuned to **200 MB**; the calculation produces 3528. As with fulcrum,
+`set_fact` outranks role defaults, so the *calculation* honours the override — a
+value pinned only in `defaults/` is silently ignored.
+
+## Monitoring: one variable, no product knowledge
+
+The check tests bitcoind's RPC and records the answer in its exit code, which
+systemd keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. Set
+`healthcheck_push_url` to report anywhere that accepts an HTTP ping.
+
+The timer had last fired **2026-08-09** while still reporting `active` and
+`enabled` — the same `OnBootSec` + `OnUnitActiveSec` dead chain as fulcrum, where
+nothing re-arms it if the service does not run in a given boot. The role runs the
+check once after enabling, which both smoke-tests it and supplies the reference
+the timer schedules from.

ansible/roles/bitcoin_knots/defaults/main.yml

@@ -0,0 +1,78 @@
+# Bitcoin Knots Configuration Variables
+
+# Version - REQUIRED: Specify exact version/tag to build
+# The only version string. There used to be a second, v-prefixed copy
+# (bitcoin_knots_version) that nothing read - two hand-maintained copies of one
+# fact, with nothing keeping them in step.
+bitcoin_knots_version_short: "29.2.knots20251110"
+
+# Directories
+bitcoin_knots_dir: /opt/bitcoin-knots
+bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source"
+bitcoin_data_dir: /var/lib/bitcoin  # Standard location for config, logs, wallets
+bitcoin_large_data_dir: /mnt/knots_data  # Custom location for blockchain data (blocks, chainstate)
+bitcoin_conf_dir: /etc/bitcoin
+
+# Network
+bitcoin_rpc_port: 8332
+# The edge host's socket-proxy/Caddy play needs this too, and a role default is
+# invisible outside this role. The authoritative value for the live deployment is
+# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the
+# protocol standard, so the role still works standalone.
+bitcoin_p2p_port: 8333
+bitcoin_rpc_bind: "0.0.0.0"
+
+# Build options
+bitcoin_build_jobs: 4  # Parallel build jobs (-j flag), adjust based on CPU cores
+bitcoin_build_prefix: /usr/local
+
+# Configuration options
+bitcoin_enable_txindex: true  # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX)
+bitcoin_max_connections: 125
+# dbcache will be calculated as 90% of host RAM automatically in playbook
+
+# ZMQ Configuration
+bitcoin_zmq_enabled: true
+bitcoin_zmq_bind: "tcp://0.0.0.0"
+bitcoin_zmq_port_rawblock: 28332
+bitcoin_zmq_port_rawtx: 28333
+bitcoin_zmq_port_hashblock: 28334
+bitcoin_zmq_port_hashtx: 28335
+
+# Service user
+bitcoin_user: bitcoin
+bitcoin_group: bitcoin
+
+# --- Health check ----------------------------------------------------------
+# Checks bitcoind RPC and records the answer in its exit code, which systemd
+# keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`.
+#
+# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means
+# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping
+# works; nothing here is specific to a monitoring product.
+healthcheck_push_url: ""
+# Bearer token for the Gatus external endpoint. Required whenever a push URL
+# is set: Gatus rejects an unauthenticated push with 401.
+healthcheck_push_token: ""
+
+# --- Logging ----------------------------------------------------------------
+# The live node logs to a file. Set to "" to use printtoconsole=1 (journald).
+bitcoin_logfile: "{{ bitcoin_data_dir }}/debug.log"
+
+# --- dbcache ----------------------------------------------------------------
+# Computed as 90% of RAM unless this is set. The live node was hand-tuned to
+# 200 MB; the calculation would have produced 3528. As with fulcrum, note that
+# set_fact outranks role defaults, so the CALCULATION has to honour this - a
+# value pinned only in defaults/ is silently ignored.
+bitcoin_dbcache_mb_override: 200
+
+# --- DATUM Gateway ----------------------------------------------------------
+# This node feeds block templates to datum_gateway on knots-box. These settings
+# were hand-added to bitcoin.conf and were missing from the template, so a
+# playbook run would have removed them and broken the mining setup.
+bitcoin_datum_gateway_enabled: true
+bitcoin_blockmaxsize: 3985000
+bitcoin_blockmaxweight: 3985000
+bitcoin_blocknotify: "killall -USR1 datum_gateway"
+bitcoin_maxmempool: 1000
+bitcoin_blockreconstructionextratxn: 1000000

ansible/roles/bitcoin_knots/handlers/main.yml

@@ -0,0 +1,14 @@
+---
+# Ungated on purpose. The hand-written handler carried
+#   when: uptime_kuma_enabled | default(false)
+# so it has been inert since the decommissioning. Two tasks notify it —
+# bitcoin.conf and the systemd unit — and neither could actually restart
+# bitcoind. A configuration change to a Bitcoin node therefore applied to disk,
+# reported success, and silently never took effect.
+#
+# Restarting bitcoind re-opens the chainstate; it does not reindex.
+- name: Restart bitcoind
+  systemd:
+    name: bitcoind
+    state: restarted
+    daemon_reload: yes

ansible/roles/bitcoin_knots/tasks/build.yml

@@ -0,0 +1,222 @@
+---
+# Every task here is guarded by `when: not bitcoind_binary_exists.stat.exists`,
+# so on a host that already has the binary the whole download / verify / build
+# sequence skips — including the two `state: absent` deletions, which target
+# /opt/bitcoin-knots/{source,bitcoin-<version>} and never the chain data in
+# /mnt/knots_data.
+- name: Check if bitcoind binary already exists
+  stat:
+    path: "{{ bitcoin_build_prefix }}/bin/bitcoind"
+  register: bitcoind_binary_exists
+  changed_when: false
+
+- name: Install gnupg for signature verification
+  apt:
+    name: gnupg
+    state: present
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Import Luke Dashjr's Bitcoin Knots signing key
+  command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964
+  register: key_import
+  changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout"
+  when: not bitcoind_binary_exists.stat.exists
+  failed_when: key_import.rc != 0
+
+- name: Display imported key fingerprint
+  command: gpg --fingerprint 90C8019E36C2E964
+  register: key_fingerprint
+  changed_when: false
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Download SHA256SUMS file
+  get_url:
+    url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS"
+    dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS"
+    mode: '0644'
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Download SHA256SUMS.asc signature file
+  get_url:
+    url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc"
+    dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc"
+    mode: '0644'
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Verify PGP signature on SHA256SUMS file
+  command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS
+  register: sha256sums_verification
+  changed_when: false
+  failed_when: false  # Don't fail here - check for 'Good signature' in next task
+  when: not bitcoind_binary_exists.stat.exists
+
+
+- name: Display SHA256SUMS verification result
+  debug:
+    msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}"
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Fail if SHA256SUMS signature verification failed
+  fail:
+    msg: "SHA256SUMS signature verification failed. Aborting build."
+  when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr)
+
+- name: Remove any existing tarball to force fresh download
+  file:
+    path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz
+    state: absent
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Download Bitcoin Knots source tarball
+  get_url:
+    url: "{{ bitcoin_source_tarball_url }}"
+    dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz"
+    mode: '0644'
+    validate_certs: yes
+    force: yes
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Calculate SHA256 checksum of downloaded tarball
+  command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz
+  register: tarball_checksum
+  changed_when: false
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Extract expected checksum from SHA256SUMS file
+  shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}'
+  register: expected_checksum
+  changed_when: false
+  when: not bitcoind_binary_exists.stat.exists
+  failed_when: expected_checksum.stdout == ""
+
+- name: Display checksum comparison
+  debug:
+    msg:
+      - "Expected: {{ expected_checksum.stdout | trim }}"
+      - "Actual: {{ tarball_checksum.stdout.split()[0] }}"
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Verify tarball checksum matches SHA256SUMS
+  fail:
+    msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}"
+  when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0]
+
+- name: Remove existing source directory if it exists (to force fresh extraction)
+  file:
+    path: "{{ bitcoin_knots_source_dir }}"
+    state: absent
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Remove extracted directory if it exists (from previous runs)
+  file:
+    path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}"
+    state: absent
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Extract verified source tarball
+  unarchive:
+    src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz
+    dest: "{{ bitcoin_knots_dir }}"
+    remote_src: yes
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Check if extracted directory exists
+  stat:
+    path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}"
+  register: extracted_dir_stat
+  changed_when: false
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Rename extracted directory to expected name
+  command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}"
+  when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists
+
+- name: Check if CMakeLists.txt exists
+  stat:
+    path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt"
+  register: cmake_exists
+  changed_when: false
+  when: not bitcoind_binary_exists.stat.exists
+
+- name: Create CMake build directory
+  file:
+    path: "{{ bitcoin_knots_source_dir }}/build"
+    state: directory
+    mode: '0755'
+  when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
+
+- name: Configure Bitcoin Knots build with CMake
+  command: >
+    cmake
+    -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }}
+    -DBUILD_BITCOIN_WALLET=OFF
+    -DCMAKE_BUILD_TYPE=Release
+    -DWITH_ZMQ=ON
+    ..
+  args:
+    chdir: "{{ bitcoin_knots_source_dir }}/build"
+  when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
+  register: configure_result
+  changed_when: true
+
+- name: Verify CMake enabled ZMQ
+  shell: |
+    set -e
+    cd "{{ bitcoin_knots_source_dir }}/build"
+    cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ'
+  when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
+  register: zmq_check
+  changed_when: false
+
+- name: Fail if CMakeLists.txt not found
+  fail:
+    msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots."
+  when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false))
+
+- name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes)
+  command: cmake --build . -j{{ bitcoin_build_jobs }}
+  args:
+    chdir: "{{ bitcoin_knots_source_dir }}/build"
+  when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
+  async: 3600
+  poll: 0
+  register: build_result
+  changed_when: true
+
+- name: Check build status
+  async_status:
+    jid: "{{ build_result.ansible_job_id }}"
+  register: build_job_result
+  until: build_job_result.finished
+  retries: 120
+  delay: 60
+  when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined
+
+- name: Fail if build failed
+  fail:
+    msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}"
+  when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false)
+
+- name: Install Bitcoin Knots binaries
+  command: cmake --install .
+  args:
+    chdir: "{{ bitcoin_knots_source_dir }}/build"
+  when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
+  changed_when: true
+
+- name: Verify bitcoind binary exists
+  stat:
+    path: "{{ bitcoin_build_prefix }}/bin/bitcoind"
+  register: bitcoind_installed
+  changed_when: false
+
+- name: Verify bitcoin-cli binary exists
+  stat:
+    path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli"
+  register: bitcoin_cli_installed
+  changed_when: false
+
+- name: Fail if binaries not found
+  fail:
+    msg: "Bitcoin Knots binaries not found after installation"
+  when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists

ansible/roles/bitcoin_knots/tasks/configure.yml

@@ -0,0 +1,20 @@
+---
+# Ownership copied verbatim from the playbook this replaces; verified
+# mechanically against `git show HEAD:` rather than retyped from memory.
+- name: Create bitcoin.conf configuration file
+  ansible.builtin.template:
+    src: bitcoin.conf.j2
+    dest: "{{ bitcoin_conf_dir }}/bitcoin.conf"
+    owner: "{{ bitcoin_user }}"
+    group: "{{ bitcoin_group }}"
+    mode: '0640'
+  notify: Restart bitcoind
+
+- name: Create systemd service file for bitcoind
+  ansible.builtin.template:
+    src: bitcoind.service.j2
+    dest: /etc/systemd/system/bitcoind.service
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Restart bitcoind

ansible/roles/bitcoin_knots/tasks/healthcheck.yml

@@ -0,0 +1,56 @@
+---
+# Everything here answers "is bitcoind healthy" and records the answer. The
+# Uptime Kuma specifics that used to follow — an embedded Python script creating
+# monitors over the API, a /tmp credentials file, push-URL extraction and a
+# systemd Environment= rewrite — are gone. Where it reports is now one variable,
+# healthcheck_push_url. See the role README.
+- name: Install curl for health check script
+  apt:
+    name: curl
+    state: present
+
+- name: Create Bitcoin Knots health check script
+  ansible.builtin.template:
+    src: healthcheck.sh.j2
+    dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+
+- name: Create systemd service for Bitcoin Knots health check
+  ansible.builtin.template:
+    src: healthcheck.service.j2
+    dest: /etc/systemd/system/bitcoin-knots-healthcheck.service
+    owner: root
+    group: root
+    mode: "0600"
+
+- name: Create systemd timer for Bitcoin Knots health check
+  ansible.builtin.template:
+    src: healthcheck.timer.j2
+    dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Reload systemd daemon for health check
+  systemd:
+    daemon_reload: yes
+
+- name: Enable and restart the Bitcoin Knots health check timer
+  systemd:
+    name: bitcoin-knots-healthcheck.timer
+    enabled: yes
+    state: restarted
+    daemon_reload: yes
+
+# Runs the check once, which is both a smoke test and the thing that actually
+# arms the timer. This timer is OnBootSec + OnUnitActiveSec with no OnCalendar:
+# OnBootSec elapses once, and OnUnitActiveSec needs the SERVICE to have run this
+# boot to have anything to schedule from. Restarting the timer does not supply
+# that reference; running the service does. The live timer had last fired on
+# 2026-08-09 while still reporting `active` and `enabled`.
+- name: Run the Bitcoin Knots health check once to arm the timer
+  command: systemctl start bitcoin-knots-healthcheck.service
+  changed_when: false

ansible/roles/bitcoin_knots/tasks/install.yml

@@ -0,0 +1,104 @@
+---
+- name: Calculate dbcache as a share of system RAM
+  set_fact:
+    bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}"
+  when: bitcoin_dbcache_mb_override | string | length == 0
+
+- name: Use the explicit dbcache override
+  set_fact:
+    bitcoin_dbcache_mb: "{{ bitcoin_dbcache_mb_override }}"
+  when: bitcoin_dbcache_mb_override | string | length > 0
+  changed_when: false
+
+- name: Display calculated dbcache value
+  debug:
+    msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)"
+
+
+- name: Install build dependencies
+  apt:
+    name:
+      - build-essential
+      - libtool
+      - autotools-dev
+      - automake
+      - pkg-config
+      - bsdmainutils
+      - python3
+      - python3-pip
+      - libevent-dev
+      - libboost-system-dev
+      - libboost-filesystem-dev
+      - libboost-test-dev
+      - libboost-thread-dev
+      - libboost-chrono-dev
+      - libboost-program-options-dev
+      - libboost-dev
+      - libssl-dev
+      - libdb-dev
+      - libminiupnpc-dev
+      - libzmq3-dev
+      - libnatpmp-dev
+      - libsqlite3-dev
+      - git
+      - curl
+      - wget
+      - cmake
+    state: present
+    update_cache: yes
+
+- name: Create bitcoin group
+  group:
+    name: "{{ bitcoin_group }}"
+    system: yes
+    state: present
+
+- name: Create bitcoin user
+  user:
+    name: "{{ bitcoin_user }}"
+    group: "{{ bitcoin_group }}"
+    system: yes
+    shell: /usr/sbin/nologin
+    home: "{{ bitcoin_data_dir }}"
+    create_home: yes
+    state: present
+
+- name: Create bitcoin-knots directory
+  file:
+    path: "{{ bitcoin_knots_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Create bitcoin-knots source directory
+  file:
+    path: "{{ bitcoin_knots_source_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Create bitcoin data directory (for config, logs, wallets)
+  file:
+    path: "{{ bitcoin_data_dir }}"
+    state: directory
+    owner: "{{ bitcoin_user }}"
+    group: "{{ bitcoin_group }}"
+    mode: '0750'
+
+- name: Create bitcoin large data directory (for blockchain)
+  file:
+    path: "{{ bitcoin_large_data_dir }}"
+    state: directory
+    owner: "{{ bitcoin_user }}"
+    group: "{{ bitcoin_group }}"
+    mode: '0750'
+
+- name: Create bitcoin config directory
+  file:
+    path: "{{ bitcoin_conf_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'

ansible/roles/bitcoin_knots/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+# import_tasks, not include_tasks: static imports stay visible to --list-tasks,
+# which is how this conversion was verified against the playbook it replaced.
+- ansible.builtin.import_tasks: install.yml
+- ansible.builtin.import_tasks: build.yml
+- ansible.builtin.import_tasks: configure.yml
+- ansible.builtin.import_tasks: service.yml
+- ansible.builtin.import_tasks: healthcheck.yml

ansible/roles/bitcoin_knots/tasks/service.yml

@@ -0,0 +1,37 @@
+---
+- name: Reload systemd daemon
+  systemd:
+    daemon_reload: yes
+
+- name: Enable and start bitcoind service
+  systemd:
+    name: bitcoind
+    enabled: yes
+    state: started
+
+- name: Wait for bitcoind RPC to be available
+  uri:
+    url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}"
+    method: POST
+    body_format: json
+    body:
+      jsonrpc: "1.0"
+      id: "healthcheck"
+      method: "getblockchaininfo"
+      params: []
+    user: "{{ bitcoin_rpc_user }}"
+    password: "{{ bitcoin_rpc_password }}"
+    status_code: 200
+    timeout: 10
+  register: rpc_check
+  until: rpc_check.status == 200
+  retries: 30
+  delay: 5
+  ignore_errors: yes
+
+- name: Display RPC connection status
+  debug:
+    msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}"
+
+# ═════════════════════════════════════════════════════════════════════════
+# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.

ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2

@@ -0,0 +1,67 @@
+# Bitcoin Knots Configuration
+# Generated by Ansible
+
+# Data directory (blockchain storage)
+datadir={{ bitcoin_large_data_dir }}
+
+# RPC Configuration
+server=1
+rpcuser={{ bitcoin_rpc_user }}
+rpcpassword={{ bitcoin_rpc_password }}
+rpcbind={{ bitcoin_rpc_bind }}
+rpcport={{ bitcoin_rpc_port }}
+rpcallowip=0.0.0.0/0
+
+# Network Configuration
+listen=1
+port={{ bitcoin_p2p_port }}
+maxconnections={{ bitcoin_max_connections }}
+
+# Performance
+dbcache={{ bitcoin_dbcache_mb }}
+
+# Transaction Index (optional)
+{% if bitcoin_enable_txindex %}
+txindex=1
+{% endif %}
+
+{# The live node carries this comment and the template never produced it, so a
+   run would have silently deleted it. Harmless in itself, but matching it keeps
+   this task at `ok` - which means any future `changed` here is a real signal
+   rather than known noise. #}
+# Pruning (optional)
+
+# Logging
+logtimestamps=1
+{% if bitcoin_logfile %}
+logfile={{ bitcoin_logfile }}
+{% else %}
+printtoconsole=1
+{% endif %}
+
+# ZMQ Configuration
+{% if bitcoin_zmq_enabled | default(false) %}
+zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }}
+zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }}
+zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }}
+zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }}
+{% endif %}
+
+# Security
+disablewallet=1
+{% if bitcoin_datum_gateway_enabled %}
+
+{# These were hand-added on the node and were NOT in this template, so running
+   the playbook would have stripped them. blocknotify is how datum_gateway
+   learns a new block landed; without it solo mining keeps grinding on a stale
+   template - a silent failure that costs money rather than raising an error.
+   Kept as a Jinja comment so the explanation stays in the repo and out of the
+   deployed config. #}
+# Specific for DATUM gateway
+blockmaxsize={{ bitcoin_blockmaxsize }}
+blockmaxweight={{ bitcoin_blockmaxweight }}
+blocknotify={{ bitcoin_blocknotify }}
+maxmempool={{ bitcoin_maxmempool }}
+blockreconstructionextratxn={{ bitcoin_blockreconstructionextratxn }}
+
+{% endif %}

ansible/roles/bitcoin_knots/templates/bitcoind.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Bitcoin Knots daemon
+After=network.target
+
+[Service]
+Type=simple
+User={{ bitcoin_user }}
+Group={{ bitcoin_group }}
+ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf
+Restart=always
+RestartSec=10
+TimeoutStopSec=600
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/bitcoin_knots/templates/healthcheck.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=Bitcoin Knots Health Check
+After=network.target bitcoind.service
+
+[Service]
+Type=oneshot
+User=root
+ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh
+Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }}
+Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }}
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2

@@ -0,0 +1,69 @@
+#!/bin/bash
+# Bitcoin Knots health check — managed by Ansible (roles/bitcoin_knots)
+#
+# The exit code is the answer and systemd keeps it:
+#     systemctl is-failed bitcoin-knots-healthcheck.service
+# Reporting anywhere else is optional and generic.
+#
+#
+
+RPC_HOST="{{ bitcoin_rpc_bind }}"
+RPC_PORT={{ bitcoin_rpc_port }}
+RPC_USER="{{ bitcoin_rpc_user }}"
+RPC_PASSWORD="{{ bitcoin_rpc_password }}"
+PUSH_URL="${HEALTHCHECK_PUSH_URL:-}"
+PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}"
+
+# Check if bitcoind RPC is responding
+check_bitcoind() {
+    local response
+    response=$(curl -s --max-time 30 \
+        --user "${RPC_USER}:${RPC_PASSWORD}" \
+        --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \
+        --header 'Content-Type: application/json' \
+        "http://${RPC_HOST}:${RPC_PORT}" 2>&1)
+    
+    if [ $? -eq 0 ]; then
+        # Check if response contains a non-null error
+        # Successful responses have "error": null, failures have "error": {...}
+        if echo "$response" | grep -q '"error":null\|"error": null'; then
+            return 0
+        else
+            return 1
+        fi
+    else
+        return 1
+    fi
+}
+
+report() {
+    local status=$1
+    local msg=$2
+    
+    # No push URL is normal, not an error: the exit code below is still a
+    # complete answer for anything reading unit state.
+    [ -n "$PUSH_URL" ] || return 0
+    
+    # URL encode spaces in message
+    local encoded_msg="${msg// /%20}"
+    
+    # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's
+    # GET with ?status=up. The callers still pass up/down, so the mapping is
+    # done here rather than at every call site.
+    local _ok=false
+    [ "${status}" = "up" ] && _ok=true
+    if ! curl -s --max-time 15 --retry 2 -o /dev/null -X POST \
+        -H "Authorization: Bearer ${PUSH_TOKEN}" \
+        "${PUSH_URL}?success=${_ok}&error=${encoded_msg}"; then
+        return 1
+    fi
+}
+
+# Main health check
+if check_bitcoind; then
+    report "up" "OK"
+    exit 0
+else
+    report "down" "bitcoind RPC not responding"
+    exit 1
+fi

ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Bitcoin Knots Health Check Timer
+Requires=bitcoind.service
+
+[Timer]
+OnBootSec=1min
+OnUnitActiveSec=1min
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/caddy_site/defaults/main.yml

@@ -15,7 +15,8 @@ caddy_site_headers_up: {}       # {"X-Forwarded-Host": "wallet.example.com"}
 # expression for the username silently passes through as literal text.
 caddy_site_basic_auth: []       # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}]
 
-# Placement. caddy_sites_dir comes from services_config.yml; this is the fallback.
+# Placement. This is now the only definition of caddy_sites_dir - services_config.yml
+# used to carry an identical copy, which was removed as redundant.
 caddy_sites_dir: /etc/caddy/sites-enabled
 
 # Rendered site files can carry credentials (basic_auth hashes), so --diff is

ansible/roles/datum_gateway/README.md

@@ -0,0 +1,73 @@
+# `datum_gateway`
+
+Builds and runs [DATUM Gateway](https://github.com/OCEAN-xyz/datum_gateway), the
+solo/pooled mining gateway, on `knots-box`. The calling playbook adds two more
+plays on the edge host: the dashboard via `caddy_site`, and the public Stratum
+port via `socket_proxy`.
+
+Converted from `deploy_datum_gateway_playbook.yml` (802 lines) under Plan 6. The
+playbook is now 68 lines and keeps all three plays.
+
+## ⚠ This is half of a system
+
+The Bitcoin Knots node on the same host feeds this gateway through
+`blocknotify=killall -USR1 datum_gateway` in `bitcoin.conf` — see
+`roles/bitcoin_knots/README.md`, where that line was found to be missing from the
+template entirely. **Changing either config means thinking about both.**
+
+Interrupting Stratum costs mining shares. Check before any run that restarts it:
+
+```bash
+ss -tn state established '( sport = :23334 )'
+```
+
+## Two pieces of drift where the node was right
+
+The repo and the node had diverged on values that matter, and the deployment
+would have applied the repo's:
+
+| | node (correct) | repo said |
+|---|---|---|
+| `datum_mining_address` | `bc1qvrj3g84…` | `bc1qdse9dsg…` |
+| `pool_pass_workers` / `_full_users` | `false` | `true` |
+
+The address is the one that would have hurt: **it is where block rewards are
+paid**, and unlike fulcrum and bitcoin-knots the `Restart datum-gateway` handler
+here was *never* gated, so the change would have applied immediately rather than
+sitting inert. Both corrected in the vault and defaults, with notes.
+
+Verify semantics rather than text when touching `config.json` — render it and
+compare parsed JSON, because the live file is single-line and the template is
+pretty-printed, so a textual diff is all noise:
+
+```python
+json.load(open('live.json')) == json.load(open('rendered.json'))
+```
+
+## `config.json` holds real secrets — diff is suppressed
+
+The file carries `bitcoind.rpcpassword` and `api.admin_password`. `--diff`
+prints rendered content, so the task sets `diff: false` by default; pass
+`-e datum_reveal_config=true` to opt in.
+
+Note `pool_pass_workers` / `pool_pass_full_users` are **booleans**, not
+passwords, despite the names — they control DATUM's pool-password passthrough.
+`mining.pool_address` is a Bitcoin address and public by nature.
+
+## Expect `changed` on the compile every run
+
+`Configure cmake build` and `Compile datum_gateway` are bare `command:` tasks
+with no `changed_when`, so they always report changed and always re-run. The
+build is reproducible — `Install datum_gateway binary` sees identical content and
+does not replace it, so the installed binary keeps its original timestamp — but
+the compile itself is wasted work on every run. That is the idempotent floor, not
+drift.
+
+## Monitoring: one variable, no product knowledge
+
+The check tests the gateway API and records the answer in its exit code, which
+systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. Set
+`healthcheck_push_url` to report anywhere accepting an HTTP ping.
+
+Unlike the other services here, only the health-check *timer* handler was gated
+by `uptime_kuma_enabled`; the main deployment restart worked throughout.

ansible/roles/datum_gateway/defaults/main.yml

@@ -0,0 +1,61 @@
+# DATUM Gateway Configuration Variables
+# https://github.com/OCEAN-xyz/datum_gateway
+
+# Version - pin to a specific tag
+datum_gateway_version: "v0.4.1beta"
+
+# Directories
+datum_gateway_dir: /opt/datum-gateway
+datum_gateway_source_dir: "{{ datum_gateway_dir }}/source"
+datum_gateway_config_dir: /etc/datum-gateway
+datum_gateway_log_dir: /var/log/datum-gateway
+
+# Binary
+datum_gateway_bin_path: /usr/local/bin/datum_gateway
+
+# Ports
+# The edge host's socket-proxy/Caddy play needs this too, and a role default is
+# invisible outside this role. The authoritative value for the live deployment is
+# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the
+# protocol standard, so the role still works standalone.
+datum_gateway_stratum_port: 23334
+datum_gateway_api_port: 7152
+
+# Stratum settings
+datum_vardiff_min: 524288  # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher)
+
+# Service user
+datum_gateway_user: datum
+datum_gateway_group: datum
+
+# Build options
+datum_gateway_build_jobs: 4
+
+# Bitcoin node connection
+# The gateway runs on the same host as Bitcoin Knots so localhost RPC works.
+# datum_bitcoin_rpc_url should include http:// and port.
+datum_bitcoin_rpc_url: "http://127.0.0.1:8332"
+# Note: bitcoin_rpc_user and bitcoin_rpc_password come from group_vars/all/vault.yml
+
+# Mining config
+datum_coinbase_tag_primary: "DATUM"
+datum_coinbase_tag_secondary: "BY ORDER OF BIP110"
+
+# Both false on the node; the vars file said true. Corrected 2026-09-13 to
+# match reality, on the same basis as datum_mining_address: the running node
+# is authoritative. These control DATUM's pool-password passthrough.
+datum_pool_pass_workers: false
+datum_pool_pass_full_users: false
+datum_pooled_mining_only: true
+
+
+# --- Health check -----------------------------------------------------------
+# Checks the DATUM Gateway API and records the answer in its exit code, which
+# systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`.
+#
+# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means
+# check, exit honestly, report nowhere.
+healthcheck_push_url: ""
+# Bearer token for the Gatus external endpoint. Required whenever a push URL
+# is set: Gatus rejects an unauthenticated push with 401.
+healthcheck_push_token: ""

ansible/roles/datum_gateway/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+- name: Restart datum-gateway
+  systemd:
+    name: datum-gateway
+    state: restarted
+    daemon_reload: yes
+
+# Ungated. This one carried `when: uptime_kuma_enabled | default(false)` while
+# the main Restart datum-gateway handler above did not — so on this service the
+# deployment restart worked and only the health-check timer restart was dead.
+- name: Restart datum-gateway health check timer
+  systemd:
+    name: datum-gateway-healthcheck.timer
+    state: restarted
+    daemon_reload: yes

ansible/roles/datum_gateway/tasks/configure.yml

@@ -0,0 +1,25 @@
+---
+# Ownership copied verbatim from the playbook this replaces and verified
+# mechanically against `git show HEAD:`.
+- name: Write DATUM Gateway config.json
+  ansible.builtin.template:
+    src: config.json.j2
+    dest: "{{ datum_gateway_config_dir }}/config.json"
+    owner: "{{ datum_gateway_user }}"
+    group: "{{ datum_gateway_group }}"
+    mode: '0640'
+  # config.json carries the bitcoind RPC password, the API admin password and
+  # the pool passwords. `--diff` prints rendered content, so running with --diff
+  # put all of them on the terminal and into any log capturing it. Suppressed by
+  # default; pass -e datum_reveal_config=true when you genuinely need the diff.
+  diff: "{{ datum_reveal_config | default(false) | bool }}"
+  notify: Restart datum-gateway
+
+- name: Create datum-gateway systemd service
+  ansible.builtin.template:
+    src: datum-gateway.service.j2
+    dest: /etc/systemd/system/datum-gateway.service
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Restart datum-gateway

ansible/roles/datum_gateway/tasks/healthcheck.yml

@@ -0,0 +1,50 @@
+---
+# Everything here answers "is DATUM Gateway healthy" and records the answer. The
+# Uptime Kuma specifics that used to follow — an embedded Python script creating
+# monitors over the API, a /tmp credentials file, a push-URL file read back and
+# parsed, and a systemd Environment= rewrite — are gone. Where it reports is now
+# one variable, healthcheck_push_url.
+- name: Create DATUM Gateway health check script
+  ansible.builtin.template:
+    src: healthcheck.sh.j2
+    dest: /usr/local/bin/datum-gateway-healthcheck-push.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+
+- name: Create datum-gateway health check systemd service
+  ansible.builtin.template:
+    src: healthcheck.service.j2
+    dest: /etc/systemd/system/datum-gateway-healthcheck.service
+    owner: root
+    group: root
+    mode: "0600"
+  notify: Restart datum-gateway health check timer
+
+- name: Create datum-gateway health check systemd timer
+  ansible.builtin.template:
+    src: healthcheck.timer.j2
+    dest: /etc/systemd/system/datum-gateway-healthcheck.timer
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Restart datum-gateway health check timer
+
+- name: Reload systemd daemon after health check units
+  systemd:
+    daemon_reload: yes
+
+# Ungated: enabling a timer is deployment, not monitoring.
+- name: Enable and restart the datum-gateway health check timer
+  systemd:
+    name: datum-gateway-healthcheck.timer
+    enabled: yes
+    state: restarted
+    daemon_reload: yes
+
+# Arms the timer and smoke-tests the check. See roles/bitcoin_knots/README.md for
+# why restarting the timer alone is not enough with OnBootSec + OnUnitActiveSec.
+- name: Run the DATUM Gateway health check once to arm the timer
+  command: systemctl start datum-gateway-healthcheck.service
+  changed_when: false

ansible/roles/datum_gateway/tasks/install.yml

@@ -0,0 +1,74 @@
+---
+- name: Install DATUM Gateway build dependencies
+  apt:
+    name:
+      - cmake
+      - build-essential
+      - git
+      - libjansson-dev
+      - libmicrohttpd-dev
+      - libsodium-dev
+      - libcurl4-openssl-dev
+      # Runtime-only (netcat for health check)
+      - netcat-openbsd
+    state: present
+    update_cache: yes
+
+# ===========================================
+# System User and Directories
+# ===========================================
+- name: Create datum system user
+  user:
+    name: "{{ datum_gateway_user }}"
+    system: yes
+    shell: /usr/sbin/nologin
+    home: "{{ datum_gateway_dir }}"
+    create_home: no
+    comment: "DATUM Gateway"
+
+- name: Create DATUM Gateway directories
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner }}"
+    group: "{{ datum_gateway_group }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ datum_gateway_dir }}",        owner: root,                   mode: "0755" }
+    - { path: "{{ datum_gateway_source_dir }}", owner: root,                   mode: "0755" }
+    - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" }
+    - { path: "{{ datum_gateway_log_dir }}",    owner: "{{ datum_gateway_user }}", mode: "0750" }
+
+# ===========================================
+# Build from Source
+# ===========================================
+- name: Clone DATUM Gateway repository at {{ datum_gateway_version }}
+  git:
+    repo: https://github.com/OCEAN-xyz/datum_gateway.git
+    dest: "{{ datum_gateway_source_dir }}"
+    version: "{{ datum_gateway_version }}"
+    force: yes
+  register: git_clone
+
+- name: Configure cmake build
+  command: cmake . -DCMAKE_BUILD_TYPE=Release
+  args:
+    chdir: "{{ datum_gateway_source_dir }}"
+
+- name: Compile datum_gateway
+  command: make -j{{ datum_gateway_build_jobs }}
+  args:
+    chdir: "{{ datum_gateway_source_dir }}"
+
+- name: Install datum_gateway binary
+  copy:
+    src: "{{ datum_gateway_source_dir }}/datum_gateway"
+    dest: "{{ datum_gateway_bin_path }}"
+    remote_src: yes
+    owner: root
+    group: root
+    mode: "0755"
+  notify: Restart datum-gateway
+
+# ===========================================
+# Configuration

ansible/roles/datum_gateway/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+# import_tasks, not include_tasks: static imports stay visible to --list-tasks.
+- ansible.builtin.import_tasks: install.yml
+- ansible.builtin.import_tasks: configure.yml
+- ansible.builtin.import_tasks: service.yml
+- ansible.builtin.import_tasks: healthcheck.yml

ansible/roles/datum_gateway/tasks/service.yml

@@ -0,0 +1,16 @@
+---
+- name: Reload systemd daemon
+  systemd:
+    daemon_reload: yes
+
+- name: Enable and start datum-gateway
+  systemd:
+    name: datum-gateway
+    enabled: yes
+    state: started
+
+# ===========================================
+# Health Check Script + Systemd Timer
+# ===========================================
+# ═════════════════════════════════════════════════════════════════════════
+# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.

ansible/roles/datum_gateway/templates/config.json.j2

@@ -0,0 +1,35 @@
+{
+    "bitcoind": {
+        "rpcuser": "{{ bitcoin_rpc_user }}",
+        "rpcpassword": "{{ bitcoin_rpc_password }}",
+        "rpcurl": "{{ datum_bitcoin_rpc_url }}",
+        "notify_fallback": true
+    },
+    "stratum": {
+        "listen_port": {{ datum_gateway_stratum_port }},
+        "vardiff_min": {{ datum_vardiff_min }}
+    },
+    "mining": {
+        "pool_address": "{{ datum_mining_address }}",
+        "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}",
+        "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}"
+    },
+    "api": {
+        "admin_password": "{{ datum_gateway_admin_password }}",
+        "listen_port": {{ datum_gateway_api_port }},
+        "modify_conf": false
+    },
+    "logger": {
+        "log_to_console": true,
+        "log_to_file": true,
+        "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log",
+        "log_rotate_daily": true,
+        "log_level_console": 2,
+        "log_level_file": 1
+    },
+    "datum": {
+        "pool_pass_workers": {{ datum_pool_pass_workers | lower }},
+        "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }},
+        "pooled_mining_only": {{ datum_pooled_mining_only | lower }}
+    }
+}

ansible/roles/datum_gateway/templates/datum-gateway.service.j2

@@ -0,0 +1,22 @@
+[Unit]
+Description=DATUM Gateway - Bitcoin Mining Gateway
+Documentation=https://github.com/OCEAN-xyz/datum_gateway
+After=network.target bitcoind.service
+Wants=bitcoind.service
+
+[Service]
+User={{ datum_gateway_user }}
+Group={{ datum_gateway_group }}
+Type=simple
+ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json
+Restart=on-failure
+RestartSec=10
+StandardOutput=journal
+StandardError=journal
+
+# Prevent config from being read by other users
+ReadWritePaths={{ datum_gateway_log_dir }}
+ReadOnlyPaths={{ datum_gateway_config_dir }}
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/datum_gateway/templates/healthcheck.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=DATUM Gateway Health Check
+After=network.target datum-gateway.service
+
+[Service]
+Type=oneshot
+User=root
+ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh
+Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }}
+Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }}
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/datum_gateway/templates/healthcheck.sh.j2

@@ -0,0 +1,39 @@
+#!/bin/bash
+# DATUM Gateway health check — managed by Ansible (roles/datum_gateway)
+#
+# The exit code is the answer and systemd keeps it:
+#     systemctl is-failed datum-gateway-healthcheck.service
+# Reporting anywhere else is optional and generic.
+PUSH_URL="${HEALTHCHECK_PUSH_URL:-}"
+PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}"
+STRATUM_PORT={{ datum_gateway_stratum_port }}
+
+check_datum() {
+    # Service must be active and stratum port must be listening
+    systemctl is-active --quiet datum-gateway && \
+    nc -z 127.0.0.1 "${STRATUM_PORT}"
+}
+
+report() {
+    local status=$1
+    local msg=$2
+    # No push URL is normal, not an error: the exit code below is still a
+    # complete answer for anything reading unit state.
+    [ -n "$PUSH_URL" ] || return 0
+    # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's
+    # GET with ?status=up. The callers still pass up/down, so the mapping is
+    # done here rather than at every call site.
+    local _ok=false
+    [ "${status}" = "up" ] && _ok=true
+    curl -s --max-time 15 --retry 2 -o /dev/null -X POST \
+        -H "Authorization: Bearer ${PUSH_TOKEN}" \
+        "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true
+}
+
+if check_datum; then
+    report "up" "OK"
+    exit 0
+else
+    report "down" "DATUM Gateway not responding"
+    exit 1
+fi

ansible/roles/datum_gateway/templates/healthcheck.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=DATUM Gateway Health Check Timer
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=1min
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/forgejo_runner/README.md

@@ -0,0 +1,59 @@
+# `forgejo_runner`
+
+Installs and runs a Forgejo Actions runner, registers it with the Forgejo
+instance, and keeps a health check on a systemd timer.
+
+Converted from `deploy_forgejo_runner_playbook.yml` (409 lines) under Plan 6.
+The playbook is now 16 lines.
+
+## Phases
+
+`tasks/main.yml` imports five files in order:
+
+| | |
+|---|---|
+| `prerequisites.yml` | Docker must be present |
+| `install.yml` | binary, system user, working directory |
+| `configure.yml` | config file, registration with the instance |
+| `service.yml` | systemd unit, start, assert it came up |
+| `healthcheck.yml` | check script, unit, timer |
+
+`import_tasks`, not `include_tasks` — static imports are visible to
+`--list-tasks`, which is how the conversion was verified against the playbook it
+replaced.
+
+## Monitoring: one variable, no product knowledge
+
+This role contains **nothing specific to any monitoring system**. What used to
+be here — an ~80-line embedded Python script creating monitors over the Uptime
+Kuma API, a `/tmp` credentials file, token extraction, a systemd `Environment=`
+rewrite, and 8 `when: uptime_kuma_enabled` guards — is gone.
+
+What remains answers the actual question, *is this service healthy*, and records
+it two ways:
+
+- **the exit code**, which systemd keeps: `systemctl is-failed
+  forgejo-runner-healthcheck.service` is a complete answer with no monitoring
+  system involved at all;
+- **a log file** at `{{ healthcheck_log_file }}`.
+
+To report health somewhere, set one variable:
+
+```yaml
+healthcheck_push_url: "https://example/api/push/TOKEN"
+```
+
+Any endpoint accepting an HTTP ping works. Empty (the default) means check, log,
+exit honestly, report nowhere — which is also the right setting for a *pull*-based
+monitor like Prometheus' textfile collector, since that reads unit state instead.
+
+The push URL is a credential (anyone holding it can forge an "up"), so callers
+pass it from the vault rather than committing it.
+
+## One behaviour change, deliberate
+
+`Assert runner is running` used to be guarded by `uptime_kuma_enabled`, so it
+never ran. It is not a monitoring task — it is the deployment checking its own
+work — and the deprecation banner swept it up by mistake. It is ungated here,
+which means a runner that fails to start now fails the play instead of
+deploying "successfully" in silence.

ansible/roles/forgejo_runner/defaults/main.yml

@@ -0,0 +1,41 @@
+---
+# Binary
+forgejo_runner_version: "6.3.1"
+forgejo_runner_arch: "linux-amd64"
+forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}"
+forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner"
+
+# Runtime
+forgejo_runner_user: "runner"
+forgejo_runner_dir: "/opt/forgejo-runner"
+forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml"
+forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm"
+
+# The Forgejo instance this runner registers with.
+forgejo_instance_url: "https://forgejo.contrapeso.xyz"
+# forgejo_runner_registration_token comes from the vault.
+
+# --- Health check -----------------------------------------------------------
+# The check answers "is this service healthy" and records the answer two ways:
+# a log file, and its own exit code. The exit code is the durable artefact —
+# systemd stores it, so `systemctl is-failed forgejo-runner-healthcheck.service`
+# answers the question with no monitoring system involved at all.
+healthcheck_interval_seconds: 60
+healthcheck_script_dir: /opt/forgejo-runner-healthcheck
+healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh"
+healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log"
+healthcheck_service_name: forgejo-runner-healthcheck
+
+# WHERE TO REPORT HEALTH — the one place to plug in monitoring.
+#
+# Empty means "check, log, exit honestly, report nowhere". Set it to any URL
+# that accepts an HTTP ping and the check will report there. Nothing in this
+# role is specific to a particular monitoring product: the Uptime Kuma API
+# calls, monitor creation and token handling that used to live here are gone.
+#
+# A pull-based monitor (Prometheus node_exporter textfile, say) needs this left
+# empty — it reads the systemd unit state instead.
+healthcheck_push_url: ""
+# Bearer token for the Gatus external endpoint. Required whenever a push URL
+# is set: Gatus rejects an unauthenticated push with 401.
+healthcheck_push_token: ""

ansible/roles/forgejo_runner/tasks/configure.yml

@@ -0,0 +1,43 @@
+---
+- name: Check if config already exists
+  stat:
+    path: "{{ forgejo_runner_config_path }}"
+  register: config_stat
+
+- name: Generate default config
+  shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}"
+  args:
+    chdir: "{{ forgejo_runner_dir }}"
+  when: not config_stat.stat.exists
+
+- name: Set config file ownership
+  file:
+    path: "{{ forgejo_runner_config_path }}"
+    owner: "{{ forgejo_runner_user }}"
+    group: "{{ forgejo_runner_user }}"
+  when: not config_stat.stat.exists
+
+# ── 6. Register runner ─────────────────────────────────────────────
+- name: Check if runner is already registered
+  stat:
+    path: "{{ forgejo_runner_dir }}/.runner"
+  register: runner_stat
+
+- name: Register runner with Forgejo instance
+  command: >
+    {{ forgejo_runner_bin_path }} register --no-interactive
+    --instance {{ forgejo_instance_url }}
+    --token {{ forgejo_runner_registration_token }}
+    --name forgejo-runner-box
+    --labels "{{ forgejo_runner_labels }}"
+  args:
+    chdir: "{{ forgejo_runner_dir }}"
+  when: not runner_stat.stat.exists
+
+- name: Set runner registration file ownership
+  file:
+    path: "{{ forgejo_runner_dir }}/.runner"
+    owner: "{{ forgejo_runner_user }}"
+    group: "{{ forgejo_runner_user }}"
+  when: not runner_stat.stat.exists
+

ansible/roles/forgejo_runner/tasks/healthcheck.yml

@@ -0,0 +1,73 @@
+---
+# Everything here answers "is the service healthy" and records the answer.
+# The Uptime Kuma specifics that used to surround it — an embedded Python script
+# that created monitors over the API, a /tmp credentials file, token extraction,
+# and a systemd Environment= rewrite — are gone. What reports where is now one
+# variable, healthcheck_push_url. See the role README.
+- name: Create healthcheck script directory
+  ansible.builtin.file:
+    path: "{{ healthcheck_script_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Create forgejo-runner healthcheck script
+  ansible.builtin.template:
+    src: healthcheck.sh.j2
+    dest: "{{ healthcheck_script_path }}"
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+
+- name: Create healthcheck systemd service
+  ansible.builtin.template:
+    src: healthcheck.service.j2
+    dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0600"
+
+- name: Create healthcheck systemd timer
+  ansible.builtin.template:
+    src: healthcheck.timer.j2
+    dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer"
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Reload systemd for healthcheck units
+  systemd:
+    daemon_reload: yes
+
+- name: Enable and start healthcheck timer
+  systemd:
+    name: "{{ healthcheck_service_name }}.timer"
+    enabled: yes
+    state: started
+
+- name: Test healthcheck script
+  command: "{{ healthcheck_script_path }}"
+  register: healthcheck_test
+  changed_when: false
+
+- name: Verify healthcheck script works
+  assert:
+    that:
+      - healthcheck_test.rc == 0
+    fail_msg: "Healthcheck script failed to execute properly"
+
+- name: Display deployment summary
+  debug:
+    msg: |
+      Forgejo Runner deployed successfully!
+
+      Runner Name: forgejo-runner-box
+      Instance: {{ forgejo_instance_url }}
+      Working Directory: {{ forgejo_runner_dir }}
+      Service: forgejo-runner.service ({{ runner_active.stdout }})
+
+      Healthcheck Monitor: {{ healthcheck_service_name }}
+      Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s
+      Reporting to: {{ healthcheck_push_url | default('', true) | regex_replace('/api/push/.*', '/api/push/***') | default('(nowhere - set healthcheck_push_url)', true) }}

ansible/roles/forgejo_runner/tasks/install.yml

@@ -0,0 +1,27 @@
+---
+- name: Download forgejo-runner binary
+  get_url:
+    url: "{{ forgejo_runner_url }}"
+    dest: "{{ forgejo_runner_bin_path }}"
+    mode: '0755'
+
+# ── 3. Create runner system user ───────────────────────────────────
+- name: Create runner system user
+  user:
+    name: "{{ forgejo_runner_user }}"
+    system: yes
+    shell: /usr/sbin/nologin
+    home: "{{ forgejo_runner_dir }}"
+    create_home: no
+    groups: docker
+    append: yes
+    comment: 'Forgejo Runner'
+
+# ── 4. Create working directory ────────────────────────────────────
+- name: Create forgejo-runner working directory
+  file:
+    path: "{{ forgejo_runner_dir }}"
+    state: directory
+    owner: "{{ forgejo_runner_user }}"
+    group: "{{ forgejo_runner_user }}"
+    mode: '0750'

ansible/roles/forgejo_runner/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+# import_tasks, not include_tasks: these are unconditional phases, and static
+# imports are visible to `--list-tasks`. That matters because the task list is
+# how this refactor was verified against the playbook it replaced.
+- ansible.builtin.import_tasks: prerequisites.yml
+- ansible.builtin.import_tasks: install.yml
+- ansible.builtin.import_tasks: configure.yml
+- ansible.builtin.import_tasks: service.yml
+- ansible.builtin.import_tasks: healthcheck.yml

ansible/roles/forgejo_runner/tasks/prerequisites.yml

@@ -0,0 +1,12 @@
+---
+- name: Check if Docker is installed
+  command: docker --version
+  register: docker_check
+  changed_when: false
+  failed_when: docker_check.rc != 0
+
+- name: Fail if Docker is not available
+  assert:
+    that:
+      - docker_check.rc == 0
+    fail_msg: "Docker is required for forgejo-runner but is not installed"

ansible/roles/forgejo_runner/tasks/service.yml

@@ -0,0 +1,33 @@
+---
+- name: Create forgejo-runner systemd service
+  ansible.builtin.template:
+    src: forgejo-runner.service.j2
+    dest: /etc/systemd/system/forgejo-runner.service
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Reload systemd
+  systemd:
+    daemon_reload: yes
+
+- name: Enable and start forgejo-runner service
+  systemd:
+    name: forgejo-runner
+    enabled: yes
+    state: started
+
+- name: Verify forgejo-runner is active
+  command: systemctl is-active forgejo-runner
+  register: runner_active
+  changed_when: false
+
+# Ungated on purpose. This was previously guarded by `uptime_kuma_enabled`, but
+# it is not a monitoring task — it is the deployment asserting its own success.
+# The deprecation banner swept it up along with the Kuma plumbing, which meant a
+# broken runner deployed "successfully" and silently.
+- name: Assert runner is running
+  assert:
+    that:
+      - runner_active.stdout == "active"
+    fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}"

ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Forgejo Runner
+Documentation=https://forgejo.org/docs/latest/admin/actions/
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+User={{ forgejo_runner_user }}
+Group={{ forgejo_runner_user }}
+WorkingDirectory={{ forgejo_runner_dir }}
+ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }}
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/forgejo_runner/templates/healthcheck.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=Forgejo Runner Healthcheck
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart={{ healthcheck_script_path }}
+Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }}
+Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }}
+User=root
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/forgejo_runner/templates/healthcheck.sh.j2

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Forgejo Runner healthcheck — managed by Ansible (roles/forgejo_runner)
+#
+# Answers "is forgejo-runner healthy" and records it two ways: this log, and the
+# exit code. The exit code is the durable artefact — systemd keeps it, so
+#     systemctl is-failed {{ healthcheck_service_name }}.service
+# answers the question with no monitoring system involved.
+#
+# Reporting is optional and generic: if a push URL is configured it also pings
+# it. Nothing here knows or cares which monitoring product is on the other end.
+
+LOG_FILE="{{ healthcheck_log_file }}"
+# Read from the environment rather than templated in, so the unit file is the
+# only place the token lives and the script is not secret.
+PUSH_URL="${HEALTHCHECK_PUSH_URL:-}"
+PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}"
+
+log_message() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
+}
+
+# Gatus external endpoint: a POST with a bearer token and success=true|false.
+#
+# This used to report ONLY success - it exited before pushing when the runner
+# was down - so a failure was invisible until the heartbeat window expired.
+# Reporting the failure is the whole point of having a check.
+report() {
+    local ok="$1" msg="$2"
+    [ -n "$PUSH_URL" ] || return 0
+    curl -s --max-time 15 --retry 2 -o /dev/null -X POST \
+        -H "Authorization: Bearer ${PUSH_TOKEN}" \
+        "${PUSH_URL}?success=${ok}&error=${msg// /%20}" || true
+}
+
+main() {
+    if ! systemctl is-active --quiet forgejo-runner; then
+        log_message "ERROR: forgejo-runner is not active"
+        report false "forgejo-runner is not active"
+        exit 1
+    fi
+
+    log_message "forgejo-runner is active"
+    report true "active"
+    exit 0
+}
+
+main

ansible/roles/forgejo_runner/templates/healthcheck.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run Forgejo Runner Healthcheck every minute
+Requires={{ healthcheck_service_name }}.service
+
+[Timer]
+OnBootSec=30sec
+OnUnitActiveSec={{ healthcheck_interval_seconds }}sec
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/fulcrum/README.md

@@ -0,0 +1,65 @@
+# `fulcrum`
+
+Deploys [Fulcrum](https://github.com/cculianu/Fulcrum), an Electrum server
+indexing the Bitcoin Knots node, on `fulcrum-box`. The second play in the
+calling playbook publishes its SSL port from the edge host via `socket_proxy`.
+
+Converted from `deploy_fulcrum_playbook.yml` (685 lines) under Plan 6. The
+playbook is now 33 lines.
+
+## The index is the expensive thing
+
+`{{ fulcrum_db_dir }}` is ~192 GB and takes days to rebuild. Nothing in this role
+touches it beyond `state: directory` with the ownership it already has
+(`fulcrum:fulcrum 0755`). Restarting Fulcrum re-opens the database; it does not
+reindex.
+
+## Three things this conversion fixed, all pre-existing
+
+**`bitcoind` pointed at the wrong machine.** The vars file carried
+`bitcoin_rpc_host: "192.168.1.140"` commented "IP of knots_box_local", but `.140`
+is **fulcrum-box itself** — knots-box is `.135`. The DHCP leases had reshuffled.
+The live config had already been hand-corrected to `knots-box`; running the
+playbook would have reverted it and broken indexing. Now addressed by Tailscale
+name, like everything else in this repo.
+
+**The restart handler was inert.** It carried
+`when: uptime_kuma_enabled | default(false)`, so the three tasks that notify it
+(SSL certificate, `fulcrum.conf`, systemd unit) could not restart anything. A
+configuration change would write to disk, report success, and silently never take
+effect. Ungated.
+
+**`db_mem` was about to quadruple.** The role computes a share of RAM; on this
+5931 MB host 75% is 4448 MB, leaving ~1.4 GB for the OS and Fulcrum's non-cache
+memory. The live value had been hand-tuned to 2048. `fulcrum_db_mem_mb_override`
+pins it. Note `set_fact` outranks role defaults, so the *calculation* has to
+honour the override — pinning it in `defaults/` alone is silently ignored.
+
+## The health check timer, and how to read it
+
+The timer is `OnBootSec` + `OnUnitActiveSec` with no `OnCalendar`. That
+combination has a failure mode worth knowing: `OnBootSec` is monotonic and
+elapses once; `OnUnitActiveSec` schedules relative to the **service** last being
+active. If the service does not run in a given boot, there is no reference to
+schedule from and the timer sits `active` and `enabled` doing nothing. That is
+exactly what had happened here — last trigger **2026-02-17**, seven months of no
+health check, with every surface-level indicator green.
+
+Restarting the timer does not supply that reference; running the service does.
+So the role runs the check once after enabling the timer, which is both the fix
+and a smoke test.
+
+**Diagnosing this is easy to get wrong**: `NextElapseUSecRealtime` is always
+empty for a monotonic timer, so it looks broken even when it is fine. Read
+`NextElapseUSecMonotonic`, or just use `systemctl list-timers`.
+
+The timer also no longer carries `Requires=fulcrum.service`. On a timer that
+means "stop watching when the watched thing stops", which is backwards for a
+health check.
+
+## Monitoring: one variable, no product knowledge
+
+The check tests the Electrum TCP port and records the answer in its exit code,
+which systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. To report
+elsewhere set `healthcheck_push_url` to any endpoint accepting an HTTP ping. The
+Uptime Kuma API calls, monitor creation and token handling are gone.

ansible/roles/fulcrum/defaults/main.yml

@@ -0,0 +1,81 @@
+# Fulcrum Configuration Variables
+
+# Version - Pinned to specific release
+fulcrum_version: "2.1.0"  # Fulcrum version to install
+
+# Directories
+fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db  # Database directory (heavy data on special mount)
+fulcrum_config_dir: /etc/fulcrum  # Config file location (standard OS path)
+fulcrum_lib_dir: /var/lib/fulcrum  # Other data files (banner, etc.) on OS disk
+fulcrum_binary_path: /usr/local/bin/Fulcrum
+
+# Network - Bitcoin RPC connection
+# Bitcoin Knots is on a different host (knots_box_local)
+# Using RPC user/password authentication (credentials from group_vars/all/vault.yml)
+# Addressed by Tailscale name, never a LAN IP. This was
+#   bitcoin_rpc_host: "192.168.1.140"  # IP of knots_box_local
+# but .140 is fulcrum-box ITSELF - knots-box is .135. The DHCP leases had
+# reshuffled (the same drift that transposed the inventory), so running this
+# playbook would have pointed Fulcrum at itself and broken indexing. The live
+# config had already been hand-corrected to knots-box; this makes the repo
+# agree with it.
+bitcoin_rpc_host: "knots-box"
+bitcoin_rpc_port: 8332  # Bitcoin Knots RPC port
+# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml
+
+# Network - Fulcrum server
+fulcrum_tcp_port: 50001
+# The edge host's socket-proxy/Caddy play needs this too, and a role default is
+# invisible outside this role. The authoritative value for the live deployment is
+# in host_vars/fulcrum_box_local/main.yml, which outranks this; the value here is the
+# protocol standard, so the role still works standalone.
+fulcrum_ssl_port: 50002
+# Binding address for Fulcrum TCP/SSL server:
+#   - "127.0.0.1" = localhost only (use when Caddy is on the same box)
+#   - "0.0.0.0" = all interfaces (use when Caddy is on a different box)
+#   - Specific IP = bind to specific network interface
+fulcrum_tcp_bind: "0.0.0.0"  # Default: localhost (change to "0.0.0.0" if Caddy is on different box)
+fulcrum_ssl_bind: "0.0.0.0"  # Binding address for SSL port
+# If Caddy is on a different box, set this to the IP address that Caddy will use to connect
+
+# SSL/TLS Configuration
+fulcrum_ssl_enabled: true
+fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt"
+fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key"
+fulcrum_ssl_cert_days: 3650  # 10 years validity for self-signed cert
+
+
+# Performance
+# db_mem will be calculated as 75% of available RAM automatically in playbook
+# db_mem is computed as this share of RAM unless fulcrum_db_mem_mb is set
+# explicitly. On a 5931 MB host 75% is 4448 MB, which leaves ~1.4 GB for the
+# OS and for Fulcrum's non-cache memory; the live config had been hand-tuned
+# down to 2048 and that setting is respected below.
+fulcrum_db_mem_percent: 0.75  # 75% of RAM for database cache
+
+# Configuration options
+fulcrum_anon_logs: true  # Anonymize client IPs and TxIDs in logs
+fulcrum_peering: false  # Disable peering with other Fulcrum servers
+fulcrum_zmq_allow_hashtx: true  # Allow ZMQ hashtx notifications
+
+# Service user
+fulcrum_user: fulcrum
+fulcrum_group: fulcrum
+
+
+# --- Health check -----------------------------------------------------------
+# Checks the Electrum TCP port and records the answer in its exit code, which
+# systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`.
+#
+# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means
+# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping
+# works; nothing here is specific to a monitoring product.
+healthcheck_push_url: ""
+# Bearer token for the Gatus external endpoint. Required whenever a push URL
+# is set: Gatus rejects an unauthenticated push with 401.
+healthcheck_push_token: ""
+
+# Explicit db_mem in MB. When set it wins over fulcrum_db_mem_percent; empty
+# means compute from RAM. Set here because the live host had been hand-tuned to
+# 2048 and a silent jump to 4448 is not something a refactor should do.
+fulcrum_db_mem_mb_override: 2048

ansible/roles/fulcrum/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+# Ungated on purpose. The hand-written handler carried
+#   when: uptime_kuma_enabled | default(false)
+# so it has been inert since the decommissioning: three tasks notify it (the SSL
+# certificate, fulcrum.conf and the systemd unit), and none of them could
+# actually restart Fulcrum. A configuration change therefore applied to disk and
+# silently never took effect — the worst kind of quiet failure, because the
+# playbook reports success and the running service keeps its old settings.
+#
+# Restarting Fulcrum re-opens its database; it does not reindex.
+- name: Restart fulcrum
+  systemd:
+    name: fulcrum
+    state: restarted
+    daemon_reload: yes

ansible/roles/fulcrum/tasks/healthcheck.yml

@@ -0,0 +1,62 @@
+---
+# Everything here answers "is Fulcrum healthy" and records the answer. The
+# Uptime Kuma specifics that used to follow — an embedded Python script creating
+# monitors over the API, a /tmp credentials file, push-URL extraction and a
+# systemd Environment= rewrite — are gone. Where it reports is now one variable,
+# healthcheck_push_url. See the role README.
+- name: Create Fulcrum health check script
+  ansible.builtin.template:
+    src: healthcheck.sh.j2
+    dest: /usr/local/bin/fulcrum-healthcheck-push.sh
+    owner: root
+    group: root
+    mode: '0755'
+    validate: "bash -n %s"
+
+- name: Create systemd service for Fulcrum health check
+  ansible.builtin.template:
+    src: healthcheck.service.j2
+    dest: /etc/systemd/system/fulcrum-healthcheck.service
+    owner: root
+    group: root
+    mode: "0600"
+
+- name: Create systemd timer for Fulcrum health check
+  ansible.builtin.template:
+    src: healthcheck.timer.j2
+    dest: /etc/systemd/system/fulcrum-healthcheck.timer
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Reload systemd daemon for health check
+  systemd:
+    daemon_reload: yes
+
+# state: restarted, not started. The hand-written timer had got itself stuck
+# `active` with no next elapse and had not fired since 2026-02-17; `started` on
+# an already-active timer is a no-op and would have left it stuck. Restarting
+# re-arms it. See the note in healthcheck.timer.j2.
+- name: Enable and restart the Fulcrum health check timer
+  systemd:
+    name: fulcrum-healthcheck.timer
+    enabled: yes
+    state: restarted
+    daemon_reload: yes
+
+# Run the check once, which is both a smoke test and the thing that actually
+# arms the timer.
+#
+# This timer is OnBootSec + OnUnitActiveSec with no OnCalendar. OnBootSec is
+# monotonic and had long since elapsed; OnUnitActiveSec schedules relative to the
+# SERVICE last being active, and the service had not run since 2026-02-17 — so
+# there was no reference to schedule from and the timer sat `active` and
+# `enabled` with NextElapseUSecMonotonic=infinity. Restarting the timer alone
+# does not supply that reference; running the service does.
+#
+# (Diagnosing this is easy to get wrong: NextElapseUSecRealtime is always empty
+# for a monotonic timer, so it looks broken even when it is fine. Read
+# NextElapseUSecMonotonic, or just use `systemctl list-timers`.)
+- name: Run the Fulcrum health check once to arm the timer
+  command: systemctl start fulcrum-healthcheck.service
+  changed_when: false

ansible/roles/fulcrum/tasks/install.yml

@@ -0,0 +1,146 @@
+---
+- name: Calculate db_mem as a share of system RAM
+  set_fact:
+    fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}"
+  when: fulcrum_db_mem_mb_override | string | length == 0
+
+- name: Use the explicit db_mem override
+  set_fact:
+    fulcrum_db_mem_mb: "{{ fulcrum_db_mem_mb_override }}"
+  when: fulcrum_db_mem_mb_override | string | length > 0
+  changed_when: false
+
+- name: Display calculated db_mem value
+  debug:
+    msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)"
+
+- name: Display Fulcrum version to install
+  debug:
+    msg: "Installing Fulcrum version {{ fulcrum_version }}"
+
+- name: Install required packages
+  apt:
+    name:
+      - curl
+      - wget
+      - openssl
+    state: present
+    update_cache: yes
+
+- name: Create fulcrum group
+  group:
+    name: "{{ fulcrum_group }}"
+    system: yes
+    state: present
+
+- name: Create fulcrum user
+  user:
+    name: "{{ fulcrum_user }}"
+    group: "{{ fulcrum_group }}"
+    system: yes
+    shell: /usr/sbin/nologin
+    home: /home/{{ fulcrum_user }}
+    create_home: yes
+    state: present
+
+- name: Create Fulcrum database directory (heavy data on special mount)
+  file:
+    path: "{{ fulcrum_db_dir }}"
+    state: directory
+    owner: "{{ fulcrum_user }}"
+    group: "{{ fulcrum_group }}"
+    mode: '0755'
+
+- name: Create Fulcrum config directory
+  file:
+    path: "{{ fulcrum_config_dir }}"
+    state: directory
+    owner: root
+    group: "{{ fulcrum_group }}"
+    mode: '0755'
+
+- name: Create Fulcrum lib directory (for banner and other data files)
+  file:
+    path: "{{ fulcrum_lib_dir }}"
+    state: directory
+    owner: "{{ fulcrum_user }}"
+    group: "{{ fulcrum_group }}"
+    mode: '0755'
+
+# ===========================================
+# SSL Certificate Generation
+# ===========================================
+- name: Check if SSL certificate already exists
+  stat:
+    path: "{{ fulcrum_ssl_cert_path }}"
+  register: fulcrum_ssl_cert_exists
+  when: fulcrum_ssl_enabled | default(false)
+
+- name: Generate self-signed SSL certificate for Fulcrum
+  command: >
+    openssl req -x509 -newkey rsa:4096
+    -keyout {{ fulcrum_ssl_key_path }}
+    -out {{ fulcrum_ssl_cert_path }}
+    -sha256 -days {{ fulcrum_ssl_cert_days }}
+    -nodes
+    -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local"
+  args:
+    creates: "{{ fulcrum_ssl_cert_path }}"
+  when: fulcrum_ssl_enabled | default(false)
+  notify: Restart fulcrum
+
+- name: Set SSL certificate permissions
+  file:
+    path: "{{ fulcrum_ssl_cert_path }}"
+    owner: "{{ fulcrum_user }}"
+    group: "{{ fulcrum_group }}"
+    mode: '0644'
+  when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false)
+
+- name: Set SSL key permissions
+  file:
+    path: "{{ fulcrum_ssl_key_path }}"
+    owner: "{{ fulcrum_user }}"
+    group: "{{ fulcrum_group }}"
+    mode: '0600'
+  when: fulcrum_ssl_enabled | default(false)
+
+- name: Check if Fulcrum binary already exists
+  stat:
+    path: "{{ fulcrum_binary_path }}"
+  register: fulcrum_binary_exists
+  changed_when: false
+
+- name: Download Fulcrum binary tarball
+  get_url:
+    url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz"
+    dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz"
+    mode: '0644'
+  when: not fulcrum_binary_exists.stat.exists
+
+- name: Extract Fulcrum binary
+  unarchive:
+    src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz"
+    dest: "/tmp"
+    remote_src: yes
+  when: not fulcrum_binary_exists.stat.exists
+
+- name: Install Fulcrum binary
+  copy:
+    src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum"
+    dest: "{{ fulcrum_binary_path }}"
+    owner: root
+    group: root
+    mode: '0755'
+    remote_src: yes
+  when: not fulcrum_binary_exists.stat.exists
+
+- name: Verify Fulcrum binary installation
+  command: "{{ fulcrum_binary_path }} --version"
+  register: fulcrum_version_check
+  changed_when: false
+
+- name: Display Fulcrum version
+  debug:
+    msg: "{{ fulcrum_version_check.stdout_lines }}"
+

ansible/roles/fulcrum/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+# import_tasks, not include_tasks: static imports stay visible to --list-tasks,
+# which is how this conversion was verified against the playbook it replaced.
+- ansible.builtin.import_tasks: install.yml
+- ansible.builtin.import_tasks: service.yml
+- ansible.builtin.import_tasks: healthcheck.yml