diff --git a/.gitignore b/.gitignore index 9ca7b8e..471cd6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,16 @@ # OpenTofu / Terraform .terraform/ .tofu/ -.terraform.lock.hcl -.tofu.lock.hcl terraform.tfstate terraform.tfstate.* crash.log -*.tfvars -*.tfvars.json + +# Provider credentials only. Non-secret infra config (the vms map) is committed +# as *.auto.tfvars, and *.lock.hcl is committed on purpose so provider hashes +# are pinned. +terraform.tfvars +terraform.tfvars.json +*secrets.auto.tfvars venv/* .env diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 0be0162..f77ae48 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -6,7 +6,6 @@ root_domain: contrapeso.xyz # Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the # playbooks are kept deliberately — the check logic is meant to be rewired to # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. -uptime_kuma_enabled: false # age recipient for all backup artefacts age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" @@ -14,3 +13,54 @@ age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr # Public key small-backups-box pulls with # Authorised on each source host for an unprivileged, dedicated user only backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" + + +# ───────────────────────────────────────────────────────────────────────────── +# Subdomains. Global because the edge host proxies for services that live on +# other machines, so no single inventory group covers the readers. Combine with +# root_domain above to build an FQDN. +# +# Moved here from services_config.yml, which 30 plays had to remember to name in +# vars_files: - a file everyone must opt into is a file someone will forget. +# ───────────────────────────────────────────────────────────────────────────── +subdomains: + # Monitoring + gatus: status + + # VPN infrastructure (spacey) + headscale: headscale + + # Core services (vipy) + vaultwarden: vault + forgejo: forgejo + lnbits: wallet + + # Secondary services (vipy) + ntfy_emergency_app: avisame + personal_blog: pablohere + + # Memos (memos-box) + memos: memos + + # Mempool block explorer (mempool-box, proxied via vipy) + mempool: mempool + + # DATUM Gateway dashboard (knots-box, proxied via vipy) + datum_gateway: datum + +# Read by plays across several groups, so global rather than group_vars/. +headscale_namespace: counter-net + +# ───────────────────────────────────────────────────────────────────────────── +# Domains whose registration expiry is monitored (infra/402_public_monitoring). +# +# Registration renewal is a manual act at the registrar, and losing a domain is +# not recoverable in the way losing a host is - so these are checked daily and +# alarm with two weeks of runway. +# +# root_domain is the estate's own domain; the rest are domains we own that are +# served from it or from a host in the inventory. +# ───────────────────────────────────────────────────────────────────────────── +monitored_domains: + - "{{ root_domain }}" + - arbret.com diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index d1f18ec..76eb7bf 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,60 +1,164 @@ $ANSIBLE_VAULT;1.1;AES256 -61356165613635386631393135656434646436303665313031346566323336313138353433316463 -3363323534613064643132663335623238366431393062340a346538396662306537663163623366 -38626166383933616331623231373137306562623637313263333237633661663436666266616433 -3862346438643638650a306634333535653633613534646630386131333236366538333765323333 -38656163303837303732663561373232393132343331663164656262393730326434373731333636 -36613538646431396536363936336562616431656665653965373864633366663836353434626434 -34383932373461333564303439623565383661646365386665393831383463663662356536356236 -36373964666236626465366161636135393734356536633466383262326537343833636630343738 -62353066316131613737373162643363653662656261363465386364323962656537373061373032 -33343763353464383438363438343965653532393831343930393562633630383932653862623637 -38386239353237356631646436356166373961333464396639383538383662326534343339313330 -65373339356364636634616532363832386631323062363530313861336238353261353334306235 -62616338396431316537346638656365356564346666366366343638356261623664393263323937 -30343261363562383332323462336435376664386134646562643836363834313237373631353731 -35653535663864643266313332356635363262363533663232656531373130336539633066376139 -32316665393831663035623962656364363831333563366135636164346335383738363336663566 -63356532643563393939383635386462663561386434323939303431653438653131363538383034 -33623933333464363032656636643033326162626163353633343062633966343332383138363963 -36383831663562616533316436366566323061386535343538393861383462333166343562316633 -30623665623035393537393965626363323132656433313339396233356666346634316332616336 -37363635363330323230373332326565343530653335383437373230366563366237633665626331 -66623336626230663361636439316337393865383035326136653264666438666566646132353036 -38616264313833316536623238633339373466613866626366383835656638623863323838653030 -63303938376164653966356435386333363731656666313234663535666165646233313137343563 -36386437393139656438333262383437656666343831313239323961373637653163643664356565 -62346431393133656530316262303763646165643336396661666431383436323562336137653031 -62626538373839613734396366653065306534636630346338316237616161613037616364356431 -38643965376133616161336633383664326230383435363334353137303162663738313331346238 -39356161616533616134356231323530306338333162343363353531303263636632613036386638 -63633136386232306234323936303563646466313935326631396565383432386130656638616266 -32383334363237336539396665336366643764633131643663643137376438323666326435626461 -64346633636431393137633537306431646564386565303933636434386462346630626537346438 -32623333666133303061646564366366326665363163396262633164323631636337346130303239 -38373936663337356134666132303165393365663763396362623434633737373538653566646134 -62316164396438303532616266313062326666633130656338653139376634306664633031333037 -31636166306565353334633435656233336233363664306264626237623366336161303134353433 -63383739666462623336386537346662633666626466393039653439346436653937633537396436 -35393339383066326630353066623132333034656539363561346462626265363263303535343961 -39316461616630326539613731303039613736393633373338646266323938326162373831346336 -30656130343463366534323030646238313465306266383034623065636665623366333063383736 -38373063393837306462303564643962373334343139626338623935336435643730646532633630 -66363730386636633639346463363365343239373265303738353732653633653437636130636664 -39386365353334653765343335303263363461313965383664326563333734626533376436626530 -63353163323637303730353564383733653365613635353764333266393532653663326533646132 -39313763373735323835626437306435373238653432393936643165663663656665316132653330 -35383038353532656434366336346235363563636264303734633138323963396562306232646236 -38366536306561653937336333373434336164663336613839353439356435333833396363636437 -38323934613735643363656233333037336465336564313966623063376566663030303230323262 -65643564666534326234306164343365383632333061316238623565353538313538396364313337 -35336230313339643736653238386231343661623337306236383665356632366236356335323530 -66373831616239376231636361333430343433303233393066323865663434643433303832373262 -36366534646161323130393931626362626139663139643263366639656531613436313533363130 -65643736363833333939613566663339623964333262643863333030623138633464386238613934 -31323639333234336264313663376465323737353766643839303665313737336534386665363034 -30326439666433373232306136306365343764643434306561343339353132346430646436343362 -32333765616262363930353435616563333736313533653339656231316230346166363335356638 -37333561366339613136306438306130343230663732333862663838396463623661303961336433 -65646332373139303462303633346432366530643130366133363937653739653036366136373434 -343864373734666431326239373866313734 +34383033613438623432656438313763613639313139626432303637383363643339613166626665 +6537663336353661373437303636613364643736663031300a623461353135623537343763383333 +37323731646262613436646231656263396532616132316237353762376363383062646132303435 +3435316666316235660a323163613932616661303935336633656332353766393536393736356539 +61396164386266303464373262373862363361363365363235383363613335663933313062663565 +39613466353161356462656539613335613536363532333431393935303430373435373737363537 +62383266323339373062313134316464303263313830646161333530383736336637333865373764 +31313165396366653763343466643561383236643836616662383439666165343139326232326364 +32633133373863616163343365656231353939366435346534383462396134653064663566343566 +66343464373865383864313264646635316233336133346365396634626261613561373965643339 +30353831393037316334313632376537393361336366326561343832373531303139323937613531 +34386164303430316162363536386137316330336638303365393462386465633862663932376563 +34323862636435306563333065383134343362333733623066656439613339376666353331373665 +32353938663364313036373237323062393430306661633732356434333232376338313835663330 +32306536393666323464663238306562613936376236613935616664653865373330626365626330 +64613066653632646638306338393331376430383165316363323437373362613866313962663432 +62356663636630356239613831353066303530363966396135376134633936313530643262353965 +38383330613962313935366633353635363638303937316362303866343433353437363334396431 +32393134626133353564643233616632376633303934653065613262353436333533376137303862 +35303861383833326537376433336634633730373833313036643365373431666438653033653930 +33346230396661636333613031353133353631306331666362386366633062666534303539316238 +39366634656333646634643032393738333330636230663366623265666533356534623465316665 +35336535326237343231653661613736626239316262386363666264613636326332366232333964 +38353635616662663133376534306366643037613732343233373336653166316330643638353539 +35353532643534613864333839393365313439343663333337373639633832383630316164306132 +63636162373034363839656638353534613733663737626432636436373164303239316436313566 +66623466396663353136323565643262383865323830616232373466373431373261306537646463 +61353966623062356438626635663461396165343366393132393965383734633632666335313030 +37333631373239303231396264393031373162373936303462363736313538343562343737356332 +63663762633032356130303964386664366137373432353533386566613037613163383963623732 +32343765376535343630656633333362393765666131653632613361363163383038323138333537 +37373333316466316562613439356365316163336366623435666166653835616531653563343664 +30303464663838646333373837363961356465613234636138633162366636633030383337643131 +62636236663066316630396132333934353139383465363034363033656161383663656462313939 +62353738396132343734343363636164323163616237613861643862303433633934656165343263 +35313737373333386335663731643664393630653839396464376639306231653934386336353630 +61393735336662663435613431386361373561363531643232303831363163636139653538393931 +66313965646562616237383439343637623835303065333730613865666638383131616261346463 +31396464306537613135653830653138323539393731626264326335336432666333333735646534 +30616439393464326631633635363466303336613135346231626232313361303666323661616638 +64363937343231363536353034363966326132333734386638613737636130646232363666643633 +61353563636466356630346232613761636432303430336461663434333636623962643764336639 +35303565613861323431393036303534303061666437366538306236373930313439353330313632 +61363363303562316664303065613863613339626632643931386438326330373938613762303334 +33316430363931393262373661623137633835656136313235613666613932313236343966306331 +36343031303632323432366337633637336335343638393564313738386162386164613339343662 +62383939663265373265613932633265626263623939303638383838646531343433393864363235 +37313761396436623833376435393537376136373162393465343764326533666533393061643965 +61306365346562333737313962633764313232323161623861336566343735353737633539656162 +34323939306661373662663331653333346264323930663636633134666532623438323537383165 +66343033363962633766303331623866306462623235373838616565653066646132633034363535 +31613462356361613535383963643662373162363363653334393937313039666266366363653537 +64356166396139393262313565303731346534646462333638316661383139346362333364613466 +38303561316263636136303431393366343936653161336238613439366266346136636465303337 +61373634376464333037303862616335623031656133636165636264386265643261373735663266 +39396265366138363430343035356337303438346165366361316230656239326633653537626339 +39613334316563376133343336343563363564653237633764393532336437303334363830396635 +31333963666462313830333163646465356337643263636462356363613266323630393866626535 +35336436663063333364353663616564626535396133386461326433323936323161613633353539 +66323935626433353162666335613839346561343264323763303034613366663233613037343538 +66376236373462383430313030373235373337653338333033306431616562353435656433356365 +37653063363435366532306334373438306632306435653334396630343863623938666335656164 +37386634366430383762316435396237313362313965353634613266616532633465646464313662 +35303433363236393465396161346431306231666432666531626136623966663864376663373562 +37653239633732346536323265643434616136326561666330633435383264333937353366653736 +61353430633363323263323163646639663330663632343038623964396433626337353232373334 +32343938356539303938666137323831646562393233323733336464316165623232326132316362 +61313631626234336164656533616231393663363364623130313439623466396231383537656562 +35393163323131383838356130643863633736613635306635306135666261623731613239303962 +61373432336638353339646665653561653738353731373032366361313562306466373365653333 +35306236336331333961613864346165613763383266613236343066623432303766333434333331 +66376263363132616132366165393235626336323665373135653462653533376138346632393363 +35396165383966326635333134663138366136336430393532373935373264616530613762386430 +37643862316232613261343430616438333831623835663239656531356666613032653837373531 +30386232643537306530323936386561646265616339376265353763663833623539653831656634 +63626564633964343663656435636635656562666236386639353332356566316631643431353330 +32643238623932643732353332633535643738663066383830376432346463393464333236386163 +64376165333266323166613564313061643832656239333165393035343836376434626162643061 +31373861393534636236383765313834356638613332626666653762343537363339383939666263 +61366230623930643534353565316165656132396664393138326139613536343738396330653735 +37623463343834626133313231393130646134363331393238663065643930383237326439343831 +32393536613361616637373761393361303936363863333838666633343063386235316237323531 +62633362316436623761616366383336613765373362393161303765613464643632666462343565 +63616430396164346263666436336463623638313130656231643131336663336565616135373433 +39326533643733333564373434333738343634383033353036646534646632383730613161393130 +62653836376435663435353739373536366466646332653030653332353761626466353631663163 +38613734383634663662376464653733306564313761336566336336656366653536643834653631 +35333963306562393234623039613338643266333762333236366331333431343830393864623830 +34323439386133613035363861623332663933313734313637623739613163663730316134303434 +35363239363836666463366334356435666632396132306163303037663430383535393934323537 +30353266373864346662333035393761646632313032383738346439343232663666343833333565 +35363564623038376666363436663830356165323234653230636264303066666230643632356237 +66633734346633366533623966393462313438333965623033343337323135353730613561363236 +30636333613034626162393137323762306432663761393766303163393536656338366664643432 +34636630323965396166356538623165663335333739323034333237306531393734306138343436 +36613937373932633234303336373530663533646635396664343935326334333239646534343336 +35396666353938326337646431353965653836323731313861613031356561376262363965623934 +65336164623134366365303062393161353965363937646363396564313138323938656330333964 +64663862363766613630333531623463616331363533343962653164376665303463336134346662 +66376238663632663932656230323638323135653666363736643065336330636236323064386361 +37646631303138616565383665663634333338613336346438366333636165366635313337373635 +62656236653961393061373262326534393339363236353336613339656462336661663739396261 +35363636653237316436656662633462306333313336313865333037636661383731336466666536 +36646537653431666261353232653233653930346538383438633466346636656432383963333765 +62346363633265306665366630383831303434336139343837616361666539656262633735363064 +30376138386531336430613563366666333964663231363465326230316233323532366632663930 +63653164373931353861653833326130643365373530663039363166366331303134366336613438 +63633336363539636265396331653630633162316366316130326531363463616464376635376538 +32376638353337343639633434613663666437313564333264383766623561393933333561663235 +37666163626538353736383435373530663438363636643665626266316338336532623235353337 +61653762383630616266353438386137653137616532633066623165383663646437343565366237 +33333463613539346136613135386466656135616136663036313661323738363066633038656139 +38353230313861666463383737303038613731316334333031643438343463666164633536313031 +39613663376436643165333362636462303230656336333234363336633262336363643736336231 +66316630613762383832613836333263303038306235663662323162633932653733626130356665 +65316630346534356261373961353130393334396365376237353439326261336339393336303030 +63323439666439313831626639323735653730356565643938376564316337613333616664643865 +30633530383865663731383635366566613565393632336337306334663937396135303539363536 +65656331386532306438383031313164626237393762303636663831666239336637666465633630 +63383961336333663537666561376532376631343132363638633364616531306363623432396332 +31343864626335613131653762363338393738343531626330663663376531336230656365363661 +64363530333965386439376161313534333435393966656231613161633034303964653564363436 +63396131623562303962383137666439373132633330363237353031306561306362666561376435 +66383535333233663736363762323662363264393365643135323834366565653932313863353437 +38363536323739336165356561613335653464363637636234313637383066633865353239393961 +31613532373361383062393238303430353662383930373764373237383035633364323661623661 +62653661623234366666616532663563316234663137616338393036333937633963323635366165 +61613032363531666338353132346365323661626265626461396362646338373536306362303338 +31646564343434633433663039366533656665643235336136333134366231376336343465336539 +37363139346239666232313366613432646265626564313838346364306133306434313337626562 +32346430316532623037633838303363346130623636386134313566383333613565316135373264 +38393261313435323433326130303032393538333963646430646366653363613830613534353832 +32643434613964613239363966376138376661636335373130356430336631333461623734383535 +35356461326136636234663738343738303064386536336462303632303461303733333666383036 +38383563393739343134363038623638393766316533373163336439336238613734656132623163 +32626564666430663339316163393934323133306238353562323866633738363737643937376435 +66306430623334643766623564363239346361393666663766306637313265313833396435626234 +35366239616132623863343365663833363934316362636638303536656631633364646235366566 +39386637613735373761643339396132323031323438316633363464336636316534643435323861 +62663732366430646538626663313035616235643537643234356434323635373962343633336266 +30323031306438656537626430356231393466313334633934623632663661306361313330633339 +33396234386536336331616161363132343765306562303932313963623037366633383765663134 +62636466636433666530363930633765626531613539363832313361666565323635326334326533 +36336439363836663033643262336139613437623633616138656564393032393263393132303031 +31626336663039633362366663306662333432643261613464303939326562653562346239383261 +37373966383635626433363831653936333964626262363839383936356634633233336365623765 +62313031353538636464303234383865613932323164386336316362323731303263346637346534 +32353734343130343364646161643431353230336364366330363261326334613936633234646264 +39623163346663613431323630303034383761663835373565663166366239303130386139396236 +62393565353761636636366462353038366664663430616332326465373364396264323064666536 +61653035633562316634336263636332363733663666626131383161656535383133613939623034 +66363632613733306336323165336534346337386163656631343332323636353539353737316131 +34616336343865656565646338653037333838663736333330376330663834373138363739633064 +61646439636365323838376131663266636235333062616532613936616339633661303634323531 +63356335386534353639323566376134373565353137333134363761666532366431633634333430 +39326662633765626230326130386333393463323433366162363432613234336634663439313339 +33653737333830336264343839396563316462613032376335356634383436353962613636333866 +32333032623635323430356636623138366635386533646133626164316438393937626462313239 +31636635363064363763396566306234363965346438653738333961623435303233396634643763 +65343763386464366636383734336335646464306262623363303934393734333666636635356365 +65353931303862343765343665373139376530656263356365366333323135383838383030613166 +30303963383833643937636539656164336537383337346436303534303035313935323633663765 +333464336235373230316337333632326138 diff --git a/ansible/host_vars/forgejo_runner_local/main.yml b/ansible/host_vars/forgejo_runner_local/main.yml new file mode 100644 index 0000000..6e852ce --- /dev/null +++ b/ansible/host_vars/forgejo_runner_local/main.yml @@ -0,0 +1,12 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - forgejo-runner diff --git a/ansible/host_vars/fulcrum_box_local/main.yml b/ansible/host_vars/fulcrum_box_local/main.yml new file mode 100644 index 0000000..aa189b0 --- /dev/null +++ b/ansible/host_vars/fulcrum_box_local/main.yml @@ -0,0 +1,17 @@ +# fulcrum-box: the Electrum server. +# +# Read by the fulcrum role here and by the socket-proxy play on the edge host, +# which publishes the port. See host_vars/knots_box_local/main.yml for why this +# lives in host_vars rather than in the role's defaults. +fulcrum_ssl_port: 50002 + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - fulcrum diff --git a/ansible/host_vars/knots_box_local/main.yml b/ansible/host_vars/knots_box_local/main.yml new file mode 100644 index 0000000..3866541 --- /dev/null +++ b/ansible/host_vars/knots_box_local/main.yml @@ -0,0 +1,26 @@ +# knots-box: Bitcoin Knots and the DATUM Gateway. +# +# These ports are read twice: by the role that deploys the service here, and by +# the socket-proxy / Caddy plays that run on the EDGE host and publish them. +# A role default is invisible to that second play, which is why these live in +# host_vars rather than roles//defaults/ - the edge play reads them as +# hostvars['knots_box_local']., and the role picks them up automatically +# because host_vars outranks role defaults. +# +# They used to live in services_config.yml, a file 30 plays had to remember to +# name in vars_files: and that four role defaults silently depended on. +bitcoin_p2p_port: 8333 +datum_gateway_api_port: 7152 +datum_gateway_stratum_port: 23334 + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - bitcoind + - datum-gateway diff --git a/ansible/host_vars/memos_box_local/main.yml b/ansible/host_vars/memos_box_local/main.yml new file mode 100644 index 0000000..f1fc19b --- /dev/null +++ b/ansible/host_vars/memos_box_local/main.yml @@ -0,0 +1,12 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - memos diff --git a/ansible/host_vars/mempool_box_local/main.yml b/ansible/host_vars/mempool_box_local/main.yml new file mode 100644 index 0000000..f46a1a5 --- /dev/null +++ b/ansible/host_vars/mempool_box_local/main.yml @@ -0,0 +1,6 @@ +# mempool-box: the Mempool block explorer. +# +# Read by the mempool role here and by the Caddy play on the edge host, which +# proxies to it. See host_vars/knots_box_local/main.yml for why this lives in +# host_vars rather than in the role's defaults. +mempool_frontend_port: 8080 diff --git a/ansible/host_vars/monitoring/main.yml b/ansible/host_vars/monitoring/main.yml new file mode 100644 index 0000000..2ef805d --- /dev/null +++ b/ansible/host_vars/monitoring/main.yml @@ -0,0 +1,12 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - caddy diff --git a/ansible/host_vars/nodito/main.yml b/ansible/host_vars/nodito/main.yml index c0002f3..c8176af 100644 --- a/ansible/host_vars/nodito/main.yml +++ b/ansible/host_vars/nodito/main.yml @@ -14,7 +14,12 @@ systemd_service_name: nodito-cpu-temp-monitor # ZFS Pool Configuration zfs_pool_name: "proxmox-tank-1" -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the +# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently +# replaced and the repo never caught up. Pool creation is guarded by +# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been +# wrong on any disaster-recovery run. +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" @@ -26,3 +31,15 @@ ups_port: auto ups_user: counterweight ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - nut-server + - nut-monitor diff --git a/ansible/host_vars/nodito/vault.yml b/ansible/host_vars/nodito/vault.yml new file mode 100644 index 0000000..e64085e --- /dev/null +++ b/ansible/host_vars/nodito/vault.yml @@ -0,0 +1,11 @@ +$ANSIBLE_VAULT;1.1;AES256 +30333035323663393939343061323234336164396465623665346165393534646366333332376463 +3364373463333664363334373964323838336531353364310a636636373539623464336630666164 +61376532616339376562373238383436306664313564663266303534346461666466383965323538 +3163313239626663310a613033336332653165333537313366636361663036383031376561613761 +31313563373062333033323037653939663762343161656264633436343361663737626366663732 +39366666626338323436383134646263643538333564313566346336323563663534653161396136 +39333565393538366238643563323630346166643461643063393631643665363566623631373762 +36643866646637306231653837363838656163613766636265383139333838396535626335343163 +32613935353330636263616333666230323436663935326133636362343836323535623237646235 +6266316363366335323162663039366137633865396237373632 diff --git a/ansible/host_vars/spacey/main.yml b/ansible/host_vars/spacey/main.yml new file mode 100644 index 0000000..a50d116 --- /dev/null +++ b/ansible/host_vars/spacey/main.yml @@ -0,0 +1,13 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - headscale + - caddy diff --git a/ansible/host_vars/vipy/main.yml b/ansible/host_vars/vipy/main.yml new file mode 100644 index 0000000..3b077df --- /dev/null +++ b/ansible/host_vars/vipy/main.yml @@ -0,0 +1,15 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - forgejo + - lnbits + - caddy + - phoenixd diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index c6eed18..0e2c914 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,7 +1,5 @@ - name: Secure Debian hosts: managed - vars_files: - - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index 9f37c70..309db56 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,7 +1,5 @@ - name: Secure Debian hosts: managed - vars_files: - - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/400_host_monitoring.yml b/ansible/infra/400_host_monitoring.yml new file mode 100644 index 0000000..3f6b119 --- /dev/null +++ b/ansible/infra/400_host_monitoring.yml @@ -0,0 +1,216 @@ +--- +# Host-level monitoring for the whole estate, reported to Gatus. +# +# Every check here PUSHES. Gatus never reaches out, which matters because nodito +# and its VMs sit behind NAT, and because four of the five checks are internal +# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool +# health and UPS mains status cannot be observed from outside the machine. +# +# Liveness is a push too, and that is a choice rather than a limitation. A +# heartbeat proves the host is running AND can reach the internet; an ICMP probe +# from one vantage point only proves it answers pings from there. And because +# Gatus alerts when a heartbeat window expires, a check that stops running +# raises the alarm by itself - a dead timer looks exactly like a dead host, +# which is the correct reading. +# +# Each host has ONE bearer token, shared across its own checks: a token can only +# write results for that host's endpoints, so a compromised host can lie about +# itself, which it could do anyway. +# +# The push URL must use Gatus's own key format (config/key/key.go): +# key = sanitize(group) + "_" + sanitize(name) +# where sanitize lowercases and replaces / _ . , space # + & with "-". So +# knots_box_local becomes knots-box-local in the URL but stays readable in the +# name. host_key below is the Jinja equivalent; do not hand-write these. + +# ───────────────────────────────────────────────────────────────────────────── +# Register everything with Gatus. +# +# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the +# host plays below take minutes, so every endpoint exists before its first push +# arrives. Registering afterwards would 404 every first report. +# +# Heartbeat windows are several times the check interval, so one missed run - a +# slow apt run, a reboot - does not raise an alarm, but a check that has +# genuinely stopped does. +# ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the host checks with Gatus + hosts: observability + become: yes + vars: + monitored: "{{ groups['managed'] | sort }}" + + tasks: + - name: Build the liveness endpoint list + ansible.builtin.set_fact: + liveness_endpoints: "{{ liveness_endpoints | default([]) + [{ + 'name': item, + 'group': 'liveness', + 'token': gatus_push_tokens[item], + 'heartbeat': '11m'}] }}" + loop: "{{ monitored }}" + + - name: Build the disk endpoint list + ansible.builtin.set_fact: + disk_endpoints: "{{ disk_endpoints | default([]) + [{ + 'name': item, + 'group': 'disk', + 'token': gatus_push_tokens[item], + 'heartbeat': '7h'}] }}" + loop: "{{ monitored }}" + + - name: Register liveness endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: liveness + gatus_endpoint_external: "{{ liveness_endpoints }}" + + - name: Register disk endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: disk + gatus_endpoint_external: "{{ disk_endpoints }}" + + - name: Register the hypervisor endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: hypervisor + gatus_endpoint_external: + - name: cpu + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "11m" + - name: zfs + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "7h" + - name: ups + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "11m" + +- name: Deploy host liveness and disk checks + hosts: managed + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" + host_token: "{{ gatus_push_tokens[inventory_hostname] }}" + + tasks: + - name: Is the host up? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: liveness + healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}" + healthcheck_check: liveness + healthcheck_interval: "5min" + healthcheck_boot_delay: "1min" + healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is the disk packed? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: disk-usage + healthcheck_description: "Disk usage for {{ inventory_hostname }}" + healthcheck_check: disk-usage + # Every 6h rather than daily. Disk usage itself moves slowly, but the + # heartbeat can only be as tight as the push frequency - a daily push + # forces a >24h window, and a stuck check then hides for a day and a + # half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the + # hosts so twelve boxes do not all report in the same second. + healthcheck_on_calendar: "*-*-* 00/6:00:00" + healthcheck_randomized_delay: "900" + healthcheck_boot_delay: "5min" + healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external" + healthcheck_push_token: "{{ host_token }}" + +- name: Deploy the hypervisor-only checks + hosts: hypervisor + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_token: "{{ gatus_push_tokens[inventory_hostname] }}" + + tasks: + - name: Is the CPU hot? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: cpu-temp + healthcheck_description: "CPU temperature for {{ inventory_hostname }}" + healthcheck_check: cpu-temp + healthcheck_packages: [curl, lm-sensors] + healthcheck_interval: "5min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is ZFS broken? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: zfs-health + healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}" + healthcheck_check: zfs-health + healthcheck_packages: [curl, jq] + healthcheck_zfs_pool: "{{ zfs_pool_name }}" + healthcheck_on_calendar: "*-*-* 00/6:20:00" + healthcheck_boot_delay: "10min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is the UPS online? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: ups-status + healthcheck_description: "UPS mains status for {{ ups_name }}" + healthcheck_check: ups-status + healthcheck_ups_name: "{{ ups_name }}" + healthcheck_interval: "5min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external" + healthcheck_push_token: "{{ host_token }}" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml new file mode 100644 index 0000000..ab564a1 --- /dev/null +++ b/ansible/infra/401_service_monitoring.yml @@ -0,0 +1,107 @@ +--- +# Is each systemd-deployed service actually running? +# +# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before +# it alarms, so a reboot or a slow check does not page anyone, but a host that +# stops reporting does. +# +# This closes the gap that let a real bug run unnoticed: a backup script left +# forgejo, lnbits, headscale and memos stopped, and NOTHING caught it. The dumps +# exited 0, the artefacts were correct, the deploy said failed=0, and liveness +# only proves the HOST is up - not that anything on it is serving. +# +# One endpoint PER UNIT, not per host. A host running four services needs four +# endpoints, or a single red light says "something on vipy is down" without +# saying which - and that is the question you actually have at 3am. But only ONE +# timer per host: the check iterates that host's units and pushes a result for +# each, the same way check-backups.sh reports per source. Four units on vipy +# would otherwise mean four scripts, four services and four timers. +# +# Which units each host runs is in host_vars//main.yml as +# monitored_services, because "what runs here" is a property of the machine. +# +# Keys are host-qualified because unit names collide - caddy runs on four +# machines. Gatus computes sanitize(group)_sanitize(name), so group "services" +# and name "vipy/caddy" give services_vipy-caddy. + +# ───────────────────────────────────────────────────────────────────────────── +# Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the +# host play above takes minutes, so every endpoint exists before its first push. +# ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the service checks with Gatus + hosts: observability + become: yes + + tasks: + # Two plain steps rather than one clever expression: first collect which + # units each host declares, then flatten that into endpoints. + - name: Collect the units each host declares + ansible.builtin.set_fact: + host_units: "{{ host_units | default([]) + [{'host': item, 'units': hostvars[item].monitored_services}] }}" + loop: "{{ groups['managed'] | sort }}" + when: hostvars[item].monitored_services | default([]) | length > 0 + + - name: Build one endpoint per unit + ansible.builtin.set_fact: + service_endpoints: "{{ service_endpoints | default([]) + [{ + 'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1, + 'group': 'services', + 'token': gatus_push_tokens[item.0.host], + 'heartbeat': '11m'}] }}" + loop: "{{ host_units | subelements('units') }}" + + - name: Register the service endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: services + gatus_endpoint_external: "{{ service_endpoints }}" + +- name: Monitor systemd services on every host that has them + hosts: managed + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" + + tasks: + - name: Is every deployed service running? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: service-health + healthcheck_description: "systemd services on {{ inventory_hostname }}" + healthcheck_check: systemd-units + healthcheck_units: "{{ monitored_services }}" + healthcheck_units_key_prefix: "services_{{ host_key }}" + healthcheck_interval: "5min" + healthcheck_boot_delay: "2min" + # The per-unit results go to keys under this collection; the role's own + # single-result push is unused here, so only the base is set. + healthcheck_push_base: "{{ gatus_api }}" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + when: monitored_services | default([]) | length > 0 diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml new file mode 100644 index 0000000..d924468 --- /dev/null +++ b/ansible/infra/402_public_monitoring.yml @@ -0,0 +1,141 @@ +--- +# Domain expiry, DNS correctness, and public endpoint reachability. +# +# These are the first checks in the estate that PULL rather than push, and that +# is the right way round for them: all three are about how the outside world +# sees us, so they must be measured from outside. Gatus polls from the +# observability host and needs nothing installed anywhere else - there is no +# script, no timer and no token, because nothing is reporting in. +# +# That also means these have no heartbeat. A heartbeat answers "did the thing +# that was supposed to report in do so"; when Gatus does the checking itself, +# failure is immediate and self-evident. + +- name: Register the public-facing checks with Gatus + hosts: observability + become: yes + + vars: + # Expected A records, derived from inventory rather than written down again. + # The estate's recurring bug is an address recorded in a second place and + # then left behind when the machine moved, so the check asserts against + # ansible_host - if a box is renumbered, inventory is the one edit. + dns_records: + - {sub: "{{ subdomains.gatus }}", host: monitoring} + - {sub: "{{ subdomains.headscale }}", host: spacey} + - {sub: "{{ subdomains.vaultwarden }}", host: vipy} + - {sub: "{{ subdomains.forgejo }}", host: vipy} + - {sub: "{{ subdomains.lnbits }}", host: vipy} + - {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy} + - {sub: "{{ subdomains.personal_blog }}", host: vipy} + - {sub: "{{ subdomains.memos }}", host: vipy} + - {sub: "{{ subdomains.mempool }}", host: vipy} + - {sub: "{{ subdomains.datum_gateway }}", host: vipy} + + # A public resolver on purpose: this must test what the internet sees, not + # what a local cache or the tailnet's MagicDNS happens to answer. + dns_resolver: "1.1.1.1" + + # Expected status per site, checked live before being written down. + # 401 is the CORRECT answer for the two behind basic auth - asserting 200 + # there would go green precisely when the auth broke. + public_sites: + - {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401} + - {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200} + - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200} + - {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200} + - {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200} + - {name: avisame, sub: "{{ subdomains.ntfy_emergency_app }}", path: "/", status: 200} + - {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200} + - {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200} + - {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200} + - {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401} + + # Ports published from the edge host by socket_proxy. + public_tcp: + - {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"} + - {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"} + - {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"} + + tasks: + # ── Domain expiry ──────────────────────────────────────────────────────── + # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the + # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole + # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so + # whatever the apex serves - a real site, or the registrar's parking page - + # is irrelevant. + # + # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway + # because it uses a free whois service that must not be hammered. + # 336h = 14 days of runway, because renewal is a manual act at the registrar. + - name: Build the domain endpoints + ansible.builtin.set_fact: + domain_endpoints: "{{ domain_endpoints | default([]) + [{ + 'name': item, + 'group': 'domain', + 'url': 'https://' ~ item, + 'interval': '24h', + 'conditions': ['[DOMAIN_EXPIRATION] > 336h'], + 'alerts': [{'type': 'signal', 'failure-threshold': 1, + 'success-threshold': 1, 'send-on-resolved': true, + 'minimum-reminder-interval': '168h'}]}] }}" + loop: "{{ monitored_domains }}" + + # ── DNS ────────────────────────────────────────────────────────────────── + # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The + # domain check stays at 24h because it does a WHOIS/RDAP lookup against a + # free service. Alert on the first failure - at a 6h interval, waiting for + # three would be nearly a day. + - name: Build the DNS endpoints + ansible.builtin.set_fact: + dns_endpoints: "{{ dns_endpoints | default([]) + [{ + 'name': item.sub ~ '.' ~ root_domain, + 'group': 'dns', + 'url': dns_resolver, + 'interval': '6h', + 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, + 'conditions': ['[DNS_RCODE] == NOERROR', + '[BODY] == ' ~ hostvars[item.host].ansible_host], + 'alerts': [{'type': 'signal', 'failure-threshold': 1, + 'success-threshold': 1, 'send-on-resolved': true, + 'minimum-reminder-interval': '24h'}]}] }}" + loop: "{{ dns_records }}" + + # ── Public HTTP ────────────────────────────────────────────────────────── + # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has + # no heartbeat window, so unlike the push checks the tolerance has to live + # in the threshold - and one failed poll of a public site is usually a blip. + - name: Build the public HTTP endpoints + ansible.builtin.set_fact: + http_endpoints: "{{ http_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'public', + 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, + 'interval': '5m', + 'conditions': ['[STATUS] == ' ~ item.status, + '[CERTIFICATE_EXPIRATION] > 168h'], + 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'success-threshold': 2, 'send-on-resolved': true, + 'minimum-reminder-interval': '6h'}]}] }}" + loop: "{{ public_sites }}" + + # ── Public TCP ─────────────────────────────────────────────────────────── + - name: Build the public TCP endpoints + ansible.builtin.set_fact: + tcp_endpoints: "{{ tcp_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'public', + 'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port, + 'interval': '5m', + 'conditions': ['[CONNECTED] == true'], + 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'success-threshold': 2, 'send-on-resolved': true, + 'minimum-reminder-interval': '6h'}]}] }}" + loop: "{{ public_tcp }}" + + - name: Register the public-facing endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: public + gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}" diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml new file mode 100644 index 0000000..c92b767 --- /dev/null +++ b/ansible/infra/403_service_probe_registration.yml @@ -0,0 +1,71 @@ +--- +# The per-service health probes. +# +# These are NOT the same thing as the systemd checks in infra/401. Those answer +# "is the unit running"; these answer "does the service actually respond" - an +# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch +# from the Mempool backend. A process can be perfectly `active` and useless, +# which is precisely the gap these close. +# +# The checks themselves live in each service's own role, deployed by that +# service's playbook. This play only registers where they report, because the +# endpoints must exist in Gatus before the first push arrives. +# +# They used to push to Uptime Kuma. The scripts now POST with a bearer token +# instead of GETting ?status=up, and each host uses its own token. + +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the per-service probes with Gatus + hosts: observability + become: yes + + vars: + probes: + - {name: bitcoin-knots, host: knots_box_local} + - {name: datum-gateway, host: knots_box_local} + - {name: fulcrum, host: fulcrum_box_local} + - {name: phoenixd, host: vipy} + - {name: forgejo-runner, host: forgejo_runner_local} + - {name: mempool-mariadb, host: mempool_box_local} + - {name: mempool-backend, host: mempool_box_local} + - {name: mempool-frontend, host: mempool_box_local} + + tasks: + - name: Build the probe endpoint list + ansible.builtin.set_fact: + probe_endpoints: "{{ probe_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'probe', + 'token': gatus_push_tokens[item.host], + 'heartbeat': '11m'}] }}" + loop: "{{ probes }}" + + - name: Register the probe endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: probes + gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/infra/409_remove_legacy_monitoring.yml b/ansible/infra/409_remove_legacy_monitoring.yml new file mode 100644 index 0000000..8f57e46 --- /dev/null +++ b/ansible/infra/409_remove_legacy_monitoring.yml @@ -0,0 +1,109 @@ +--- +# Remove the Uptime-Kuma-era monitoring that 400/401/402 replaced. +# +# Deleting the playbooks that installed these is NOT enough: the units are on +# the hosts, enabled, and keep firing regardless of what the repo says. Two of +# them still push to https://uptime.contrapeso.xyz every 15 minutes. A playbook +# that is deleted without a cleanup leaves its output running forever, with +# nothing in the repo left to explain it. +# +# What replaced what, all verified against the deployed scripts before removal: +# +# disk-usage-monitor -> disk-usage-healthcheck (infra/400) +# The old one checked ONLY "/" at 80%. The replacement walks every real +# filesystem, excluding tmpfs/devtmpfs/squashfs/overlay, at 85%. Strictly +# more coverage, so nothing is lost. +# +# system-healthcheck -> liveness-healthcheck (infra/400) +# The old script computed uptime and pushed. That is exactly a liveness +# heartbeat and nothing more. +# +# nodito-cpu-temp-monitor -> cpu-temp-healthcheck (infra/400) +# zfs-health-monitor -> zfs-health-healthcheck (infra/400) +# The ZFS check logic was ported verbatim - same five conditions - so only +# the reporting transport changed. +# +# NOT removed, because they are not monitoring: +# zfs-monthly-scrub.{timer,service} the actual scrub (infra/nodito/32) +# pull-backups, check-backups the backup machinery (playbooks/backups) +# +# This play is idempotent and kept permanently rather than run once and deleted: +# on a host that never had these it does nothing, and it guarantees a rebuilt or +# restored machine cannot quietly bring them back. + +- name: Remove the legacy Uptime Kuma monitoring units + hosts: managed + become: yes + + vars: + legacy_units: + - disk-usage-monitor + - system-healthcheck + - nodito-cpu-temp-monitor + - zfs-health-monitor + legacy_dirs: + - /opt/disk-monitoring + - /opt/system-healthcheck + - /opt/nodito-monitoring + - /opt/zfs-monitoring + + tasks: + - name: Find which legacy units exist here + ansible.builtin.stat: + path: "/etc/systemd/system/{{ item.0 }}.{{ item.1 }}" + register: legacy_unit_files + loop: "{{ legacy_units | product(['timer', 'service']) | list }}" + + # Stop and disable BEFORE deleting the unit file: systemd cannot disable a + # unit whose file has already gone, which would leave a dangling symlink in + # multi-user.target.wants and a warning on every daemon-reload. + - name: Stop and disable the legacy units + ansible.builtin.systemd: + name: "{{ item.item.0 }}.{{ item.item.1 }}" + state: stopped + enabled: no + loop: "{{ legacy_unit_files.results }}" + loop_control: + label: "{{ item.item.0 }}.{{ item.item.1 }}" + when: item.stat.exists + failed_when: false + + - name: Remove the legacy unit files + ansible.builtin.file: + path: "/etc/systemd/system/{{ item.item.0 }}.{{ item.item.1 }}" + state: absent + loop: "{{ legacy_unit_files.results }}" + loop_control: + label: "{{ item.item.0 }}.{{ item.item.1 }}" + when: item.stat.exists + + - name: Reload systemd + ansible.builtin.systemd: + daemon_reload: yes + + - name: Remove the legacy monitoring scripts and their logs + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ legacy_dirs }}" + + # An orphan predating all of this: mode 0644, not executable, referenced by + # no unit and no cron entry, pushing to a Kuma monitor. Superseded by + # ups-status-healthcheck. + - name: Remove the orphaned hand-written UPS heartbeat + ansible.builtin.file: + path: /usr/local/bin/ups-heartbeat.sh + state: absent + + - name: Confirm nothing still pushes to Uptime Kuma + ansible.builtin.shell: >- + grep -rl "uptime.contrapeso.xyz" /etc/systemd/system /usr/local/bin /opt 2>/dev/null || true + register: kuma_refs + changed_when: false + + - name: Report any remaining references + ansible.builtin.debug: + msg: >- + {{ 'clean - nothing references Uptime Kuma' + if kuma_refs.stdout | trim | length == 0 + else 'STILL REFERENCING KUMA: ' ~ kuma_refs.stdout_lines | join(', ') }} diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml deleted file mode 100644 index dcd9bdc..0000000 --- a/ansible/infra/410_disk_usage_alerts.yml +++ /dev/null @@ -1,343 +0,0 @@ -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. -# -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. -# -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ -- name: Deploy Disk Usage Monitoring - hosts: managed - become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml - - ../infra_secrets.yml - - vars: - disk_usage_threshold_percent: 80 - disk_check_interval_minutes: 15 - monitored_mount_point: "/" - monitoring_script_dir: /opt/disk-monitoring - monitoring_script_path: "{{ monitoring_script_dir }}/disk_usage_monitor.sh" - log_file: "{{ monitoring_script_dir }}/disk_usage_monitor.log" - systemd_service_name: disk-usage-monitor - # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname and mount point - set_fact: - monitor_name: "disk-usage-{{ host_name.stdout }}-{{ monitored_mount_point | replace('/', 'root') }}" - monitor_friendly_name: "Disk Usage: {{ host_name.stdout }} ({{ monitored_mount_point }})" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': True, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when usage exceeds {{ disk_usage_threshold_percent }}%" - "{{ (disk_check_interval_minutes * 60) + 60 }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_disk_usage_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for disk monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create disk usage monitoring script - copy: - dest: "{{ monitoring_script_path }}" - content: | - #!/bin/bash - - # Disk Usage Monitoring Script - # Monitors disk usage and sends alerts to Uptime Kuma - # Mode: "No news is good news" - only sends alerts when disk usage is HIGH - - LOG_FILE="{{ log_file }}" - USAGE_THRESHOLD="{{ disk_usage_threshold_percent }}" - UPTIME_KUMA_URL="{{ uptime_kuma_disk_usage_push_url }}" - MOUNT_POINT="{{ monitored_mount_point }}" - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to get disk usage percentage - get_disk_usage() { - local mount_point="$1" - local usage="" - - # Get disk usage percentage (without % sign) - usage=$(df -h "$mount_point" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') - - if [ -z "$usage" ]; then - log_message "ERROR: Could not read disk usage for $mount_point" - return 1 - fi - - echo "$usage" - } - - # Function to get disk usage details - get_disk_details() { - local mount_point="$1" - df -h "$mount_point" 2>/dev/null | awk 'NR==2 {print "Used: "$3" / Total: "$2" ("$5" full)"}' - } - - # Function to send alert to Uptime Kuma when disk usage exceeds threshold - # With upside-down mode enabled, sending status=up will trigger an alert - send_uptime_kuma_alert() { - local usage="$1" - local details="$2" - local message="DISK FULL WARNING: ${MOUNT_POINT} is ${usage}% full (Threshold: ${USAGE_THRESHOLD}%) - ${details}" - - log_message "ALERT: $message" - - # Send push notification to Uptime Kuma with status=up - # In upside-down mode, status=up is treated as down/alert - response=$(curl -s -w "\n%{http_code}" -G \ - --data-urlencode "status=up" \ - --data-urlencode "msg=$message" \ - "$UPTIME_KUMA_URL" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" - else - log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" - fi - } - - # Main monitoring logic - main() { - log_message "Starting disk usage check for $MOUNT_POINT" - - # Get current disk usage - current_usage=$(get_disk_usage "$MOUNT_POINT") - - if [ $? -ne 0 ] || [ -z "$current_usage" ]; then - log_message "ERROR: Could not read disk usage" - exit 1 - fi - - # Get disk details - disk_details=$(get_disk_details "$MOUNT_POINT") - - log_message "Current disk usage: ${current_usage}% - $disk_details" - - # Check if usage exceeds threshold - if [ "$current_usage" -gt "$USAGE_THRESHOLD" ]; then - log_message "WARNING: Disk usage ${current_usage}% exceeds threshold ${USAGE_THRESHOLD}%" - send_uptime_kuma_alert "$current_usage" "$disk_details" - else - log_message "Disk usage is within normal range - no alert needed (no news is good news)" - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for disk usage monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.service" - content: | - [Unit] - Description=Disk Usage Monitor - After=network.target - - [Service] - Type=oneshot - ExecStart={{ monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for disk usage monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" - content: | - [Unit] - Description=Run Disk Usage Monitor every {{ disk_check_interval_minutes }} minute(s) - Requires={{ systemd_service_name }}.service - - [Timer] - OnBootSec={{ disk_check_interval_minutes }}min - OnUnitActiveSec={{ disk_check_interval_minutes }}min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start disk usage monitoring timer - systemd: - name: "{{ systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test disk usage monitoring script - command: "{{ monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "Disk usage monitoring script failed to execute properly" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml deleted file mode 100644 index 05532a7..0000000 --- a/ansible/infra/420_system_healthcheck.yml +++ /dev/null @@ -1,325 +0,0 @@ -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. -# -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. -# -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ -- name: Deploy System Healthcheck Monitoring - hosts: managed - become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml - - ../infra_secrets.yml - - vars: - healthcheck_interval_seconds: 60 # Send healthcheck every 60 seconds (1 minute) - healthcheck_timeout_seconds: 90 # Uptime Kuma should alert if no ping received within 90s - healthcheck_retries: 1 # Number of retries before alerting - monitoring_script_dir: /opt/system-healthcheck - monitoring_script_path: "{{ monitoring_script_dir }}/system_healthcheck.sh" - log_file: "{{ monitoring_script_dir }}/system_healthcheck.log" - systemd_service_name: system-healthcheck - # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "system-healthcheck-{{ host_name.stdout }}" - monitor_friendly_name: "System Healthcheck: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_healthcheck_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_healthcheck_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Regular healthcheck ping every {{ healthcheck_interval_seconds }}s" - "{{ healthcheck_timeout_seconds }}" - "{{ healthcheck_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_healthcheck_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for healthcheck monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create system healthcheck script - copy: - dest: "{{ monitoring_script_path }}" - content: | - #!/bin/bash - - # System Healthcheck Script - # Sends regular heartbeat pings to Uptime Kuma - # This ensures the system is running and able to communicate - - LOG_FILE="{{ log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_healthcheck_push_url }}" - HOSTNAME=$(hostname) - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to send healthcheck ping to Uptime Kuma - send_healthcheck() { - local uptime_seconds=$(awk '{print int($1)}' /proc/uptime) - local uptime_days=$((uptime_seconds / 86400)) - local uptime_hours=$(((uptime_seconds % 86400) / 3600)) - local uptime_minutes=$(((uptime_seconds % 3600) / 60)) - - local message="System healthy - Uptime: ${uptime_days}d ${uptime_hours}h ${uptime_minutes}m" - - log_message "Sending healthcheck ping: $message" - - # Send push notification to Uptime Kuma with status=up - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Healthcheck ping sent successfully (HTTP $http_code)" - else - log_message "ERROR: Failed to send healthcheck ping (HTTP $http_code)" - return 1 - fi - } - - # Main healthcheck logic - main() { - log_message "Starting system healthcheck for $HOSTNAME" - - # Send healthcheck ping - if send_healthcheck; then - log_message "Healthcheck completed successfully" - else - log_message "ERROR: Healthcheck failed" - exit 1 - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for system healthcheck - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.service" - content: | - [Unit] - Description=System Healthcheck Monitor - After=network.target - - [Service] - Type=oneshot - ExecStart={{ monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for system healthcheck - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" - content: | - [Unit] - Description=Run System Healthcheck every minute - Requires={{ systemd_service_name }}.service - - [Timer] - OnBootSec=30sec - OnUnitActiveSec={{ healthcheck_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start system healthcheck timer - systemd: - name: "{{ systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test system healthcheck script - command: "{{ monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "System healthcheck script failed to execute properly" - - - name: Display monitor information - debug: - msg: | - ✓ System healthcheck monitoring deployed successfully! - - Monitor Name: {{ monitor_friendly_name }} - Monitor Group: {{ uptime_kuma_monitor_group }} - Healthcheck Interval: Every {{ healthcheck_interval_seconds }} seconds (1 minute) - Timeout: {{ healthcheck_timeout_seconds }} seconds (90s) - Retries: {{ healthcheck_retries }} - - The system will send a heartbeat ping every minute. - Uptime Kuma will alert if no ping is received within 90 seconds (with 1 retry). - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_healthcheck_monitor.py - state: absent - delegate_to: localhost - become: no - diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml deleted file mode 100644 index 5c9e855..0000000 --- a/ansible/infra/430_cpu_temp_alerts.yml +++ /dev/null @@ -1,329 +0,0 @@ -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. -# -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. -# -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ -- name: Deploy CPU Temperature Monitoring - hosts: hypervisor - become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml - - ../infra_secrets.yml - - vars: - temp_threshold_celsius: 80 - temp_check_interval_minutes: 1 - monitoring_script_dir: /opt/nodito-monitoring - monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" - log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" - systemd_service_name: nodito-cpu-temp-monitor - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "cpu-temp-{{ host_name.stdout }}" - monitor_friendly_name: "CPU Temperature: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma CPU temperature monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_cpu_temp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - monitors = api.get_monitors() - notifications = api.get_notifications() - - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name=group_name) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': True, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - api.edit_monitor(existing_monitor['id'], **monitor_data) - else: - api.add_monitor(**monitor_data) - - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_cpu_temp_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when temperature exceeds {{ temp_threshold_celsius }}°C" - "{{ (temp_check_interval_minutes * 60) + 60 }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_cpu_temp_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for temperature monitoring - package: - name: - - lm-sensors - - curl - - jq - - bc - state: present - - - name: Create monitoring script directory - file: - path: "{{ monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create CPU temperature monitoring script - copy: - dest: "{{ monitoring_script_path }}" - content: | - #!/bin/bash - - # CPU Temperature Monitoring Script - # Monitors CPU temperature and sends alerts to Uptime Kuma - - LOG_FILE="{{ log_file }}" - TEMP_THRESHOLD="{{ temp_threshold_celsius }}" - UPTIME_KUMA_URL="{{ uptime_kuma_cpu_temp_push_url }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - get_cpu_temp() { - local temp="" - - if command -v sensors >/dev/null 2>&1; then - temp=$(sensors 2>/dev/null | grep -E "Core 0|Package id 0|Tdie|Tctl" | head -1 | grep -oE '[0-9]+\.[0-9]+°C' | grep -oE '[0-9]+\.[0-9]+') - fi - - if [ -z "$temp" ] && [ -f /sys/class/thermal/thermal_zone0/temp ]; then - temp=$(cat /sys/class/thermal/thermal_zone0/temp) - temp=$(echo "scale=1; $temp/1000" | bc -l 2>/dev/null || echo "$temp") - fi - - if [ -z "$temp" ] && command -v acpi >/dev/null 2>&1; then - temp=$(acpi -t 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1) - fi - - echo "$temp" - } - - send_uptime_kuma_alert() { - local temp="$1" - local message="CPU Temperature Alert: ${temp}°C (Threshold: ${TEMP_THRESHOLD}°C)" - - log_message "ALERT: $message" - - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/°/%C2%B0/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g') - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" - else - log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" - fi - } - - main() { - log_message "Starting CPU temperature check" - - current_temp=$(get_cpu_temp) - - if [ -z "$current_temp" ]; then - log_message "ERROR: Could not read CPU temperature" - exit 1 - fi - - log_message "Current CPU temperature: ${current_temp}°C" - - if (( $(echo "$current_temp > $TEMP_THRESHOLD" | bc -l) )); then - log_message "WARNING: CPU temperature ${current_temp}°C exceeds threshold ${TEMP_THRESHOLD}°C" - send_uptime_kuma_alert "$current_temp" - else - log_message "CPU temperature is within normal range" - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for CPU temperature monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.service" - content: | - [Unit] - Description=CPU Temperature Monitor - After=network.target - - [Service] - Type=oneshot - ExecStart={{ monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for CPU temperature monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" - content: | - [Unit] - Description=Run CPU Temperature Monitor every {{ temp_check_interval_minutes }} minute(s) - Requires={{ systemd_service_name }}.service - - [Timer] - OnBootSec={{ temp_check_interval_minutes }}min - OnUnitActiveSec={{ temp_check_interval_minutes }}min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start CPU temperature monitoring timer - systemd: - name: "{{ systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test CPU temperature monitoring script - command: "{{ monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "CPU temperature monitoring script failed to execute properly" - - - name: Display monitoring configuration - debug: - msg: - - "CPU Temperature Monitoring configured successfully" - - "Temperature threshold: {{ temp_threshold_celsius }}°C" - - "Check interval: {{ temp_check_interval_minutes }} minute(s)" - - "Monitor Name: {{ monitor_friendly_name }}" - - "Monitor Group: {{ uptime_kuma_monitor_group }}" - - "Uptime Kuma Push URL: {{ uptime_kuma_cpu_temp_push_url }}" - - "Monitoring script: {{ monitoring_script_path }}" - - "Systemd Service: {{ systemd_service_name }}.service" - - "Systemd Timer: {{ systemd_service_name }}.timer" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_cpu_temp_monitor.py - state: absent - delegate_to: localhost - become: no - diff --git a/ansible/infra/900_install_rsync.yml b/ansible/infra/900_install_rsync.yml index 6c6b10e..b690009 100644 --- a/ansible/infra/900_install_rsync.yml +++ b/ansible/infra/900_install_rsync.yml @@ -1,7 +1,5 @@ - name: Install rsync hosts: managed - vars_files: - - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 5b5e4ce..4decb5d 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -1,14 +1,10 @@ - name: Join machine to headscale mesh network hosts: managed become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml vars: headscale_host_name: "spacey" headscale_subdomain: "{{ subdomains.headscale }}" headscale_domain: "https://{{ headscale_subdomain }}.{{ root_domain }}" - headscale_namespace: "{{ service_settings.headscale.namespace }}" tasks: - name: Set facts for headscale server connection diff --git a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml index 86f693f..4edab06 100644 --- a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml +++ b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml @@ -1,8 +1,6 @@ - name: Bootstrap Nodito SSH Key Access hosts: hypervisor become: true - vars_files: - - ../infra_vars.yml tasks: - name: Install sudo package diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml index 378a674..0fab184 100644 --- a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -1,8 +1,6 @@ - name: Switch Proxmox VE from Enterprise to Community Repositories hosts: hypervisor become: true - vars_files: - - ../infra_vars.yml tasks: - name: Check for deb822 sources format diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 8cdde6a..2efbf8c 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -1,9 +1,6 @@ - name: Setup ZFS RAID 1 Pool for Proxmox Storage hosts: hypervisor become: true - vars_files: - - ../infra_vars.yml - - nodito_vars.yml tasks: - name: Verify Proxmox VE is running @@ -139,17 +136,18 @@ Config file exists: {{ storage_cfg_file.stat.exists }} Storage check result: {{ storage_exists_check.rc }} Pool exists: {{ zfs_pool_exists.rc == 0 }} - Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }} Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }} - - name: Remove existing storage if it exists - command: pvesm remove {{ zfs_pool_name }} - register: pvesm_remove_result - failed_when: false - when: - - zfs_pool_exists.rc == 0 - - storage_exists_check.rc == 0 - + # Registration is add-only on purpose. There used to be a "Remove existing + # storage if it exists" task here that ran `pvesm remove` whenever the + # storage WAS present, paired with an add that only ran when it was ABSENT. + # The two conditions are mutually exclusive, so a real run against a + # correctly-configured hypervisor removed the storage entry backing every VM + # and never put it back. It would also have dropped `mountpoint /var/lib/vz`, + # which the live entry has and which `pvesm add` below does not set. + # + # If the storage entry ever needs its options changed, edit + # /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here. - name: Add ZFS pool storage to Proxmox using pvesm command: > pvesm add zfspool {{ zfs_pool_name }} @@ -171,511 +169,56 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# ───────────────────────────────────────────────────────────────────────────── +# The monthly scrub. # -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. +# The ZFS HEALTH CHECK that used to share this play is gone: it is now the +# zfs-health check in infra/400_host_monitoring.yml, which carries the same five +# conditions - pool state, device states, resilver in progress, read/write/ +# checksum errors, and errors from the last scan - but reports to Gatus like +# every other host check instead of owning its own push plumbing. # -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ -- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs +# The scrub itself stays here, because it is not monitoring: it is the +# maintenance that gives the health check something true to report. A pool that +# is never scrubbed has no idea whether it is healthy. +# ───────────────────────────────────────────────────────────────────────────── +- name: Schedule the monthly ZFS scrub hosts: hypervisor become: true vars_files: - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml vars: - zfs_check_interval_seconds: 86400 # 24 hours - zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) - zfs_check_retries: 1 - zfs_monitoring_script_dir: /opt/zfs-monitoring - zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" - zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" - zfs_systemd_health_service_name: zfs-health-monitor zfs_systemd_scrub_service_name: zfs-monthly-scrub - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "zfs-health-{{ host_name.stdout }}" - monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma ZFS health monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_zfs_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma ZFS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_zfs_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" - "{{ zfs_check_timeout_seconds }}" - "{{ zfs_check_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for ZFS monitoring - package: - name: - - curl - - jq - state: present - - - name: Create monitoring script directory - file: - path: "{{ zfs_monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create ZFS health monitoring script - copy: - dest: "{{ zfs_monitoring_script_path }}" - content: | - #!/bin/bash - - # ZFS Pool Health Monitoring Script - # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy - # If any issues detected, does NOT send heartbeat (triggers timeout alert) - - LOG_FILE="{{ zfs_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" - POOL_NAME="{{ zfs_pool_name }}" - HOSTNAME=$(hostname) - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to check pool health using JSON output - check_pool_health() { - local pool="$1" - local issues_found=0 - - # Get pool status as JSON - local pool_json - pool_json=$(zpool status -j "$pool" 2>&1) - - if [ $? -ne 0 ]; then - log_message "ERROR: Failed to get pool status for $pool" - log_message " -> $pool_json" - return 1 - fi - - # Check 1: Pool state must be ONLINE - local pool_state - pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') - - if [ "$pool_state" != "ONLINE" ]; then - log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" - issues_found=1 - else - log_message "OK: Pool state is ONLINE" - fi - - # Check 2: Check all vdevs and devices for non-ONLINE states - local bad_states - bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.state? and .state != "ONLINE") | - "\(.name // "unknown"): \(.state)" - ' 2>/dev/null) - - if [ -n "$bad_states" ]; then - log_message "ISSUE: Found devices not in ONLINE state:" - echo "$bad_states" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: All devices are ONLINE" - fi - - # Check 3: Check for resilvering in progress - local scan_function scan_state - scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - - if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then - local resilver_progress - resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') - log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" - issues_found=1 - fi - - # Check 4: Check for read/write/checksum errors on all devices - # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison - local devices_with_errors - devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | - "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" - ' 2>/dev/null) - - if [ -n "$devices_with_errors" ]; then - log_message "ISSUE: Found devices with I/O errors:" - echo "$devices_with_errors" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: No read/write/checksum errors detected" - fi - - # Check 5: Check for scan errors (from last scrub/resilver) - local scan_errors - scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') - - if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then - log_message "ISSUE: Last scan reported $scan_errors errors" - issues_found=1 - else - log_message "OK: No scan errors" - fi - - return $issues_found - } - - # Function to get last scrub info for status message - get_scrub_info() { - local pool="$1" - local pool_json - pool_json=$(zpool status -j "$pool" 2>/dev/null) - - local scan_func scan_state scan_start - scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') - - if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then - echo "scrub in progress (started $scan_start)" - elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then - echo "last scrub: $scan_start" - else - echo "no scrub history" - fi - } - - # Function to send heartbeat to Uptime Kuma - send_heartbeat() { - local message="$1" - - log_message "Sending heartbeat to Uptime Kuma: $message" - - # URL encode the message - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent successfully (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - # Main health check logic - main() { - log_message "==========================================" - log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" - - # Run all health checks - if check_pool_health "$POOL_NAME"; then - # All checks passed - send heartbeat - local scrub_info - scrub_info=$(get_scrub_info "$POOL_NAME") - - local message="Pool $POOL_NAME healthy ($scrub_info)" - send_heartbeat "$message" - - log_message "Health check completed: ALL OK" - exit 0 - else - # Issues found - do NOT send heartbeat (will trigger timeout alert) - log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" - log_message "Uptime Kuma will alert after timeout due to missing heartbeat" - exit 1 - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for ZFS health monitoring - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" - content: | - [Unit] - Description=ZFS Pool Health Monitor - After=zfs.target network.target - - [Service] - Type=oneshot - ExecStart={{ zfs_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for daily ZFS health monitoring - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Pool Health Monitor daily - Requires={{ zfs_systemd_health_service_name }}.service - - [Timer] - OnBootSec=5min - OnUnitActiveSec={{ zfs_check_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - name: Create systemd service for ZFS monthly scrub - copy: + template: + src: templates/zfs-monthly-scrub.service.j2 dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" - content: | - [Unit] - Description=ZFS Monthly Scrub for {{ zfs_pool_name }} - After=zfs.target - - [Service] - Type=oneshot - ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target owner: root group: root mode: '0644' - name: Create systemd timer for monthly ZFS scrub - copy: + template: + src: templates/zfs-monthly-scrub.timer.j2 dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Scrub on last day of every month at 4:00 AM - Requires={{ zfs_systemd_scrub_service_name }}.service - - [Timer] - OnCalendar=*-*~01 04:00:00 - Persistent=true - - [Install] - WantedBy=timers.target owner: root group: root mode: '0644' - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start ZFS health monitoring timer - systemd: - name: "{{ zfs_systemd_health_service_name }}.timer" - enabled: yes - state: started - - - name: Enable and start ZFS monthly scrub timer + - name: Enable and start the monthly scrub timer systemd: name: "{{ zfs_systemd_scrub_service_name }}.timer" enabled: yes state: started + daemon_reload: yes - - name: Test ZFS health monitoring script - command: "{{ zfs_monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "ZFS health monitoring script failed - check pool health" - - - name: Display monitoring configuration + - name: Report the scrub schedule debug: - msg: | - ✓ ZFS Pool Health Monitoring deployed successfully! - - Monitor Name: {{ monitor_friendly_name }} - Monitor Group: {{ uptime_kuma_monitor_group }} - Pool Name: {{ zfs_pool_name }} - - Health Check: - - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) - - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) - - Script: {{ zfs_monitoring_script_path }} - - Log: {{ zfs_log_file }} - - Service: {{ zfs_systemd_health_service_name }}.service - - Timer: {{ zfs_systemd_health_service_name }}.timer - - Monthly Scrub: - - Schedule: Last day of month at 4:00 AM - - Service: {{ zfs_systemd_scrub_service_name }}.service - - Timer: {{ zfs_systemd_scrub_service_name }}.timer - - Conditions monitored: - - Pool state (must be ONLINE) - - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) - - Resilver status (alerts if resilvering) - - Read/Write/Checksum errors - - Scrub errors - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_zfs_monitor.py - state: absent - delegate_to: localhost - become: no + msg: >- + Monthly scrub of {{ zfs_pool_name }}: + last day of each month at 04:00. + Health is reported separately by the zfs-health check + (infra/400_host_monitoring.yml). diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index 3b687ee..faf93a5 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -1,9 +1,6 @@ - name: Create Proxmox template from Debian cloud image (no VM clone) hosts: hypervisor become: true - vars_files: - - ../../infra_vars.yml - - nodito_vars.yml vars: # Defaults (override via vars_files or --extra-vars as needed) diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index f39fb56..12fe720 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -1,12 +1,33 @@ - name: Setup NUT (Network UPS Tools) for CyberPower UPS hosts: hypervisor become: true - vars_files: - - ../../infra_vars.yml - - nodito_vars.yml - - nodito_secrets.yml tasks: + # ------------------------------------------------------------------ + # Safety catch + # + # /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by + # hand in January 2026 and carry a working password. host_vars/nodito/vault.yml + # (formerly infra/nodito/nodito_secrets.yml) still holds the literal string + # CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that + # working pair with a placeholder and restart NUT - leaving the hypervisor's + # UPS unmonitored and unable to trigger a clean shutdown on mains loss. + # + # Until the real password is put in the vault, stop here. + # ansible-vault edit host_vars/nodito/vault.yml + # ------------------------------------------------------------------ + - name: Refuse to run with a placeholder UPS password + assert: + that: + - ups_password is defined + - ups_password | length > 0 + - ups_password != "CHANGE_ME_TO_SECURE_PASSWORD" + fail_msg: >- + ups_password is unset or still the placeholder. Applying this play would + overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on + nodito and restart NUT. Put the real password in the vault first: + ansible-vault edit host_vars/nodito/vault.yml + # ------------------------------------------------------------------ # Installation # ------------------------------------------------------------------ @@ -75,90 +96,45 @@ # Configuration files # ------------------------------------------------------------------ - name: Configure NUT mode (standalone) - copy: + template: dest: /etc/nut/nut.conf - content: | - # Managed by Ansible - MODE=standalone + src: templates/nut.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure UPS device - copy: + template: dest: /etc/nut/ups.conf - content: | - # Managed by Ansible - [{{ ups_name }}] - driver = {{ ups_driver }} - port = {{ ups_port }} - desc = "{{ ups_desc }}" - offdelay = {{ ups_offdelay }} - ondelay = {{ ups_ondelay }} + src: templates/ups.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd to listen on localhost - copy: + template: dest: /etc/nut/upsd.conf - content: | - # Managed by Ansible - LISTEN 127.0.0.1 3493 + src: templates/upsd.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd users - copy: + template: dest: /etc/nut/upsd.users - content: | - # Managed by Ansible - [{{ ups_user }}] - password = {{ ups_password }} - upsmon master + src: templates/upsd.users.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsmon - copy: + template: dest: /etc/nut/upsmon.conf - content: | - # Managed by Ansible - MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master - - MINSUPPLIES 1 - SHUTDOWNCMD "/sbin/shutdown -h +0" - POLLFREQ 5 - POLLFREQALERT 5 - HOSTSYNC 15 - DEADTIME 15 - POWERDOWNFLAG /etc/killpower - - # Notifications - NOTIFYMSG ONLINE "UPS %s on line power" - NOTIFYMSG ONBATT "UPS %s on battery" - NOTIFYMSG LOWBATT "UPS %s battery is low" - NOTIFYMSG FSD "UPS %s: forced shutdown in progress" - NOTIFYMSG COMMOK "Communications with UPS %s established" - NOTIFYMSG COMMBAD "Communications with UPS %s lost" - NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" - NOTIFYMSG REPLBATT "UPS %s battery needs replacing" - - # Log all events to syslog - NOTIFYFLAG ONLINE SYSLOG - NOTIFYFLAG ONBATT SYSLOG - NOTIFYFLAG LOWBATT SYSLOG - NOTIFYFLAG FSD SYSLOG - NOTIFYFLAG COMMOK SYSLOG - NOTIFYFLAG COMMBAD SYSLOG - NOTIFYFLAG SHUTDOWN SYSLOG - NOTIFYFLAG REPLBATT SYSLOG + src: templates/upsmon.conf.j2 owner: root group: nut mode: "0640" @@ -249,334 +225,11 @@ - nut-server - nut-monitor - -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# The UPS heartbeat play that used to live here is gone. What it deployed - +# /opt/ups-monitoring plus a ups-heartbeat timer - is now the ups-status check +# in infra/400_host_monitoring.yml, which reports to Gatus like every other +# host check instead of carrying its own push plumbing. # -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. -# -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ -- name: Setup UPS Heartbeat Monitoring with Uptime Kuma - hosts: hypervisor - become: true - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml - - nodito_secrets.yml - - vars: - ups_heartbeat_interval_seconds: 60 - ups_heartbeat_timeout_seconds: 120 - ups_heartbeat_retries: 1 - ups_monitoring_script_dir: /opt/ups-monitoring - ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" - ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" - ups_systemd_service_name: ups-heartbeat - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "ups-{{ host_name.stdout }}" - monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma UPS monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_ups_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - monitors = api.get_monitors() - notifications = api.get_notifications() - - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name=group_name) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - api.edit_monitor(existing_monitor['id'], **monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - api.add_monitor(**monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma UPS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_ups_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" - "{{ ups_heartbeat_timeout_seconds }}" - "{{ ups_heartbeat_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL as fact - set_fact: - uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - - name: Install required packages for UPS monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ ups_monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create UPS heartbeat monitoring script - copy: - dest: "{{ ups_monitoring_script_path }}" - content: | - #!/bin/bash - - # UPS Heartbeat Monitoring Script - # Sends heartbeat to Uptime Kuma only when UPS is on mains power - # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) - - LOG_FILE="{{ ups_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" - UPS_NAME="{{ ups_name }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - send_heartbeat() { - local message="$1" - - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent: $message (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - main() { - local status charge runtime load - - status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) - - if [ -z "$status" ]; then - log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" - exit 1 - fi - - charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) - runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) - load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) - - if [[ "$status" == *"OL"* ]]; then - local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" - send_heartbeat "$message" - exit 0 - else - log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" - exit 1 - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for UPS heartbeat - copy: - dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" - content: | - [Unit] - Description=UPS Heartbeat Monitor - After=network.target nut-monitor.service - - [Service] - Type=oneshot - ExecStart={{ ups_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for UPS heartbeat - copy: - dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" - content: | - [Unit] - Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds - Requires={{ ups_systemd_service_name }}.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start UPS heartbeat timer - systemd: - name: "{{ ups_systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test UPS heartbeat script - command: "{{ ups_monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "UPS heartbeat script failed - check UPS status and communication" - - - name: Display monitoring configuration - debug: - msg: - - "UPS Monitoring configured successfully" - - "" - - "NUT Configuration:" - - " UPS Name: {{ ups_name }}" - - " UPS Description: {{ ups_desc }}" - - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" - - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" - - "" - - "Uptime Kuma Monitoring:" - - " Monitor Name: {{ monitor_friendly_name }}" - - " Monitor Group: {{ uptime_kuma_monitor_group }}" - - " Push URL: {{ uptime_kuma_ups_push_url }}" - - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" - - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" - - "" - - "Scripts and Services:" - - " Script: {{ ups_monitoring_script_path }}" - - " Log: {{ ups_log_file }}" - - " Service: {{ ups_systemd_service_name }}.service" - - " Timer: {{ ups_systemd_service_name }}.timer" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_ups_monitor.py - state: absent - delegate_to: localhost - become: no +# This playbook is now purely NUT setup: the driver, upsd, upsmon and the +# shutdown behaviour. Monitoring whether the UPS is on mains is a separate +# concern and belongs with the other host checks. diff --git a/ansible/infra/nodito/nodito_secrets.yml b/ansible/infra/nodito/nodito_secrets.yml deleted file mode 100644 index c34b1e8..0000000 --- a/ansible/infra/nodito/nodito_secrets.yml +++ /dev/null @@ -1,12 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -39386134336137333139343861353965353665626161303634323563386563356364373265616163 -3237623438636161323366313363356335613566303338630a663732316663333136653737663935 -34373466316538663962636134303530323837316563336136653636643430363566376363323666 -3531613636393331310a306633613030343138656237663061653438336562323135633732356563 -33396136656564353465636361613161326465303166373131343562363338303966666362643532 -33383561366539393930633766363533313363653733393263376634316362643863376635366638 -36396333633231616662323465323932656565396138313264613832346261616231333265636439 -61666334613662373839613833613663333436373365376534643662656335316536303739616437 -65333635613330353536353162646234323266316338643435653864666364313734386665303830 -34383961633734646134373866623330663038366130306265656466653562643764346162313333 -353664363335303433393330306332666336 diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml deleted file mode 100644 index c0002f3..0000000 --- a/ansible/infra/nodito/nodito_vars.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Nodito CPU Temperature Monitoring Configuration - -# Temperature Monitoring Configuration -temp_threshold_celsius: 80 -temp_check_interval_minutes: 1 - -# Script Configuration -monitoring_script_dir: /opt/nodito-monitoring -monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" -log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" - -# System Configuration -systemd_service_name: nodito-cpu-temp-monitor - -# ZFS Pool Configuration -zfs_pool_name: "proxmox-tank-1" -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror -zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror -zfs_pool_mountpoint: "/var/lib/vz" - -# UPS Configuration (CyberPower CP900EPFCLCD via USB) -ups_name: cyberpower -ups_desc: "CyberPower CP900EPFCLCD" -ups_driver: usbhid-ups -ups_port: auto -ups_user: counterweight -ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power -ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power diff --git a/ansible/infra/nodito/templates/nut.conf.j2 b/ansible/infra/nodito/templates/nut.conf.j2 new file mode 100644 index 0000000..1f8a72f --- /dev/null +++ b/ansible/infra/nodito/templates/nut.conf.j2 @@ -0,0 +1,2 @@ +# Managed by Ansible +MODE=standalone diff --git a/ansible/infra/nodito/templates/ups.conf.j2 b/ansible/infra/nodito/templates/ups.conf.j2 new file mode 100644 index 0000000..0fa0a60 --- /dev/null +++ b/ansible/infra/nodito/templates/ups.conf.j2 @@ -0,0 +1,9 @@ +# Managed by Ansible +maxretry = 3 + +[{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} diff --git a/ansible/infra/nodito/templates/upsd.conf.j2 b/ansible/infra/nodito/templates/upsd.conf.j2 new file mode 100644 index 0000000..baddf21 --- /dev/null +++ b/ansible/infra/nodito/templates/upsd.conf.j2 @@ -0,0 +1,2 @@ +# Managed by Ansible +LISTEN 127.0.0.1 3493 diff --git a/ansible/infra/nodito/templates/upsd.users.j2 b/ansible/infra/nodito/templates/upsd.users.j2 new file mode 100644 index 0000000..8d43784 --- /dev/null +++ b/ansible/infra/nodito/templates/upsd.users.j2 @@ -0,0 +1,4 @@ +# Managed by Ansible +[{{ ups_user }}] + password = {{ ups_password }} + upsmon master diff --git a/ansible/infra/nodito/templates/upsmon.conf.j2 b/ansible/infra/nodito/templates/upsmon.conf.j2 new file mode 100644 index 0000000..dc301e1 --- /dev/null +++ b/ansible/infra/nodito/templates/upsmon.conf.j2 @@ -0,0 +1,34 @@ +# Managed by Ansible +MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + +MINSUPPLIES 1 +SHUTDOWNCMD "/sbin/shutdown -h +0" +POLLFREQ 5 +POLLFREQALERT 5 +HOSTSYNC 15 +DEADTIME 15 +POWERDOWNFLAG "/etc/killpower" +OFFDURATION 30 +RBWARNTIME 43200 +NOCOMMWARNTIME 300 +FINALDELAY 5 + +# Notifications +NOTIFYMSG ONLINE "UPS %s on line power" +NOTIFYMSG ONBATT "UPS %s on battery" +NOTIFYMSG LOWBATT "UPS %s battery is low" +NOTIFYMSG FSD "UPS %s: forced shutdown in progress" +NOTIFYMSG COMMOK "Communications with UPS %s established" +NOTIFYMSG COMMBAD "Communications with UPS %s lost" +NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" +NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + +# Log all events to syslog +NOTIFYFLAG ONLINE SYSLOG +NOTIFYFLAG ONBATT SYSLOG +NOTIFYFLAG LOWBATT SYSLOG +NOTIFYFLAG FSD SYSLOG +NOTIFYFLAG COMMOK SYSLOG +NOTIFYFLAG COMMBAD SYSLOG +NOTIFYFLAG SHUTDOWN SYSLOG +NOTIFYFLAG REPLBATT SYSLOG diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 new file mode 100644 index 0000000..04b3514 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=ZFS Monthly Scrub for {{ zfs_pool_name }} +After=zfs.target + +[Service] +Type=oneshot +ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} +User=root +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 new file mode 100644 index 0000000..36ad3d7 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Run ZFS Scrub on last day of every month at 4:00 AM +Requires={{ zfs_systemd_scrub_service_name }}.service + +[Timer] +OnCalendar=*-*~01 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml deleted file mode 100644 index 4803d86..0000000 --- a/ansible/infra_secrets.yml +++ /dev/null @@ -1,60 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -34616531626537376463326332306434383163393761363536633133363161373631376437653234 -3831656536313531343433376264643261396634633965370a303931336236323766313065636535 -31343338616364386132623232373266663665353563666535383232666262663062323864303932 -3439373734316639650a633732333064633335383833663261326464336538636666323063646331 -66653464636534663935616461636132613162303530663237376661343338396133323431623561 -30303563303465633231306339326561313533333536376433393130376634303932643339656336 -30616130343733626162366561393939393766623138616537393032376665396139333561623830 -33623130336235633034666561383165616439316334323664623661343438663733373833306335 -30396165313731636435636263353133326135346236323030653734353831626238666663656364 -62386164363361373464383730333733626333333736306535336635613634646535383237326332 -37636532663933663739303634333235343137333363316430643263326231613635643633636437 -30336462633233396430633939616661643261666136656363363461343230653738613166346434 -63376363666462346539343063343562623962303130616535306439653134633630353963373337 -64313239353162363862643432666438383330636638333464323731643163643236313535313030 -65656231383738356537326661663163376634613031396436646633376139313237343534653065 -64323532346133313265346334353530393633396339316330366536646565353836396662303866 -65373030393463306664623235626437613965303837313365373632643935656630303936373837 -64663539623764656636613263346638376665626262663430333231633735653563643864343835 -62303637383332653366383066326536316336306539623230353066343739316430356437313365 -33323031613130643832636133636263303565623330306135333762363036633737343933623437 -33633436363062643436376632336235666330656265316333323361316430656631383734343238 -65616366666536363866343361666563643336306532666332643230656264303565353032393932 -36613738326239376161383064646634623639356439663965323237303361343838373737636235 -39656465333063633136643463356564636537326339653033633063366265636138656466646264 -32643961343863656464353439346531633036613138353333353631626665396239326465303632 -39336664333038636435366333623336353963323535316436646136313739373537376537353735 -66333265633232346536313933333836643439656666636236626266323333383935383565616139 -38626133356364613562393664643563346539366364636332613162663337356232333735393431 -30343635393462356431646336636663333735343164616261343836363366616162613563613066 -66333034613234626431356536323832393466363135653231336238356334353638613838626430 -63636630313533363563376161663637353362383431663130646666646433353864373736613662 -62343666633862643839666532656132653939663437373164393536653739376265303130353636 -62616538326330333136626238306133363634326365366637363961333635356133613564373131 -64626139656633663065353862343135393233333231386166346532656534623361623061336434 -37616565626134333863376532393861376132323434656433613731613834633963386635353336 -39653263346637303836663763653264396139383363616461393333333362616332663863363834 -61613832326466646133626337663364333566326633313664613236346135653332373035313034 -62356530313065393737393634313166313039616635363365633031336434396463326462316236 -62353838636533303664663132353838636632353733316639643964666139383539333761663531 -37366530373165653063303032383461326535613336343635626538373635356266396666346133 -61383434323565306438356634646338666232363238393932353365363461376130323363363236 -34643862653937353461383265663933646264373365623465313666643662633334366437646434 -34343332613832366235663563353435323738356339383338333361383561336366376238393237 -38656265663164376666616366623062393530366261623361383365376235643035353838313235 -32396430376139313430316439333539343266393965353030636638656661316137313437663832 -64366661323865383262323036613230316233666566386133363633303931323464663238323861 -62333132346165353830653062353763306232366563323430383933653163373562326536393362 -66353931376262383931613931303034356537363137366235643430323465623162363031373034 -64653563666533363062333831336463376530306639616631303266386136666233653638653732 -31313537393838333730303235363661326333646531653935633935373362303732653565656438 -36626330623063373832333739643666396135653838643164376264393863646433353036643731 -34623636623366353466333237373265323936643134613230326565303663306431373634356334 -37613732393262626135306632623336613364323632316636623037656538313236666562633438 -37306361333130376538366232393637646263656239303431623963626265386634313735373162 -64393733303132383464386330626135336237303563326238633437346164333738646431333730 -62616533363638643261306565373835303832313539656564386132393863373039653938616261 -36343031356361336563316230343433633033353130373933313361326164633765316433616162 -62356338393634653330623933313433343630386264396337373532376335316362333164363963 -363663666365336564663232363462613162 diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example deleted file mode 100644 index c95234d..0000000 --- a/ansible/infra_secrets.yml.example +++ /dev/null @@ -1,38 +0,0 @@ -# Uptime Kuma login credentials -# Used by the disk monitoring playbook to create monitors automatically - - -# ntfy credentials -# Used for notification channel setup in Uptime Kuma - -ntfy_username: "your_ntfy_username" -ntfy_password: "your_ntfy_password" - -# headscale-ui credentials -# Used for HTTP basic authentication via Caddy -# Provide either: -# - headscale_ui_password: plain text password (will be hashed automatically) -# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate) - -headscale_ui_username: "admin" -headscale_ui_password: "your_secure_password_here" -# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password - -bitcoin_rpc_user: "bitcoinrpc" -bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" - -# Mempool MariaDB credentials -# Used by: services/mempool/deploy_mempool_playbook.yml -mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" - -# Forgejo Runner registration token -# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml -# See: services/forgejo-runner/SETUP.md for how to obtain this token -forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE" - -# DATUM Gateway secrets -# Used by: services/datum-gateway/deploy_datum_gateway_playbook.yml -datum_mining_address: "YOUR_BITCOIN_ADDRESS_FOR_BLOCK_REWARDS" -datum_gateway_admin_password: "CHANGE_ME_TO_SECURE_PASSWORD" -datum_dashboard_username: "admin" -datum_dashboard_password_hash: "$2a$14$..." # Generate with: caddy hash-password diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml deleted file mode 100644 index 36d35f8..0000000 --- a/ansible/infra_vars.yml +++ /dev/null @@ -1,9 +0,0 @@ -new_user: counterweight -ssh_port: 22 -allow_ssh_from: "any" -root_domain: contrapeso.xyz - -# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the -# playbooks are kept deliberately — the check logic is meant to be rewired to -# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. -uptime_kuma_enabled: false diff --git a/ansible/inventory.ini b/ansible/inventory.ini index b9b2906..c39439d 100644 --- a/ansible/inventory.ini +++ b/ansible/inventory.ini @@ -1,7 +1,7 @@ [vps] vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +monitoring ansible_host=64.226.70.190 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua [nodito_host] nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua @@ -15,11 +15,10 @@ memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counter forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -nonkeiwaisi_local ansible_host=nonkeiwaisi-box lan_ip=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua # Local connection to laptop: this assumes you're running ansible commands from your personal laptop [lapy] -localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C +localhost ansible_connection=local ansible_user=counterweight [arbret] prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua @@ -27,8 +26,12 @@ prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 [edge] vipy -[monitoring] -watchtower +# The group is `observability`, NOT `monitoring` — there is a HOST named +# `monitoring` on line 5, and a group with the same name makes `hosts: monitoring` +# ambiguous. Ansible resolved it to the host and warned: +# [WARNING]: Found both group and host with same name: monitoring +[observability] +monitoring [vpn_control] spacey @@ -64,7 +67,7 @@ nodito_vms # Hosts that run Caddy and therefore have /etc/caddy/sites-enabled. [caddy:children] edge -monitoring +observability vpn_control [backup_store] diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 0686ab6..0be3807 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -7,6 +7,10 @@ ansible.builtin.include_role: name: backup_store vars: + # check-backups.sh reports one result per source plus one for the store + # itself, so it needs the collection URL and appends each key. + backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" backup_store_sources: - name: arbret source: "arbret@prd-arbret:/opt/arbret/backups/" @@ -26,3 +30,78 @@ - name: forgejo source: "backup-pull@prd-vipy:/opt/backups/forgejo/" retention_days: 14 + +# ───────────────────────────────────────────────────────────────────────────── +# Register the backup checks with Gatus. +# +# Two groups on purpose, because they answer different questions and fail for +# different reasons: +# +# backup-dump did the SOURCE produce an artefact? Pushed by each dump right +# after it runs, so a broken dump is visible within minutes. +# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is +# retention pruning? Pushed by check-backups.sh at 05:30. +# +# The store alone could catch almost everything, because the artefact filename +# carries the source's dump timestamp - a source whose timer died still pulls +# "ok" forever, but the timestamp gives it away. What the source side adds is +# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it +# cannot tell you whether the dump broke or the pull did. +# +# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` +# deliberately excludes, so nothing of ours runs there. It is store-checked only. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the backup checks with Gatus + hosts: observability + become: yes + vars: + # Sources we deploy the dump for, and the host each one runs on. + dump_sources: + - {name: headscale, host: spacey} + - {name: memos, host: memos_box_local} + - {name: vaultwarden, host: vipy} + - {name: lnbits, host: vipy} + - {name: forgejo, host: vipy} + store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] + + tasks: + # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter + # than a day plus slack. The store-side check catches the same fault within + # 6h by reading the artefact's dump timestamp out of the filename, so this is + # the slow backstop rather than the primary signal. + - name: Build the dump endpoint list + ansible.builtin.set_fact: + dump_endpoints: "{{ dump_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'backup-dump', + 'token': gatus_push_tokens[item.host], + 'heartbeat': '26h'}] }}" + loop: "{{ dump_sources }}" + + - name: Build the store endpoint list + ansible.builtin.set_fact: + store_endpoints: "{{ store_endpoints | default([]) + [{ + 'name': item, + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '7h'}] }}" + loop: "{{ store_sources }}" + + - name: Register the backup endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + # Push endpoints: the heartbeat window is the tolerance, so alert on + # the first failure rather than waiting for three 7h windows to pass. + gatus_endpoint_default_alerts: + - type: signal + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 12h + gatus_endpoint_name: backups + gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ + 'name': 'pull job', + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '7h'}] }}" diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml index 7de6006..b195190 100644 --- a/ansible/roles/backup_source/defaults/main.yml +++ b/ansible/roles/backup_source/defaults/main.yml @@ -29,3 +29,16 @@ backup_source_retention_days: 7 # Schedule. The box pulls at 04:00, so dumps must land before that. backup_source_on_calendar: "*-*-* 02:00:00" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Where to report that this dump ran and produced a plausible artefact. +# Gatus external endpoint: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty is valid and is not an error: the unit's exit code is still the answer, +# and the STORE will independently notice a stale dump within ~26h because the +# artefact filename carries this dump's timestamp. Reporting here only buys +# earlier detection and tells you it was the DUMP that broke rather than the +# pull. +backup_source_push_url: "" +backup_source_push_token: "" diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index 4283b84..a964c5d 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -16,12 +16,27 @@ # Declared here rather than assumed. Stage 1 installed it by hand; this is what # makes a rebuilt host get it too. -- name: Ensure age is installed +# Cache refresh is best-effort on purpose. An unrelated third-party repo with a +# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource +# key) makes `apt-get update` return warnings, which the apt module treats as a +# hard failure — and that must not stop backups being configured. Installing the +# package is NOT best-effort: if age is genuinely unavailable, the next task fails. +- name: Refresh the apt cache (best effort) ansible.builtin.apt: - name: age - state: present update_cache: yes cache_valid_time: 3600 + failed_when: false + changed_when: false + +- name: Ensure age is installed + ansible.builtin.apt: + name: + - age + # curl is needed only when backup_source_push_url is set, but installing it + # unconditionally keeps the task idempotent and it is present on every + # Debian host here anyway. + - curl + state: present # The pull account: unprivileged, no sudo, exists only so small-backups-box can # read the dump directory. Trust points one way — the box can read backups, and @@ -73,14 +88,18 @@ mode: '0750' validate: "bash -n %s" +# The .service carries the push token in an Environment= line, so it is 0600. +# The .timer holds nothing secret and stays world-readable. - name: "Install the {{ backup_source_name }}-backup systemd units" ansible.builtin.template: - src: "backup.{{ item }}.j2" - dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" + src: "backup.{{ item.unit }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}" owner: root group: root - mode: '0644' - loop: [service, timer] + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} notify: Reload systemd for backup units - name: "Enable the {{ backup_source_name }}-backup timer" diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 index 91ad063..7a1df21 100644 --- a/ansible/roles/backup_source/templates/backup.service.j2 +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -9,6 +9,8 @@ After={{ backup_source_stop_service if '.' in backup_source_stop_service else ba [Service] Type=oneshot ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +Environment=BACKUP_PUSH_URL={{ backup_source_push_url }} +Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }} StandardOutput=journal StandardError=journal SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index 18fabd6..018c313 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -41,18 +41,76 @@ chmod 700 "$BACKUP_DIR" # here or they accumulate forever. rm -f "${BACKUP_DIR}/${NAME}_"*.partial +# --- Reporting ------------------------------------------------------------- +# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed +# backup even though the script "finished". Both are reported as failures. +PUSH_URL="${BACKUP_PUSH_URL:-}" +PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}" + +report() { + local success="$1" message="$2" + [ -n "$PUSH_URL" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true +} + +# Reports on ANY exit path, so a dump that dies halfway still reports rather +# than going quiet. The size of the FINISHED artefact decides success, not +# merely reaching the end of the script. +# +# This is called FROM the single EXIT trap below - it must never register an +# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather +# than adding to it, so a second trap here silently discards the one that +# restarts the service, and a backup run leaves the service stopped. That is +# precisely the failure the restart trap exists to prevent. +report_outcome() { + local rc="$1" + if [ "$rc" -ne 0 ]; then + report "false" "${NAME} dump exited ${rc}" + elif [ ! -s "$ARTIFACT" ]; then + report "false" "${NAME} produced no artefact at ${ARTIFACT}" + else + report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)" + fi +} + +# --- One EXIT handler, doing both jobs ------------------------------------- +# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous +# handler rather than adding to it. Registering a second one here would +# silently discard the service restart and leave the service stopped after +# every backup - which is the exact bug the restart exists to prevent, and it +# is invisible until someone notices the service is down. +on_exit() { + local rc=$? {% if backup_source_stop_service or backup_source_stop_command %} -# --- Stop the service, and guarantee it comes back --- + log "Restarting ${SERVICE}..." + eval "$START_CMD" || true +{% endif %} + report_outcome "$rc" +} +trap on_exit EXIT + +{% if backup_source_stop_service or backup_source_stop_command %} +# --- Stop the service; the trap above guarantees it comes back ------------- # The trap is the point: without it a failed dump leaves the service down until # the next timer fires. Every hand-written script this replaced had that bug. +# It is armed BEFORE the stop, so even a failure during the stop restarts. log "Stopping ${SERVICE}..." eval "$STOP_CMD" -trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT {% endif %} # --- Dump straight into age; plaintext never touches the disk --- log "Writing ${ARTIFACT}..." {{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +{% if backup_source_pull_user %} +# Match the final ownership immediately, so even a partial left by a later +# failure is not an unreadable obstacle to the pull. +chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial" +chmod 640 "${ARTIFACT}.partial" +{% endif %} mv "${ARTIFACT}.partial" "$ARTIFACT" {% if backup_source_pull_user %} # Readable by the pull account and nobody else. The contents are age-encrypted diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml index c0c128b..e6d17a3 100644 --- a/ansible/roles/backup_store/defaults/main.yml +++ b/ansible/roles/backup_store/defaults/main.yml @@ -9,3 +9,21 @@ backup_store_on_calendar: "*-*-* 04:00:00" # source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" # retention_days: 90 backup_store_sources: [] + +# ── Reporting ──────────────────────────────────────────────────────────────── +# check-backups.sh reports one result PER SOURCE plus one for the store itself, +# so the base URL is the endpoints collection and the script appends each key. +# Empty is valid: the script still prints its report and exits 0/1. +backup_store_check_push_base: "" +backup_store_check_push_token: "" + +# Every six hours, offset past the 04:00 pull so the first run of the day sees a +# finished pull. The BACKUPS are daily, but this check is not - it reads the +# source's dump timestamp out of the artefact filename, so running it more often +# catches "the source stopped dumping" within hours rather than a day, and lets +# the Gatus heartbeat be 7h instead of 30h. +backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00" + +# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the +# pull is at 04:00, so 26h tolerates exactly one missed night before alarming. +backup_store_check_max_age_hours: 26 diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml index d138830..55500c6 100644 --- a/ansible/roles/backup_store/tasks/main.yml +++ b/ansible/roles/backup_store/tasks/main.yml @@ -33,6 +33,42 @@ validate: "bash -n %s" become: yes +# An assertion that last night actually worked. Generated from the same source +# list as the puller, so it can never drift out of sync with what is supposed to +# be arriving. Runs on a timer AND is useful by hand. +- name: Install the backup check script + ansible.builtin.template: + src: check-backups.sh.j2 + dest: /usr/local/bin/check-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + +# The .service carries the push token, so it is 0600; the .timer is not secret. +- name: Install the check-backups systemd units + ansible.builtin.template: + src: "check-backups.{{ item.unit }}.j2" + dest: "/etc/systemd/system/check-backups.{{ item.unit }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} + become: yes + +# restarted, not started: `started` is a no-op on an already-active timer, so a +# changed schedule would never be picked up. +- name: Enable the check-backups timer + ansible.builtin.systemd: + name: check-backups.timer + enabled: yes + state: restarted + daemon_reload: yes + become: yes + - name: Install the pull-backups systemd units ansible.builtin.template: src: "pull-backups.{{ item }}.j2" diff --git a/ansible/roles/backup_store/templates/check-backups.service.j2 b/ansible/roles/backup_store/templates/check-backups.service.j2 new file mode 100644 index 0000000..3e50b6a --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Verify the nightly backup pull actually worked +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }} +Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }} +Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 new file mode 100644 index 0000000..37c9916 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.sh.j2 @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Assert the nightly backups actually worked. +# +# Run as {{ ansible_user_id }} on this host. Needs no sudo. +# +# What it CANNOT do: verify contents. The age identity lives only on lapy, so +# this host cannot decrypt anything it holds — by design. These are freshness, +# completeness and integrity checks. To verify content, decrypt on lapy: +# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups//" \ +# | age -d -i ~/.age/counterweight_age | tar -tzf - | head +# +# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL. +set -uo pipefail + +STORE="{{ backup_store_dir }}" +MAX_AGE_H="${1:-26}" # an artefact older than this is stale +NOW=$(date +%s) +fails=0; warns=0 + +# Colour only when attached to a terminal: this gets piped into files and, later, +# probably into a notification. +if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' +else R=''; Y=''; G=''; N=''; fi + +# Per-source verdicts, so each source can be reported independently. A single +# aggregate red light tells you backups are broken; it does not tell you which +# one, which is the thing you need at 3am. +declare -A SRC_FAIL SRC_MSG +CURRENT="" + +red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); + [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; } +yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } +ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; + [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; } + +# --- Reporting ------------------------------------------------------------- +# Each source gets its own Gatus external endpoint, plus one for the store +# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means +# report nowhere, which is valid: the exit code is still the whole answer. +PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}" +PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}" + +report() { + local key="$1" success="$2" message="$3" + [ -n "$PUSH_BASE" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true +} + +hours_since() { echo $(( (NOW - $1) / 3600 )); } + +# Pull the dump timestamp out of _YYYYmmdd_HHMMSS.. This is when +# the SOURCE produced it, which is the thing that actually matters: a source +# whose timer died still pulls "ok" forever, because yesterday's artefact is +# still sitting there. Checking only the pull would miss exactly that. +dump_epoch() { + local base ts + base=$(basename "$1") + ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1 + [ -n "$ts" ] || return 1 + date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null +} + +check_source() { + local name="$1" keep="$2" dir="$STORE/$1" + printf '\n%s\n' "== $name" + CURRENT="$name" + SRC_FAIL[$name]=0 + SRC_MSG[$name]="" + + [ -d "$dir" ] || { red "$name: no directory $dir"; return; } + + local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l) + [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; } + + local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l) + [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these" + + local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1) + local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p) + + # 1. Did the SOURCE dump recently? + local de; de=$(dump_epoch "$newest") + if [ -z "${de:-}" ]; then + yell "$name: cannot parse a dump timestamp from $(basename "$newest")" + else + local dh; dh=$(hours_since "$de") + if [ "$dh" -lt 0 ]; then + # A future-dated artefact would otherwise stay "fresh" forever and the + # staleness check would never fire again — the exact silent failure this + # script exists to catch. + red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?" + elif [ "$dh" -gt "$MAX_AGE_H" ]; then + red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run" + else + ok "$name: dumped ${dh}h ago" + fi + fi + + # 2. Did the PULL bring it over recently? + local ph; ph=$(hours_since "$(stat -c %Y "$newest")") + if [ "$ph" -gt "$MAX_AGE_H" ]; then + red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)" + else + ok "$name: pulled ${ph}h ago" + fi + + # 3. Is it plausibly a real backup? + local sz; sz=$(stat -c %s "$newest") + if [ "$sz" -eq 0 ]; then + red "$name: newest artefact is ZERO bytes" + elif [ -n "$prev" ]; then + local psz; psz=$(stat -c %s "$prev") + if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then + # Not automatically wrong: headscale legitimately shrank 297K -> 20K when + # a clean stop checkpointed its write-ahead log into the database. + yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect" + else + ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)" + fi + else + ok "$name: $(numfmt --to=iec "$sz") (first artefact)" + fi + + # 4. Is retention pruning? Allow generous slack for multiple dumps per day. + if [ "$n" -gt $(( keep * 3 + 10 )) ]; then + yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working" + fi +} + +echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" +echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." + +# --- the pull job itself --- +# Reserved key, reported as backup-store_pull-job. The store's own machinery is +# a different alarm from any one source being stale, and it is a LEADING +# indicator where the per-source checks are lagging ones: those only fire once +# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer, +# a failed pull job or a filling disk are all visible here immediately, and they +# name the cause instead of showing six stale sources with no explanation. +CURRENT="__store" +SRC_FAIL[__store]=0 +SRC_MSG[__store]="" +printf '\n%s\n' "== pull-backups.service" +result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) +status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) +when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null) +[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})" +if [ -n "$when" ]; then + wh=$(hours_since "$(date -d "$when" +%s)") + [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?" +fi +systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ + && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \ + || red "pull-backups.timer is NOT enabled" + +# --- each source --- +{% for src in backup_store_sources %} +check_source "{{ src.name }}" {{ src.retention_days }} +CURRENT="" +# The store key is what Gatus computes from group+name: sanitize("backup-store") +# + "_" + sanitize("{{ src.name }}"). +report "backup-store_{{ src.name }}" \ + "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[{{ src.name }}]:-no result}" +{% endfor %} + +# --- capacity --- +CURRENT="__store" +printf '\n%s\n' "== disk" +use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') +avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') +if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" +elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" +else ok "store is ${use}% full, ${avail} free"; fi + +CURRENT="" +report "backup-store_pull-job" \ + "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[__store]:-no result}" + +printf '\n%s\n' "-----" +if [ "$fails" -gt 0 ]; then + echo "RESULT: $fails failure(s), $warns warning(s)" + echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager" + exit 1 +fi +if [ "$warns" -gt 0 ]; then + echo "RESULT: all checks passed, $warns warning(s)" +else + echo "RESULT: all checks passed" +fi +exit 0 diff --git a/ansible/roles/backup_store/templates/check-backups.timer.j2 b/ansible/roles/backup_store/templates/check-backups.timer.j2 new file mode 100644 index 0000000..ba5c5a8 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run the backup verification after the nightly pull +Requires=check-backups.service + +[Timer] +OnCalendar={{ backup_store_check_on_calendar }} +# Run a missed occurrence on the next boot rather than skipping the day. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 index 267900c..1db7d9c 100644 --- a/ansible/roles/backup_store/templates/pull-backups.sh.j2 +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -22,7 +22,12 @@ pull_{{ src.name | replace('-', '_') }}() { local dir="${STORE}/{{ src.name }}" mkdir -p "$dir" log "Pulling {{ src.name }} from {{ src.source }}..." - if rsync -az --timeout=120 \ + # --exclude '*.partial': a dump that died mid-write leaves one behind, owned + # root:root 0600 because the chown only happens after a successful mv. Without + # this exclude the pull account cannot read it and rsync fails for the WHOLE + # source — so one failed dump would silently block every subsequent pull of + # that service. An incomplete artefact is never worth transferring anyway. + if rsync -az --timeout=120 --exclude '*.partial' \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ "{{ src.source }}" "$dir/"; then log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" diff --git a/ansible/roles/bitcoin_knots/README.md b/ansible/roles/bitcoin_knots/README.md new file mode 100644 index 0000000..8e35900 --- /dev/null +++ b/ansible/roles/bitcoin_knots/README.md @@ -0,0 +1,85 @@ +# `bitcoin_knots` + +Builds Bitcoin Knots from source with PGP + SHA256 verification of the release +tarball, runs it as a full node on `knots-box`, and keeps a health check on a +systemd timer. The second play in the calling playbook publishes the P2P port +from the edge host via `socket_proxy`. + +Converted from `deploy_bitcoin_knots_playbook.yml` (892 lines) under Plan 6. The +playbook is now 40 lines. + +## The build is guarded; the chain is never touched + +`build.yml` is 32 tasks, every one carrying +`when: not bitcoind_binary_exists.stat.exists`. On a host that already has the +binary the whole download / verify / 30-60 minute compile skips — **including the +two `state: absent` deletions**, which target `/opt/bitcoin-knots/source` and the +extracted build directory. + +The chain lives elsewhere and nothing here touches it: + +| | | +|---|---| +| `bitcoin_knots_dir` | `/opt/bitcoin-knots` — build tree, safe to delete | +| `bitcoin_data_dir` | `/var/lib/bitcoin` — config, logs, wallets | +| `bitcoin_large_data_dir` | `/mnt/knots_data` — **~875 GB of blockchain** | + +The signature-verification tasks are the security control of this role. They are +copied verbatim; do not "simplify" them. + +## ⚠ This node is half of the mining setup + +`bitcoin.conf` carries a DATUM Gateway section that was hand-added on the node +and was **missing from the playbook's template**: + +```ini +blockmaxsize=3985000 +blockmaxweight=3985000 +blocknotify=killall -USR1 datum_gateway +maxmempool=1000 +blockreconstructionextratxn=1000000 +``` + +`blocknotify` is how `datum_gateway` learns a new block landed. Running the old +playbook would have deleted all of it, and solo mining would have carried on +grinding against a stale template — a silent failure that costs money rather +than raising an error. The template now carries it behind +`bitcoin_datum_gateway_enabled`. + +**bitcoin-knots and datum-gateway are one system, not two services.** Changing +either config means thinking about both. + +## The restart handler, and why exactness matters now + +The hand-written `Restart bitcoind` handler carried +`when: uptime_kuma_enabled | default(false)`, so it had been inert since the +decommissioning: `bitcoin.conf` and the systemd unit both notify it and neither +could restart anything. A config change applied to disk, reported success, and +never took effect. + +It is ungated here — which raises the bar for the template. **Any** residual +difference between the template and the live file, down to a trailing newline, +means the task reports `changed` and restarts a Bitcoin node on every run. It +took four rounds of `--check --diff` to reach `changed=0`: the DATUM section, an +explanatory comment that was rendering into the deployed file (now a `{# #}` +Jinja comment), a `# Pruning (optional)` comment the live file had, and one +trailing blank line. + +## `dbcache` + +Computed as 90% of RAM unless `bitcoin_dbcache_mb_override` is set. The live node +was hand-tuned to **200 MB**; the calculation produces 3528. As with fulcrum, +`set_fact` outranks role defaults, so the *calculation* honours the override — a +value pinned only in `defaults/` is silently ignored. + +## Monitoring: one variable, no product knowledge + +The check tests bitcoind's RPC and records the answer in its exit code, which +systemd keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. Set +`healthcheck_push_url` to report anywhere that accepts an HTTP ping. + +The timer had last fired **2026-08-09** while still reporting `active` and +`enabled` — the same `OnBootSec` + `OnUnitActiveSec` dead chain as fulcrum, where +nothing re-arms it if the service does not run in a given boot. The role runs the +check once after enabling, which both smoke-tests it and supplies the reference +the timer schedules from. diff --git a/ansible/roles/bitcoin_knots/defaults/main.yml b/ansible/roles/bitcoin_knots/defaults/main.yml new file mode 100644 index 0000000..1045e77 --- /dev/null +++ b/ansible/roles/bitcoin_knots/defaults/main.yml @@ -0,0 +1,78 @@ +# Bitcoin Knots Configuration Variables + +# Version - REQUIRED: Specify exact version/tag to build +# The only version string. There used to be a second, v-prefixed copy +# (bitcoin_knots_version) that nothing read - two hand-maintained copies of one +# fact, with nothing keeping them in step. +bitcoin_knots_version_short: "29.2.knots20251110" + +# Directories +bitcoin_knots_dir: /opt/bitcoin-knots +bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" +bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets +bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) +bitcoin_conf_dir: /etc/bitcoin + +# Network +bitcoin_rpc_port: 8332 +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +bitcoin_p2p_port: 8333 +bitcoin_rpc_bind: "0.0.0.0" + +# Build options +bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores +bitcoin_build_prefix: /usr/local + +# Configuration options +bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) +bitcoin_max_connections: 125 +# dbcache will be calculated as 90% of host RAM automatically in playbook + +# ZMQ Configuration +bitcoin_zmq_enabled: true +bitcoin_zmq_bind: "tcp://0.0.0.0" +bitcoin_zmq_port_rawblock: 28332 +bitcoin_zmq_port_rawtx: 28333 +bitcoin_zmq_port_hashblock: 28334 +bitcoin_zmq_port_hashtx: 28335 + +# Service user +bitcoin_user: bitcoin +bitcoin_group: bitcoin + +# --- Health check ---------------------------------------------------------- +# Checks bitcoind RPC and records the answer in its exit code, which systemd +# keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. +# +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping +# works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" + +# --- Logging ---------------------------------------------------------------- +# The live node logs to a file. Set to "" to use printtoconsole=1 (journald). +bitcoin_logfile: "{{ bitcoin_data_dir }}/debug.log" + +# --- dbcache ---------------------------------------------------------------- +# Computed as 90% of RAM unless this is set. The live node was hand-tuned to +# 200 MB; the calculation would have produced 3528. As with fulcrum, note that +# set_fact outranks role defaults, so the CALCULATION has to honour this - a +# value pinned only in defaults/ is silently ignored. +bitcoin_dbcache_mb_override: 200 + +# --- DATUM Gateway ---------------------------------------------------------- +# This node feeds block templates to datum_gateway on knots-box. These settings +# were hand-added to bitcoin.conf and were missing from the template, so a +# playbook run would have removed them and broken the mining setup. +bitcoin_datum_gateway_enabled: true +bitcoin_blockmaxsize: 3985000 +bitcoin_blockmaxweight: 3985000 +bitcoin_blocknotify: "killall -USR1 datum_gateway" +bitcoin_maxmempool: 1000 +bitcoin_blockreconstructionextratxn: 1000000 diff --git a/ansible/roles/bitcoin_knots/handlers/main.yml b/ansible/roles/bitcoin_knots/handlers/main.yml new file mode 100644 index 0000000..49ac219 --- /dev/null +++ b/ansible/roles/bitcoin_knots/handlers/main.yml @@ -0,0 +1,14 @@ +--- +# Ungated on purpose. The hand-written handler carried +# when: uptime_kuma_enabled | default(false) +# so it has been inert since the decommissioning. Two tasks notify it — +# bitcoin.conf and the systemd unit — and neither could actually restart +# bitcoind. A configuration change to a Bitcoin node therefore applied to disk, +# reported success, and silently never took effect. +# +# Restarting bitcoind re-opens the chainstate; it does not reindex. +- name: Restart bitcoind + systemd: + name: bitcoind + state: restarted + daemon_reload: yes diff --git a/ansible/roles/bitcoin_knots/tasks/build.yml b/ansible/roles/bitcoin_knots/tasks/build.yml new file mode 100644 index 0000000..5a1b900 --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/build.yml @@ -0,0 +1,222 @@ +--- +# Every task here is guarded by `when: not bitcoind_binary_exists.stat.exists`, +# so on a host that already has the binary the whole download / verify / build +# sequence skips — including the two `state: absent` deletions, which target +# /opt/bitcoin-knots/{source,bitcoin-} and never the chain data in +# /mnt/knots_data. +- name: Check if bitcoind binary already exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_binary_exists + changed_when: false + +- name: Install gnupg for signature verification + apt: + name: gnupg + state: present + when: not bitcoind_binary_exists.stat.exists + +- name: Import Luke Dashjr's Bitcoin Knots signing key + command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 + register: key_import + changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" + when: not bitcoind_binary_exists.stat.exists + failed_when: key_import.rc != 0 + +- name: Display imported key fingerprint + command: gpg --fingerprint 90C8019E36C2E964 + register: key_fingerprint + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Download SHA256SUMS file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + +- name: Download SHA256SUMS.asc signature file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + +- name: Verify PGP signature on SHA256SUMS file + command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS + register: sha256sums_verification + changed_when: false + failed_when: false # Don't fail here - check for 'Good signature' in next task + when: not bitcoind_binary_exists.stat.exists + + +- name: Display SHA256SUMS verification result + debug: + msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" + when: not bitcoind_binary_exists.stat.exists + +- name: Fail if SHA256SUMS signature verification failed + fail: + msg: "SHA256SUMS signature verification failed. Aborting build." + when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) + +- name: Remove any existing tarball to force fresh download + file: + path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + state: absent + when: not bitcoind_binary_exists.stat.exists + +- name: Download Bitcoin Knots source tarball + get_url: + url: "{{ bitcoin_source_tarball_url }}" + dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + mode: '0644' + validate_certs: yes + force: yes + when: not bitcoind_binary_exists.stat.exists + +- name: Calculate SHA256 checksum of downloaded tarball + command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + register: tarball_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Extract expected checksum from SHA256SUMS file + shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' + register: expected_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + failed_when: expected_checksum.stdout == "" + +- name: Display checksum comparison + debug: + msg: + - "Expected: {{ expected_checksum.stdout | trim }}" + - "Actual: {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists + +- name: Verify tarball checksum matches SHA256SUMS + fail: + msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] + +- name: Remove existing source directory if it exists (to force fresh extraction) + file: + path: "{{ bitcoin_knots_source_dir }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + +- name: Remove extracted directory if it exists (from previous runs) + file: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + +- name: Extract verified source tarball + unarchive: + src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + dest: "{{ bitcoin_knots_dir }}" + remote_src: yes + when: not bitcoind_binary_exists.stat.exists + +- name: Check if extracted directory exists + stat: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + register: extracted_dir_stat + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Rename extracted directory to expected name + command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" + when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists + +- name: Check if CMakeLists.txt exists + stat: + path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" + register: cmake_exists + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Create CMake build directory + file: + path: "{{ bitcoin_knots_source_dir }}/build" + state: directory + mode: '0755' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + +- name: Configure Bitcoin Knots build with CMake + command: > + cmake + -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} + -DBUILD_BITCOIN_WALLET=OFF + -DCMAKE_BUILD_TYPE=Release + -DWITH_ZMQ=ON + .. + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: configure_result + changed_when: true + +- name: Verify CMake enabled ZMQ + shell: | + set -e + cd "{{ bitcoin_knots_source_dir }}/build" + cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: zmq_check + changed_when: false + +- name: Fail if CMakeLists.txt not found + fail: + msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." + when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) + +- name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) + command: cmake --build . -j{{ bitcoin_build_jobs }} + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + async: 3600 + poll: 0 + register: build_result + changed_when: true + +- name: Check build status + async_status: + jid: "{{ build_result.ansible_job_id }}" + register: build_job_result + until: build_job_result.finished + retries: 120 + delay: 60 + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined + +- name: Fail if build failed + fail: + msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) + +- name: Install Bitcoin Knots binaries + command: cmake --install . + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + changed_when: true + +- name: Verify bitcoind binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_installed + changed_when: false + +- name: Verify bitcoin-cli binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" + register: bitcoin_cli_installed + changed_when: false + +- name: Fail if binaries not found + fail: + msg: "Bitcoin Knots binaries not found after installation" + when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists diff --git a/ansible/roles/bitcoin_knots/tasks/configure.yml b/ansible/roles/bitcoin_knots/tasks/configure.yml new file mode 100644 index 0000000..52fd3fd --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/configure.yml @@ -0,0 +1,20 @@ +--- +# Ownership copied verbatim from the playbook this replaces; verified +# mechanically against `git show HEAD:` rather than retyped from memory. +- name: Create bitcoin.conf configuration file + ansible.builtin.template: + src: bitcoin.conf.j2 + dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0640' + notify: Restart bitcoind + +- name: Create systemd service file for bitcoind + ansible.builtin.template: + src: bitcoind.service.j2 + dest: /etc/systemd/system/bitcoind.service + owner: root + group: root + mode: '0644' + notify: Restart bitcoind diff --git a/ansible/roles/bitcoin_knots/tasks/healthcheck.yml b/ansible/roles/bitcoin_knots/tasks/healthcheck.yml new file mode 100644 index 0000000..c6c8db3 --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/healthcheck.yml @@ -0,0 +1,56 @@ +--- +# Everything here answers "is bitcoind healthy" and records the answer. The +# Uptime Kuma specifics that used to follow — an embedded Python script creating +# monitors over the API, a /tmp credentials file, push-URL extraction and a +# systemd Environment= rewrite — are gone. Where it reports is now one variable, +# healthcheck_push_url. See the role README. +- name: Install curl for health check script + apt: + name: curl + state: present + +- name: Create Bitcoin Knots health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create systemd service for Bitcoin Knots health check + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/bitcoin-knots-healthcheck.service + owner: root + group: root + mode: "0600" + +- name: Create systemd timer for Bitcoin Knots health check + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer + owner: root + group: root + mode: '0644' + +- name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + +- name: Enable and restart the Bitcoin Knots health check timer + systemd: + name: bitcoin-knots-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Runs the check once, which is both a smoke test and the thing that actually +# arms the timer. This timer is OnBootSec + OnUnitActiveSec with no OnCalendar: +# OnBootSec elapses once, and OnUnitActiveSec needs the SERVICE to have run this +# boot to have anything to schedule from. Restarting the timer does not supply +# that reference; running the service does. The live timer had last fired on +# 2026-08-09 while still reporting `active` and `enabled`. +- name: Run the Bitcoin Knots health check once to arm the timer + command: systemctl start bitcoin-knots-healthcheck.service + changed_when: false diff --git a/ansible/roles/bitcoin_knots/tasks/install.yml b/ansible/roles/bitcoin_knots/tasks/install.yml new file mode 100644 index 0000000..3003b6e --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/install.yml @@ -0,0 +1,104 @@ +--- +- name: Calculate dbcache as a share of system RAM + set_fact: + bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" + when: bitcoin_dbcache_mb_override | string | length == 0 + +- name: Use the explicit dbcache override + set_fact: + bitcoin_dbcache_mb: "{{ bitcoin_dbcache_mb_override }}" + when: bitcoin_dbcache_mb_override | string | length > 0 + changed_when: false + +- name: Display calculated dbcache value + debug: + msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" + + +- name: Install build dependencies + apt: + name: + - build-essential + - libtool + - autotools-dev + - automake + - pkg-config + - bsdmainutils + - python3 + - python3-pip + - libevent-dev + - libboost-system-dev + - libboost-filesystem-dev + - libboost-test-dev + - libboost-thread-dev + - libboost-chrono-dev + - libboost-program-options-dev + - libboost-dev + - libssl-dev + - libdb-dev + - libminiupnpc-dev + - libzmq3-dev + - libnatpmp-dev + - libsqlite3-dev + - git + - curl + - wget + - cmake + state: present + update_cache: yes + +- name: Create bitcoin group + group: + name: "{{ bitcoin_group }}" + system: yes + state: present + +- name: Create bitcoin user + user: + name: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ bitcoin_data_dir }}" + create_home: yes + state: present + +- name: Create bitcoin-knots directory + file: + path: "{{ bitcoin_knots_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Create bitcoin-knots source directory + file: + path: "{{ bitcoin_knots_source_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Create bitcoin data directory (for config, logs, wallets) + file: + path: "{{ bitcoin_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + +- name: Create bitcoin large data directory (for blockchain) + file: + path: "{{ bitcoin_large_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + +- name: Create bitcoin config directory + file: + path: "{{ bitcoin_conf_dir }}" + state: directory + owner: root + group: root + mode: '0755' diff --git a/ansible/roles/bitcoin_knots/tasks/main.yml b/ansible/roles/bitcoin_knots/tasks/main.yml new file mode 100644 index 0000000..ab869a7 --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/main.yml @@ -0,0 +1,8 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: build.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/bitcoin_knots/tasks/service.yml b/ansible/roles/bitcoin_knots/tasks/service.yml new file mode 100644 index 0000000..6f6149e --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/service.yml @@ -0,0 +1,37 @@ +--- +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start bitcoind service + systemd: + name: bitcoind + enabled: yes + state: started + +- name: Wait for bitcoind RPC to be available + uri: + url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" + method: POST + body_format: json + body: + jsonrpc: "1.0" + id: "healthcheck" + method: "getblockchaininfo" + params: [] + user: "{{ bitcoin_rpc_user }}" + password: "{{ bitcoin_rpc_password }}" + status_code: 200 + timeout: 10 + register: rpc_check + until: rpc_check.status == 200 + retries: 30 + delay: 5 + ignore_errors: yes + +- name: Display RPC connection status + debug: + msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + +# ═════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 new file mode 100644 index 0000000..5277a3c --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 @@ -0,0 +1,67 @@ +# Bitcoin Knots Configuration +# Generated by Ansible + +# Data directory (blockchain storage) +datadir={{ bitcoin_large_data_dir }} + +# RPC Configuration +server=1 +rpcuser={{ bitcoin_rpc_user }} +rpcpassword={{ bitcoin_rpc_password }} +rpcbind={{ bitcoin_rpc_bind }} +rpcport={{ bitcoin_rpc_port }} +rpcallowip=0.0.0.0/0 + +# Network Configuration +listen=1 +port={{ bitcoin_p2p_port }} +maxconnections={{ bitcoin_max_connections }} + +# Performance +dbcache={{ bitcoin_dbcache_mb }} + +# Transaction Index (optional) +{% if bitcoin_enable_txindex %} +txindex=1 +{% endif %} + +{# The live node carries this comment and the template never produced it, so a + run would have silently deleted it. Harmless in itself, but matching it keeps + this task at `ok` - which means any future `changed` here is a real signal + rather than known noise. #} +# Pruning (optional) + +# Logging +logtimestamps=1 +{% if bitcoin_logfile %} +logfile={{ bitcoin_logfile }} +{% else %} +printtoconsole=1 +{% endif %} + +# ZMQ Configuration +{% if bitcoin_zmq_enabled | default(false) %} +zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} +zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} +zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} +zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} +{% endif %} + +# Security +disablewallet=1 +{% if bitcoin_datum_gateway_enabled %} + +{# These were hand-added on the node and were NOT in this template, so running + the playbook would have stripped them. blocknotify is how datum_gateway + learns a new block landed; without it solo mining keeps grinding on a stale + template - a silent failure that costs money rather than raising an error. + Kept as a Jinja comment so the explanation stays in the repo and out of the + deployed config. #} +# Specific for DATUM gateway +blockmaxsize={{ bitcoin_blockmaxsize }} +blockmaxweight={{ bitcoin_blockmaxweight }} +blocknotify={{ bitcoin_blocknotify }} +maxmempool={{ bitcoin_maxmempool }} +blockreconstructionextratxn={{ bitcoin_blockreconstructionextratxn }} + +{% endif %} diff --git a/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 b/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 new file mode 100644 index 0000000..ac1140f --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Bitcoin Knots daemon +After=network.target + +[Service] +Type=simple +User={{ bitcoin_user }} +Group={{ bitcoin_group }} +ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf +Restart=always +RestartSec=10 +TimeoutStopSec=600 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 new file mode 100644 index 0000000..9df056e --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Bitcoin Knots Health Check +After=network.target bitcoind.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..538b8dd --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 @@ -0,0 +1,69 @@ +#!/bin/bash +# Bitcoin Knots health check — managed by Ansible (roles/bitcoin_knots) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed bitcoin-knots-healthcheck.service +# Reporting anywhere else is optional and generic. +# +# + +RPC_HOST="{{ bitcoin_rpc_bind }}" +RPC_PORT={{ bitcoin_rpc_port }} +RPC_USER="{{ bitcoin_rpc_user }}" +RPC_PASSWORD="{{ bitcoin_rpc_password }}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +# Check if bitcoind RPC is responding +check_bitcoind() { + local response + response=$(curl -s --max-time 30 \ + --user "${RPC_USER}:${RPC_PASSWORD}" \ + --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ + --header 'Content-Type: application/json' \ + "http://${RPC_HOST}:${RPC_PORT}" 2>&1) + + if [ $? -eq 0 ]; then + # Check if response contains a non-null error + # Successful responses have "error": null, failures have "error": {...} + if echo "$response" | grep -q '"error":null\|"error": null'; then + return 0 + else + return 1 + fi + else + return 1 + fi +} + +report() { + local status=$1 + local msg=$2 + + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + # URL encode spaces in message + local encoded_msg="${msg// /%20}" + + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + if ! curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${encoded_msg}"; then + return 1 + fi +} + +# Main health check +if check_bitcoind; then + report "up" "OK" + exit 0 +else + report "down" "bitcoind RPC not responding" + exit 1 +fi diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..d5857ab --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Bitcoin Knots Health Check Timer +Requires=bitcoind.service + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/caddy_site/defaults/main.yml b/ansible/roles/caddy_site/defaults/main.yml index cafa3dc..0a59906 100644 --- a/ansible/roles/caddy_site/defaults/main.yml +++ b/ansible/roles/caddy_site/defaults/main.yml @@ -15,7 +15,8 @@ caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"} # expression for the username silently passes through as literal text. caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}] -# Placement. caddy_sites_dir comes from services_config.yml; this is the fallback. +# Placement. This is now the only definition of caddy_sites_dir - services_config.yml +# used to carry an identical copy, which was removed as redundant. caddy_sites_dir: /etc/caddy/sites-enabled # Rendered site files can carry credentials (basic_auth hashes), so --diff is diff --git a/ansible/roles/datum_gateway/README.md b/ansible/roles/datum_gateway/README.md new file mode 100644 index 0000000..cdf35e5 --- /dev/null +++ b/ansible/roles/datum_gateway/README.md @@ -0,0 +1,73 @@ +# `datum_gateway` + +Builds and runs [DATUM Gateway](https://github.com/OCEAN-xyz/datum_gateway), the +solo/pooled mining gateway, on `knots-box`. The calling playbook adds two more +plays on the edge host: the dashboard via `caddy_site`, and the public Stratum +port via `socket_proxy`. + +Converted from `deploy_datum_gateway_playbook.yml` (802 lines) under Plan 6. The +playbook is now 68 lines and keeps all three plays. + +## ⚠ This is half of a system + +The Bitcoin Knots node on the same host feeds this gateway through +`blocknotify=killall -USR1 datum_gateway` in `bitcoin.conf` — see +`roles/bitcoin_knots/README.md`, where that line was found to be missing from the +template entirely. **Changing either config means thinking about both.** + +Interrupting Stratum costs mining shares. Check before any run that restarts it: + +```bash +ss -tn state established '( sport = :23334 )' +``` + +## Two pieces of drift where the node was right + +The repo and the node had diverged on values that matter, and the deployment +would have applied the repo's: + +| | node (correct) | repo said | +|---|---|---| +| `datum_mining_address` | `bc1qvrj3g84…` | `bc1qdse9dsg…` | +| `pool_pass_workers` / `_full_users` | `false` | `true` | + +The address is the one that would have hurt: **it is where block rewards are +paid**, and unlike fulcrum and bitcoin-knots the `Restart datum-gateway` handler +here was *never* gated, so the change would have applied immediately rather than +sitting inert. Both corrected in the vault and defaults, with notes. + +Verify semantics rather than text when touching `config.json` — render it and +compare parsed JSON, because the live file is single-line and the template is +pretty-printed, so a textual diff is all noise: + +```python +json.load(open('live.json')) == json.load(open('rendered.json')) +``` + +## `config.json` holds real secrets — diff is suppressed + +The file carries `bitcoind.rpcpassword` and `api.admin_password`. `--diff` +prints rendered content, so the task sets `diff: false` by default; pass +`-e datum_reveal_config=true` to opt in. + +Note `pool_pass_workers` / `pool_pass_full_users` are **booleans**, not +passwords, despite the names — they control DATUM's pool-password passthrough. +`mining.pool_address` is a Bitcoin address and public by nature. + +## Expect `changed` on the compile every run + +`Configure cmake build` and `Compile datum_gateway` are bare `command:` tasks +with no `changed_when`, so they always report changed and always re-run. The +build is reproducible — `Install datum_gateway binary` sees identical content and +does not replace it, so the installed binary keeps its original timestamp — but +the compile itself is wasted work on every run. That is the idempotent floor, not +drift. + +## Monitoring: one variable, no product knowledge + +The check tests the gateway API and records the answer in its exit code, which +systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. Set +`healthcheck_push_url` to report anywhere accepting an HTTP ping. + +Unlike the other services here, only the health-check *timer* handler was gated +by `uptime_kuma_enabled`; the main deployment restart worked throughout. diff --git a/ansible/roles/datum_gateway/defaults/main.yml b/ansible/roles/datum_gateway/defaults/main.yml new file mode 100644 index 0000000..0c30aa7 --- /dev/null +++ b/ansible/roles/datum_gateway/defaults/main.yml @@ -0,0 +1,61 @@ +# DATUM Gateway Configuration Variables +# https://github.com/OCEAN-xyz/datum_gateway + +# Version - pin to a specific tag +datum_gateway_version: "v0.4.1beta" + +# Directories +datum_gateway_dir: /opt/datum-gateway +datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" +datum_gateway_config_dir: /etc/datum-gateway +datum_gateway_log_dir: /var/log/datum-gateway + +# Binary +datum_gateway_bin_path: /usr/local/bin/datum_gateway + +# Ports +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +datum_gateway_stratum_port: 23334 +datum_gateway_api_port: 7152 + +# Stratum settings +datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) + +# Service user +datum_gateway_user: datum +datum_gateway_group: datum + +# Build options +datum_gateway_build_jobs: 4 + +# Bitcoin node connection +# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. +# datum_bitcoin_rpc_url should include http:// and port. +datum_bitcoin_rpc_url: "http://127.0.0.1:8332" +# Note: bitcoin_rpc_user and bitcoin_rpc_password come from group_vars/all/vault.yml + +# Mining config +datum_coinbase_tag_primary: "DATUM" +datum_coinbase_tag_secondary: "BY ORDER OF BIP110" + +# Both false on the node; the vars file said true. Corrected 2026-09-13 to +# match reality, on the same basis as datum_mining_address: the running node +# is authoritative. These control DATUM's pool-password passthrough. +datum_pool_pass_workers: false +datum_pool_pass_full_users: false +datum_pooled_mining_only: true + + +# --- Health check ----------------------------------------------------------- +# Checks the DATUM Gateway API and records the answer in its exit code, which +# systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. +# +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/datum_gateway/handlers/main.yml b/ansible/roles/datum_gateway/handlers/main.yml new file mode 100644 index 0000000..b7cfe25 --- /dev/null +++ b/ansible/roles/datum_gateway/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: Restart datum-gateway + systemd: + name: datum-gateway + state: restarted + daemon_reload: yes + +# Ungated. This one carried `when: uptime_kuma_enabled | default(false)` while +# the main Restart datum-gateway handler above did not — so on this service the +# deployment restart worked and only the health-check timer restart was dead. +- name: Restart datum-gateway health check timer + systemd: + name: datum-gateway-healthcheck.timer + state: restarted + daemon_reload: yes diff --git a/ansible/roles/datum_gateway/tasks/configure.yml b/ansible/roles/datum_gateway/tasks/configure.yml new file mode 100644 index 0000000..463dec0 --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/configure.yml @@ -0,0 +1,25 @@ +--- +# Ownership copied verbatim from the playbook this replaces and verified +# mechanically against `git show HEAD:`. +- name: Write DATUM Gateway config.json + ansible.builtin.template: + src: config.json.j2 + dest: "{{ datum_gateway_config_dir }}/config.json" + owner: "{{ datum_gateway_user }}" + group: "{{ datum_gateway_group }}" + mode: '0640' + # config.json carries the bitcoind RPC password, the API admin password and + # the pool passwords. `--diff` prints rendered content, so running with --diff + # put all of them on the terminal and into any log capturing it. Suppressed by + # default; pass -e datum_reveal_config=true when you genuinely need the diff. + diff: "{{ datum_reveal_config | default(false) | bool }}" + notify: Restart datum-gateway + +- name: Create datum-gateway systemd service + ansible.builtin.template: + src: datum-gateway.service.j2 + dest: /etc/systemd/system/datum-gateway.service + owner: root + group: root + mode: '0644' + notify: Restart datum-gateway diff --git a/ansible/roles/datum_gateway/tasks/healthcheck.yml b/ansible/roles/datum_gateway/tasks/healthcheck.yml new file mode 100644 index 0000000..b5ccb90 --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/healthcheck.yml @@ -0,0 +1,50 @@ +--- +# Everything here answers "is DATUM Gateway healthy" and records the answer. The +# Uptime Kuma specifics that used to follow — an embedded Python script creating +# monitors over the API, a /tmp credentials file, a push-URL file read back and +# parsed, and a systemd Environment= rewrite — are gone. Where it reports is now +# one variable, healthcheck_push_url. +- name: Create DATUM Gateway health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/datum-gateway-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create datum-gateway health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/datum-gateway-healthcheck.service + owner: root + group: root + mode: "0600" + notify: Restart datum-gateway health check timer + +- name: Create datum-gateway health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/datum-gateway-healthcheck.timer + owner: root + group: root + mode: '0644' + notify: Restart datum-gateway health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated: enabling a timer is deployment, not monitoring. +- name: Enable and restart the datum-gateway health check timer + systemd: + name: datum-gateway-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Arms the timer and smoke-tests the check. See roles/bitcoin_knots/README.md for +# why restarting the timer alone is not enough with OnBootSec + OnUnitActiveSec. +- name: Run the DATUM Gateway health check once to arm the timer + command: systemctl start datum-gateway-healthcheck.service + changed_when: false diff --git a/ansible/roles/datum_gateway/tasks/install.yml b/ansible/roles/datum_gateway/tasks/install.yml new file mode 100644 index 0000000..61dcadb --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/install.yml @@ -0,0 +1,74 @@ +--- +- name: Install DATUM Gateway build dependencies + apt: + name: + - cmake + - build-essential + - git + - libjansson-dev + - libmicrohttpd-dev + - libsodium-dev + - libcurl4-openssl-dev + # Runtime-only (netcat for health check) + - netcat-openbsd + state: present + update_cache: yes + +# =========================================== +# System User and Directories +# =========================================== +- name: Create datum system user + user: + name: "{{ datum_gateway_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ datum_gateway_dir }}" + create_home: no + comment: "DATUM Gateway" + +- name: Create DATUM Gateway directories + file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ datum_gateway_group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + +# =========================================== +# Build from Source +# =========================================== +- name: Clone DATUM Gateway repository at {{ datum_gateway_version }} + git: + repo: https://github.com/OCEAN-xyz/datum_gateway.git + dest: "{{ datum_gateway_source_dir }}" + version: "{{ datum_gateway_version }}" + force: yes + register: git_clone + +- name: Configure cmake build + command: cmake . -DCMAKE_BUILD_TYPE=Release + args: + chdir: "{{ datum_gateway_source_dir }}" + +- name: Compile datum_gateway + command: make -j{{ datum_gateway_build_jobs }} + args: + chdir: "{{ datum_gateway_source_dir }}" + +- name: Install datum_gateway binary + copy: + src: "{{ datum_gateway_source_dir }}/datum_gateway" + dest: "{{ datum_gateway_bin_path }}" + remote_src: yes + owner: root + group: root + mode: "0755" + notify: Restart datum-gateway + +# =========================================== +# Configuration diff --git a/ansible/roles/datum_gateway/tasks/main.yml b/ansible/roles/datum_gateway/tasks/main.yml new file mode 100644 index 0000000..57d9a45 --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/datum_gateway/tasks/service.yml b/ansible/roles/datum_gateway/tasks/service.yml new file mode 100644 index 0000000..c8581fd --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/service.yml @@ -0,0 +1,16 @@ +--- +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start datum-gateway + systemd: + name: datum-gateway + enabled: yes + state: started + +# =========================================== +# Health Check Script + Systemd Timer +# =========================================== +# ═════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/datum_gateway/templates/config.json.j2 b/ansible/roles/datum_gateway/templates/config.json.j2 new file mode 100644 index 0000000..e17a386 --- /dev/null +++ b/ansible/roles/datum_gateway/templates/config.json.j2 @@ -0,0 +1,35 @@ +{ + "bitcoind": { + "rpcuser": "{{ bitcoin_rpc_user }}", + "rpcpassword": "{{ bitcoin_rpc_password }}", + "rpcurl": "{{ datum_bitcoin_rpc_url }}", + "notify_fallback": true + }, + "stratum": { + "listen_port": {{ datum_gateway_stratum_port }}, + "vardiff_min": {{ datum_vardiff_min }} + }, + "mining": { + "pool_address": "{{ datum_mining_address }}", + "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", + "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" + }, + "api": { + "admin_password": "{{ datum_gateway_admin_password }}", + "listen_port": {{ datum_gateway_api_port }}, + "modify_conf": false + }, + "logger": { + "log_to_console": true, + "log_to_file": true, + "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", + "log_rotate_daily": true, + "log_level_console": 2, + "log_level_file": 1 + }, + "datum": { + "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, + "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, + "pooled_mining_only": {{ datum_pooled_mining_only | lower }} + } +} diff --git a/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 b/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 new file mode 100644 index 0000000..18f7a3c --- /dev/null +++ b/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 @@ -0,0 +1,22 @@ +[Unit] +Description=DATUM Gateway - Bitcoin Mining Gateway +Documentation=https://github.com/OCEAN-xyz/datum_gateway +After=network.target bitcoind.service +Wants=bitcoind.service + +[Service] +User={{ datum_gateway_user }} +Group={{ datum_gateway_group }} +Type=simple +ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +# Prevent config from being read by other users +ReadWritePaths={{ datum_gateway_log_dir }} +ReadOnlyPaths={{ datum_gateway_config_dir }} + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.service.j2 b/ansible/roles/datum_gateway/templates/healthcheck.service.j2 new file mode 100644 index 0000000..e21672e --- /dev/null +++ b/ansible/roles/datum_gateway/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=DATUM Gateway Health Check +After=network.target datum-gateway.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 b/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..43ec6b2 --- /dev/null +++ b/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 @@ -0,0 +1,39 @@ +#!/bin/bash +# DATUM Gateway health check — managed by Ansible (roles/datum_gateway) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed datum-gateway-healthcheck.service +# Reporting anywhere else is optional and generic. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +STRATUM_PORT={{ datum_gateway_stratum_port }} + +check_datum() { + # Service must be active and stratum port must be listening + systemctl is-active --quiet datum-gateway && \ + nc -z 127.0.0.1 "${STRATUM_PORT}" +} + +report() { + local status=$1 + local msg=$2 + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true +} + +if check_datum; then + report "up" "OK" + exit 0 +else + report "down" "DATUM Gateway not responding" + exit 1 +fi diff --git a/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 b/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..c14d1d2 --- /dev/null +++ b/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=DATUM Gateway Health Check Timer + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/forgejo_runner/README.md b/ansible/roles/forgejo_runner/README.md new file mode 100644 index 0000000..e7f761b --- /dev/null +++ b/ansible/roles/forgejo_runner/README.md @@ -0,0 +1,59 @@ +# `forgejo_runner` + +Installs and runs a Forgejo Actions runner, registers it with the Forgejo +instance, and keeps a health check on a systemd timer. + +Converted from `deploy_forgejo_runner_playbook.yml` (409 lines) under Plan 6. +The playbook is now 16 lines. + +## Phases + +`tasks/main.yml` imports five files in order: + +| | | +|---|---| +| `prerequisites.yml` | Docker must be present | +| `install.yml` | binary, system user, working directory | +| `configure.yml` | config file, registration with the instance | +| `service.yml` | systemd unit, start, assert it came up | +| `healthcheck.yml` | check script, unit, timer | + +`import_tasks`, not `include_tasks` — static imports are visible to +`--list-tasks`, which is how the conversion was verified against the playbook it +replaced. + +## Monitoring: one variable, no product knowledge + +This role contains **nothing specific to any monitoring system**. What used to +be here — an ~80-line embedded Python script creating monitors over the Uptime +Kuma API, a `/tmp` credentials file, token extraction, a systemd `Environment=` +rewrite, and 8 `when: uptime_kuma_enabled` guards — is gone. + +What remains answers the actual question, *is this service healthy*, and records +it two ways: + +- **the exit code**, which systemd keeps: `systemctl is-failed + forgejo-runner-healthcheck.service` is a complete answer with no monitoring + system involved at all; +- **a log file** at `{{ healthcheck_log_file }}`. + +To report health somewhere, set one variable: + +```yaml +healthcheck_push_url: "https://example/api/push/TOKEN" +``` + +Any endpoint accepting an HTTP ping works. Empty (the default) means check, log, +exit honestly, report nowhere — which is also the right setting for a *pull*-based +monitor like Prometheus' textfile collector, since that reads unit state instead. + +The push URL is a credential (anyone holding it can forge an "up"), so callers +pass it from the vault rather than committing it. + +## One behaviour change, deliberate + +`Assert runner is running` used to be guarded by `uptime_kuma_enabled`, so it +never ran. It is not a monitoring task — it is the deployment checking its own +work — and the deprecation banner swept it up by mistake. It is ungated here, +which means a runner that fails to start now fails the play instead of +deploying "successfully" in silence. diff --git a/ansible/roles/forgejo_runner/defaults/main.yml b/ansible/roles/forgejo_runner/defaults/main.yml new file mode 100644 index 0000000..aba6b61 --- /dev/null +++ b/ansible/roles/forgejo_runner/defaults/main.yml @@ -0,0 +1,41 @@ +--- +# Binary +forgejo_runner_version: "6.3.1" +forgejo_runner_arch: "linux-amd64" +forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" +forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" + +# Runtime +forgejo_runner_user: "runner" +forgejo_runner_dir: "/opt/forgejo-runner" +forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" +forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" + +# The Forgejo instance this runner registers with. +forgejo_instance_url: "https://forgejo.contrapeso.xyz" +# forgejo_runner_registration_token comes from the vault. + +# --- Health check ----------------------------------------------------------- +# The check answers "is this service healthy" and records the answer two ways: +# a log file, and its own exit code. The exit code is the durable artefact — +# systemd stores it, so `systemctl is-failed forgejo-runner-healthcheck.service` +# answers the question with no monitoring system involved at all. +healthcheck_interval_seconds: 60 +healthcheck_script_dir: /opt/forgejo-runner-healthcheck +healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" +healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" +healthcheck_service_name: forgejo-runner-healthcheck + +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. +# +# Empty means "check, log, exit honestly, report nowhere". Set it to any URL +# that accepts an HTTP ping and the check will report there. Nothing in this +# role is specific to a particular monitoring product: the Uptime Kuma API +# calls, monitor creation and token handling that used to live here are gone. +# +# A pull-based monitor (Prometheus node_exporter textfile, say) needs this left +# empty — it reads the systemd unit state instead. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/forgejo_runner/tasks/configure.yml b/ansible/roles/forgejo_runner/tasks/configure.yml new file mode 100644 index 0000000..ab96d52 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/configure.yml @@ -0,0 +1,43 @@ +--- +- name: Check if config already exists + stat: + path: "{{ forgejo_runner_config_path }}" + register: config_stat + +- name: Generate default config + shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not config_stat.stat.exists + +- name: Set config file ownership + file: + path: "{{ forgejo_runner_config_path }}" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not config_stat.stat.exists + +# ── 6. Register runner ───────────────────────────────────────────── +- name: Check if runner is already registered + stat: + path: "{{ forgejo_runner_dir }}/.runner" + register: runner_stat + +- name: Register runner with Forgejo instance + command: > + {{ forgejo_runner_bin_path }} register --no-interactive + --instance {{ forgejo_instance_url }} + --token {{ forgejo_runner_registration_token }} + --name forgejo-runner-box + --labels "{{ forgejo_runner_labels }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not runner_stat.stat.exists + +- name: Set runner registration file ownership + file: + path: "{{ forgejo_runner_dir }}/.runner" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not runner_stat.stat.exists + diff --git a/ansible/roles/forgejo_runner/tasks/healthcheck.yml b/ansible/roles/forgejo_runner/tasks/healthcheck.yml new file mode 100644 index 0000000..42fbaf7 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/healthcheck.yml @@ -0,0 +1,73 @@ +--- +# Everything here answers "is the service healthy" and records the answer. +# The Uptime Kuma specifics that used to surround it — an embedded Python script +# that created monitors over the API, a /tmp credentials file, token extraction, +# and a systemd Environment= rewrite — are gone. What reports where is now one +# variable, healthcheck_push_url. See the role README. +- name: Create healthcheck script directory + ansible.builtin.file: + path: "{{ healthcheck_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Create forgejo-runner healthcheck script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ healthcheck_script_path }}" + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create healthcheck systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" + owner: root + group: root + mode: "0600" + +- name: Create healthcheck systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" + owner: root + group: root + mode: '0644' + +- name: Reload systemd for healthcheck units + systemd: + daemon_reload: yes + +- name: Enable and start healthcheck timer + systemd: + name: "{{ healthcheck_service_name }}.timer" + enabled: yes + state: started + +- name: Test healthcheck script + command: "{{ healthcheck_script_path }}" + register: healthcheck_test + changed_when: false + +- name: Verify healthcheck script works + assert: + that: + - healthcheck_test.rc == 0 + fail_msg: "Healthcheck script failed to execute properly" + +- name: Display deployment summary + debug: + msg: | + Forgejo Runner deployed successfully! + + Runner Name: forgejo-runner-box + Instance: {{ forgejo_instance_url }} + Working Directory: {{ forgejo_runner_dir }} + Service: forgejo-runner.service ({{ runner_active.stdout }}) + + Healthcheck Monitor: {{ healthcheck_service_name }} + Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s + Reporting to: {{ healthcheck_push_url | default('', true) | regex_replace('/api/push/.*', '/api/push/***') | default('(nowhere - set healthcheck_push_url)', true) }} diff --git a/ansible/roles/forgejo_runner/tasks/install.yml b/ansible/roles/forgejo_runner/tasks/install.yml new file mode 100644 index 0000000..b95ab4c --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/install.yml @@ -0,0 +1,27 @@ +--- +- name: Download forgejo-runner binary + get_url: + url: "{{ forgejo_runner_url }}" + dest: "{{ forgejo_runner_bin_path }}" + mode: '0755' + +# ── 3. Create runner system user ─────────────────────────────────── +- name: Create runner system user + user: + name: "{{ forgejo_runner_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ forgejo_runner_dir }}" + create_home: no + groups: docker + append: yes + comment: 'Forgejo Runner' + +# ── 4. Create working directory ──────────────────────────────────── +- name: Create forgejo-runner working directory + file: + path: "{{ forgejo_runner_dir }}" + state: directory + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + mode: '0750' diff --git a/ansible/roles/forgejo_runner/tasks/main.yml b/ansible/roles/forgejo_runner/tasks/main.yml new file mode 100644 index 0000000..aa71001 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/main.yml @@ -0,0 +1,9 @@ +--- +# import_tasks, not include_tasks: these are unconditional phases, and static +# imports are visible to `--list-tasks`. That matters because the task list is +# how this refactor was verified against the playbook it replaced. +- ansible.builtin.import_tasks: prerequisites.yml +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/forgejo_runner/tasks/prerequisites.yml b/ansible/roles/forgejo_runner/tasks/prerequisites.yml new file mode 100644 index 0000000..d441709 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/prerequisites.yml @@ -0,0 +1,12 @@ +--- +- name: Check if Docker is installed + command: docker --version + register: docker_check + changed_when: false + failed_when: docker_check.rc != 0 + +- name: Fail if Docker is not available + assert: + that: + - docker_check.rc == 0 + fail_msg: "Docker is required for forgejo-runner but is not installed" diff --git a/ansible/roles/forgejo_runner/tasks/service.yml b/ansible/roles/forgejo_runner/tasks/service.yml new file mode 100644 index 0000000..f971e52 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/service.yml @@ -0,0 +1,33 @@ +--- +- name: Create forgejo-runner systemd service + ansible.builtin.template: + src: forgejo-runner.service.j2 + dest: /etc/systemd/system/forgejo-runner.service + owner: root + group: root + mode: '0644' + +- name: Reload systemd + systemd: + daemon_reload: yes + +- name: Enable and start forgejo-runner service + systemd: + name: forgejo-runner + enabled: yes + state: started + +- name: Verify forgejo-runner is active + command: systemctl is-active forgejo-runner + register: runner_active + changed_when: false + +# Ungated on purpose. This was previously guarded by `uptime_kuma_enabled`, but +# it is not a monitoring task — it is the deployment asserting its own success. +# The deprecation banner swept it up along with the Kuma plumbing, which meant a +# broken runner deployed "successfully" and silently. +- name: Assert runner is running + assert: + that: + - runner_active.stdout == "active" + fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" diff --git a/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 b/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 new file mode 100644 index 0000000..d3db25d --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Forgejo Runner +Documentation=https://forgejo.org/docs/latest/admin/actions/ +After=docker.service +Requires=docker.service + +[Service] +Type=simple +User={{ forgejo_runner_user }} +Group={{ forgejo_runner_user }} +WorkingDirectory={{ forgejo_runner_dir }} +ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 new file mode 100644 index 0000000..173f42b --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Forgejo Runner Healthcheck +After=network.target + +[Service] +Type=oneshot +ExecStart={{ healthcheck_script_path }} +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +User=root +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..9aa5ac1 --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 @@ -0,0 +1,47 @@ +#!/bin/bash +# Forgejo Runner healthcheck — managed by Ansible (roles/forgejo_runner) +# +# Answers "is forgejo-runner healthy" and records it two ways: this log, and the +# exit code. The exit code is the durable artefact — systemd keeps it, so +# systemctl is-failed {{ healthcheck_service_name }}.service +# answers the question with no monitoring system involved. +# +# Reporting is optional and generic: if a push URL is configured it also pings +# it. Nothing here knows or cares which monitoring product is on the other end. + +LOG_FILE="{{ healthcheck_log_file }}" +# Read from the environment rather than templated in, so the unit file is the +# only place the token lives and the script is not secret. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Gatus external endpoint: a POST with a bearer token and success=true|false. +# +# This used to report ONLY success - it exited before pushing when the runner +# was down - so a failure was invisible until the heartbeat window expired. +# Reporting the failure is the whole point of having a check. +report() { + local ok="$1" msg="$2" + [ -n "$PUSH_URL" ] || return 0 + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${ok}&error=${msg// /%20}" || true +} + +main() { + if ! systemctl is-active --quiet forgejo-runner; then + log_message "ERROR: forgejo-runner is not active" + report false "forgejo-runner is not active" + exit 1 + fi + + log_message "forgejo-runner is active" + report true "active" + exit 0 +} + +main diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..4cba51a --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run Forgejo Runner Healthcheck every minute +Requires={{ healthcheck_service_name }}.service + +[Timer] +OnBootSec=30sec +OnUnitActiveSec={{ healthcheck_interval_seconds }}sec +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/fulcrum/README.md b/ansible/roles/fulcrum/README.md new file mode 100644 index 0000000..ddefabb --- /dev/null +++ b/ansible/roles/fulcrum/README.md @@ -0,0 +1,65 @@ +# `fulcrum` + +Deploys [Fulcrum](https://github.com/cculianu/Fulcrum), an Electrum server +indexing the Bitcoin Knots node, on `fulcrum-box`. The second play in the +calling playbook publishes its SSL port from the edge host via `socket_proxy`. + +Converted from `deploy_fulcrum_playbook.yml` (685 lines) under Plan 6. The +playbook is now 33 lines. + +## The index is the expensive thing + +`{{ fulcrum_db_dir }}` is ~192 GB and takes days to rebuild. Nothing in this role +touches it beyond `state: directory` with the ownership it already has +(`fulcrum:fulcrum 0755`). Restarting Fulcrum re-opens the database; it does not +reindex. + +## Three things this conversion fixed, all pre-existing + +**`bitcoind` pointed at the wrong machine.** The vars file carried +`bitcoin_rpc_host: "192.168.1.140"` commented "IP of knots_box_local", but `.140` +is **fulcrum-box itself** — knots-box is `.135`. The DHCP leases had reshuffled. +The live config had already been hand-corrected to `knots-box`; running the +playbook would have reverted it and broken indexing. Now addressed by Tailscale +name, like everything else in this repo. + +**The restart handler was inert.** It carried +`when: uptime_kuma_enabled | default(false)`, so the three tasks that notify it +(SSL certificate, `fulcrum.conf`, systemd unit) could not restart anything. A +configuration change would write to disk, report success, and silently never take +effect. Ungated. + +**`db_mem` was about to quadruple.** The role computes a share of RAM; on this +5931 MB host 75% is 4448 MB, leaving ~1.4 GB for the OS and Fulcrum's non-cache +memory. The live value had been hand-tuned to 2048. `fulcrum_db_mem_mb_override` +pins it. Note `set_fact` outranks role defaults, so the *calculation* has to +honour the override — pinning it in `defaults/` alone is silently ignored. + +## The health check timer, and how to read it + +The timer is `OnBootSec` + `OnUnitActiveSec` with no `OnCalendar`. That +combination has a failure mode worth knowing: `OnBootSec` is monotonic and +elapses once; `OnUnitActiveSec` schedules relative to the **service** last being +active. If the service does not run in a given boot, there is no reference to +schedule from and the timer sits `active` and `enabled` doing nothing. That is +exactly what had happened here — last trigger **2026-02-17**, seven months of no +health check, with every surface-level indicator green. + +Restarting the timer does not supply that reference; running the service does. +So the role runs the check once after enabling the timer, which is both the fix +and a smoke test. + +**Diagnosing this is easy to get wrong**: `NextElapseUSecRealtime` is always +empty for a monotonic timer, so it looks broken even when it is fine. Read +`NextElapseUSecMonotonic`, or just use `systemctl list-timers`. + +The timer also no longer carries `Requires=fulcrum.service`. On a timer that +means "stop watching when the watched thing stops", which is backwards for a +health check. + +## Monitoring: one variable, no product knowledge + +The check tests the Electrum TCP port and records the answer in its exit code, +which systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. To report +elsewhere set `healthcheck_push_url` to any endpoint accepting an HTTP ping. The +Uptime Kuma API calls, monitor creation and token handling are gone. diff --git a/ansible/roles/fulcrum/defaults/main.yml b/ansible/roles/fulcrum/defaults/main.yml new file mode 100644 index 0000000..3e97fa0 --- /dev/null +++ b/ansible/roles/fulcrum/defaults/main.yml @@ -0,0 +1,81 @@ +# Fulcrum Configuration Variables + +# Version - Pinned to specific release +fulcrum_version: "2.1.0" # Fulcrum version to install + +# Directories +fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) +fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) +fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk +fulcrum_binary_path: /usr/local/bin/Fulcrum + +# Network - Bitcoin RPC connection +# Bitcoin Knots is on a different host (knots_box_local) +# Using RPC user/password authentication (credentials from group_vars/all/vault.yml) +# Addressed by Tailscale name, never a LAN IP. This was +# bitcoin_rpc_host: "192.168.1.140" # IP of knots_box_local +# but .140 is fulcrum-box ITSELF - knots-box is .135. The DHCP leases had +# reshuffled (the same drift that transposed the inventory), so running this +# playbook would have pointed Fulcrum at itself and broken indexing. The live +# config had already been hand-corrected to knots-box; this makes the repo +# agree with it. +bitcoin_rpc_host: "knots-box" +bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml + +# Network - Fulcrum server +fulcrum_tcp_port: 50001 +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/fulcrum_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +fulcrum_ssl_port: 50002 +# Binding address for Fulcrum TCP/SSL server: +# - "127.0.0.1" = localhost only (use when Caddy is on the same box) +# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) +# - Specific IP = bind to specific network interface +fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) +fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port +# If Caddy is on a different box, set this to the IP address that Caddy will use to connect + +# SSL/TLS Configuration +fulcrum_ssl_enabled: true +fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" +fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" +fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert + + +# Performance +# db_mem will be calculated as 75% of available RAM automatically in playbook +# db_mem is computed as this share of RAM unless fulcrum_db_mem_mb is set +# explicitly. On a 5931 MB host 75% is 4448 MB, which leaves ~1.4 GB for the +# OS and for Fulcrum's non-cache memory; the live config had been hand-tuned +# down to 2048 and that setting is respected below. +fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache + +# Configuration options +fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs +fulcrum_peering: false # Disable peering with other Fulcrum servers +fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications + +# Service user +fulcrum_user: fulcrum +fulcrum_group: fulcrum + + +# --- Health check ----------------------------------------------------------- +# Checks the Electrum TCP port and records the answer in its exit code, which +# systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. +# +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping +# works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" + +# Explicit db_mem in MB. When set it wins over fulcrum_db_mem_percent; empty +# means compute from RAM. Set here because the live host had been hand-tuned to +# 2048 and a silent jump to 4448 is not something a refactor should do. +fulcrum_db_mem_mb_override: 2048 diff --git a/ansible/roles/fulcrum/handlers/main.yml b/ansible/roles/fulcrum/handlers/main.yml new file mode 100644 index 0000000..c0d9cc4 --- /dev/null +++ b/ansible/roles/fulcrum/handlers/main.yml @@ -0,0 +1,15 @@ +--- +# Ungated on purpose. The hand-written handler carried +# when: uptime_kuma_enabled | default(false) +# so it has been inert since the decommissioning: three tasks notify it (the SSL +# certificate, fulcrum.conf and the systemd unit), and none of them could +# actually restart Fulcrum. A configuration change therefore applied to disk and +# silently never took effect — the worst kind of quiet failure, because the +# playbook reports success and the running service keeps its old settings. +# +# Restarting Fulcrum re-opens its database; it does not reindex. +- name: Restart fulcrum + systemd: + name: fulcrum + state: restarted + daemon_reload: yes diff --git a/ansible/roles/fulcrum/tasks/healthcheck.yml b/ansible/roles/fulcrum/tasks/healthcheck.yml new file mode 100644 index 0000000..983bffb --- /dev/null +++ b/ansible/roles/fulcrum/tasks/healthcheck.yml @@ -0,0 +1,62 @@ +--- +# Everything here answers "is Fulcrum healthy" and records the answer. The +# Uptime Kuma specifics that used to follow — an embedded Python script creating +# monitors over the API, a /tmp credentials file, push-URL extraction and a +# systemd Environment= rewrite — are gone. Where it reports is now one variable, +# healthcheck_push_url. See the role README. +- name: Create Fulcrum health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/fulcrum-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create systemd service for Fulcrum health check + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/fulcrum-healthcheck.service + owner: root + group: root + mode: "0600" + +- name: Create systemd timer for Fulcrum health check + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/fulcrum-healthcheck.timer + owner: root + group: root + mode: '0644' + +- name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + +# state: restarted, not started. The hand-written timer had got itself stuck +# `active` with no next elapse and had not fired since 2026-02-17; `started` on +# an already-active timer is a no-op and would have left it stuck. Restarting +# re-arms it. See the note in healthcheck.timer.j2. +- name: Enable and restart the Fulcrum health check timer + systemd: + name: fulcrum-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Run the check once, which is both a smoke test and the thing that actually +# arms the timer. +# +# This timer is OnBootSec + OnUnitActiveSec with no OnCalendar. OnBootSec is +# monotonic and had long since elapsed; OnUnitActiveSec schedules relative to the +# SERVICE last being active, and the service had not run since 2026-02-17 — so +# there was no reference to schedule from and the timer sat `active` and +# `enabled` with NextElapseUSecMonotonic=infinity. Restarting the timer alone +# does not supply that reference; running the service does. +# +# (Diagnosing this is easy to get wrong: NextElapseUSecRealtime is always empty +# for a monotonic timer, so it looks broken even when it is fine. Read +# NextElapseUSecMonotonic, or just use `systemctl list-timers`.) +- name: Run the Fulcrum health check once to arm the timer + command: systemctl start fulcrum-healthcheck.service + changed_when: false diff --git a/ansible/roles/fulcrum/tasks/install.yml b/ansible/roles/fulcrum/tasks/install.yml new file mode 100644 index 0000000..1216a96 --- /dev/null +++ b/ansible/roles/fulcrum/tasks/install.yml @@ -0,0 +1,146 @@ +--- +- name: Calculate db_mem as a share of system RAM + set_fact: + fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" + when: fulcrum_db_mem_mb_override | string | length == 0 + +- name: Use the explicit db_mem override + set_fact: + fulcrum_db_mem_mb: "{{ fulcrum_db_mem_mb_override }}" + when: fulcrum_db_mem_mb_override | string | length > 0 + changed_when: false + +- name: Display calculated db_mem value + debug: + msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" + +- name: Display Fulcrum version to install + debug: + msg: "Installing Fulcrum version {{ fulcrum_version }}" + +- name: Install required packages + apt: + name: + - curl + - wget + - openssl + state: present + update_cache: yes + +- name: Create fulcrum group + group: + name: "{{ fulcrum_group }}" + system: yes + state: present + +- name: Create fulcrum user + user: + name: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + system: yes + shell: /usr/sbin/nologin + home: /home/{{ fulcrum_user }} + create_home: yes + state: present + +- name: Create Fulcrum database directory (heavy data on special mount) + file: + path: "{{ fulcrum_db_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + +- name: Create Fulcrum config directory + file: + path: "{{ fulcrum_config_dir }}" + state: directory + owner: root + group: "{{ fulcrum_group }}" + mode: '0755' + +- name: Create Fulcrum lib directory (for banner and other data files) + file: + path: "{{ fulcrum_lib_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + +# =========================================== +# SSL Certificate Generation +# =========================================== +- name: Check if SSL certificate already exists + stat: + path: "{{ fulcrum_ssl_cert_path }}" + register: fulcrum_ssl_cert_exists + when: fulcrum_ssl_enabled | default(false) + +- name: Generate self-signed SSL certificate for Fulcrum + command: > + openssl req -x509 -newkey rsa:4096 + -keyout {{ fulcrum_ssl_key_path }} + -out {{ fulcrum_ssl_cert_path }} + -sha256 -days {{ fulcrum_ssl_cert_days }} + -nodes + -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" + args: + creates: "{{ fulcrum_ssl_cert_path }}" + when: fulcrum_ssl_enabled | default(false) + notify: Restart fulcrum + +- name: Set SSL certificate permissions + file: + path: "{{ fulcrum_ssl_cert_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) + +- name: Set SSL key permissions + file: + path: "{{ fulcrum_ssl_key_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0600' + when: fulcrum_ssl_enabled | default(false) + +- name: Check if Fulcrum binary already exists + stat: + path: "{{ fulcrum_binary_path }}" + register: fulcrum_binary_exists + changed_when: false + +- name: Download Fulcrum binary tarball + get_url: + url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + mode: '0644' + when: not fulcrum_binary_exists.stat.exists + +- name: Extract Fulcrum binary + unarchive: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp" + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + +- name: Install Fulcrum binary + copy: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" + dest: "{{ fulcrum_binary_path }}" + owner: root + group: root + mode: '0755' + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + +- name: Verify Fulcrum binary installation + command: "{{ fulcrum_binary_path }} --version" + register: fulcrum_version_check + changed_when: false + +- name: Display Fulcrum version + debug: + msg: "{{ fulcrum_version_check.stdout_lines }}" + diff --git a/ansible/roles/fulcrum/tasks/main.yml b/ansible/roles/fulcrum/tasks/main.yml new file mode 100644 index 0000000..bc7ff05 --- /dev/null +++ b/ansible/roles/fulcrum/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/fulcrum/tasks/service.yml b/ansible/roles/fulcrum/tasks/service.yml new file mode 100644 index 0000000..716b44d --- /dev/null +++ b/ansible/roles/fulcrum/tasks/service.yml @@ -0,0 +1,54 @@ +--- +- name: Create Fulcrum banner file + ansible.builtin.template: + src: banner.txt.j2 + dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + +- name: Create Fulcrum configuration file + ansible.builtin.template: + src: fulcrum.conf.j2 + dest: "{{ fulcrum_config_dir }}/fulcrum.conf" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0640' + notify: Restart fulcrum + +- name: Create systemd service file for Fulcrum + ansible.builtin.template: + src: fulcrum.service.j2 + dest: /etc/systemd/system/fulcrum.service + owner: root + group: root + mode: '0644' + notify: Restart fulcrum + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start Fulcrum service + systemd: + name: fulcrum + enabled: yes + state: started + +- name: Wait for Fulcrum to start + wait_for: + port: "{{ fulcrum_tcp_port }}" + host: "{{ fulcrum_tcp_bind }}" + delay: 5 + timeout: 30 + ignore_errors: yes + +- name: Check Fulcrum service status + systemd: + name: fulcrum + register: fulcrum_service_status + changed_when: false + +- name: Display Fulcrum service status + debug: + msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" diff --git a/ansible/roles/fulcrum/templates/banner.txt.j2 b/ansible/roles/fulcrum/templates/banner.txt.j2 new file mode 100644 index 0000000..1ce4ced --- /dev/null +++ b/ansible/roles/fulcrum/templates/banner.txt.j2 @@ -0,0 +1,3 @@ +counterinfra + +PER ASPERA AD ASTRA diff --git a/ansible/roles/fulcrum/templates/fulcrum.conf.j2 b/ansible/roles/fulcrum/templates/fulcrum.conf.j2 new file mode 100644 index 0000000..69f072c --- /dev/null +++ b/ansible/roles/fulcrum/templates/fulcrum.conf.j2 @@ -0,0 +1,29 @@ +# Fulcrum Configuration +# Generated by Ansible + +# Bitcoin Core/Knots RPC settings +bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} +rpcuser = {{ bitcoin_rpc_user }} +rpcpassword = {{ bitcoin_rpc_password }} + +# Fulcrum server general settings +datadir = {{ fulcrum_db_dir }} +tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} +peering = {{ 'true' if fulcrum_peering else 'false' }} +zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} + +# SSL/TLS Configuration +{% if fulcrum_ssl_enabled | default(false) %} +ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} +cert = {{ fulcrum_ssl_cert_path }} +key = {{ fulcrum_ssl_key_path }} +{% endif %} + +# Anonymize client IP addresses and TxIDs in logs +anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} + +# Max RocksDB Memory in MiB +db_mem = {{ fulcrum_db_mem_mb }}.0 + +# Banner +banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt diff --git a/ansible/roles/fulcrum/templates/fulcrum.service.j2 b/ansible/roles/fulcrum/templates/fulcrum.service.j2 new file mode 100644 index 0000000..02f4aa5 --- /dev/null +++ b/ansible/roles/fulcrum/templates/fulcrum.service.j2 @@ -0,0 +1,24 @@ +# MiniBolt: systemd unit for Fulcrum +# /etc/systemd/system/fulcrum.service + +[Unit] +Description=Fulcrum +After=network.target + +StartLimitBurst=2 +StartLimitIntervalSec=20 + +[Service] +ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf + +User={{ fulcrum_user }} +Group={{ fulcrum_group }} + +# Process management +#################### +Type=simple +KillSignal=SIGINT +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.service.j2 b/ansible/roles/fulcrum/templates/healthcheck.service.j2 new file mode 100644 index 0000000..b519808 --- /dev/null +++ b/ansible/roles/fulcrum/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Fulcrum Health Check +After=network.target fulcrum.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.sh.j2 b/ansible/roles/fulcrum/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..4449fed --- /dev/null +++ b/ansible/roles/fulcrum/templates/healthcheck.sh.j2 @@ -0,0 +1,41 @@ +#!/bin/bash +# Fulcrum health check — managed by Ansible (roles/fulcrum) +# +# Checks that Fulcrum's Electrum TCP port is accepting connections, and records +# the answer in the exit code, which systemd keeps: +# systemctl is-failed fulcrum-healthcheck.service +# That is a complete answer with no monitoring system involved. Reporting +# elsewhere is optional and generic — set healthcheck_push_url. + +FULCRUM_HOST="{{ fulcrum_tcp_bind }}" +FULCRUM_PORT={{ fulcrum_tcp_port }} +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +check_fulcrum() { + timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null +} + +report() { + local status=$1 msg=$2 + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true +} + +if check_fulcrum; then + report "up" "OK" + exit 0 +else + echo "Fulcrum TCP port ${FULCRUM_PORT} not responding" + report "down" "Fulcrum TCP port not responding" + exit 1 +fi diff --git a/ansible/roles/fulcrum/templates/healthcheck.timer.j2 b/ansible/roles/fulcrum/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..df16ff3 --- /dev/null +++ b/ansible/roles/fulcrum/templates/healthcheck.timer.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Fulcrum Health Check Timer +# NOTE: this deliberately does NOT carry `Requires=fulcrum.service`, which the +# hand-written unit had. Requires on a timer means the timer is stopped when the +# required unit stops — i.e. "if the thing I am watching goes down, stop +# watching it", which is backwards for a health check and leaves nothing to +# re-arm the timer when the service returns. The live timer had been `active` +# and `enabled` with NextElapseUSecMonotonic=infinity and a last trigger of +# 2026-02-17: seven months with no health check and no outward sign of it. + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/gatus/README.md b/ansible/roles/gatus/README.md new file mode 100644 index 0000000..129057c --- /dev/null +++ b/ansible/roles/gatus/README.md @@ -0,0 +1,110 @@ +# gatus + +Deploys [Gatus](https://github.com/TwiN/gatus) — health checks, a status page, +and alerting — **as the upstream container image**, on the +`observability` group. + +## Why the container + +Upstream publishes **no binary release assets**. The image is the only artefact +they ship and therefore the only one they test, so it is what `docker run` in +their README gets you, and it is what this role deploys. + +Building from source is entirely possible — their Dockerfile is a bare +`CGO_ENABLED=0 go build`, the Vue dashboard is compiled in via `//go:embed +static` in `web/static.go`, and the sqlite driver is pure-Go +`modernc.org/sqlite` so nothing needs linking. This role did that at first. The +reasons it doesn't now: + +- It produces a binary upstream never ran. +- It means compiling the AWS SDK, gRPC and the Google API libraries on the + smallest box in the estate. On this VPS that load was heavy enough that + unrelated Ansible tasks timed out while it ran. + +The cost of the container is a daemon on the machine whose job is to notice +when everything else breaks. That is a real trade, made deliberately. + +## Pinned by digest, not by tag + +```yaml +gatus_image_digest: "sha256:c5f210d0…" +gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" +``` + +A tag is mutable — `v5.36.0` can be repushed — so pinning the tag alone is a +weaker promise than it looks. The digest is a content address: if it resolves, +it is byte-for-byte the image reviewed here, and `docker compose pull` either +fetches exactly that or fails. `gatus_version` is kept alongside it purely so a +human can read which release it is; **the two must be updated together.** + +## What `FROM scratch` means for running it + +The image has no `/etc/passwd`, so there is no user to drop to by name and the +default is root. The compose file runs it by numeric id (`10001:10001`) and the +data directory on the host is owned to match. Everything else is locked down to +approximate what the systemd unit used to do natively: + +| systemd | compose | +|---|---| +| `ProtectSystem=strict` | `read_only: true` | +| `NoNewPrivileges=true` | `security_opt: [no-new-privileges:true]` | +| `CapabilityBoundingSet=` | `cap_drop: [ALL]` | +| `AmbientCapabilities=CAP_NET_RAW` | `cap_add: [NET_RAW]` (for `icmp://`) | + +## Configuration is a directory, not a file + +`GATUS_CONFIG_PATH` points at `/opt/gatus/config`, and Gatus merges every `*.yaml` +underneath it — maps deep-merge, lists append. This role owns exactly one file: + +``` +/opt/gatus/config/00-base.yaml web, storage, ui, alerting, security (this role) +/opt/gatus/config/endpoints/*.yaml one file per service (gatus_endpoint) +``` + +**A primitive defined in two files is ambiguous and upstream refuses it.** So +anything that is not a list belongs in `00-base.yaml` and nowhere else. Endpoints +are lists, so each service's file appends cleanly — the same shape as +`caddy_site`, where each service contributes its own vhost. + +## Pull and push + +Gatus polls. For anything with a reachable HTTP or TCP surface that is the +better check, because it tests the path a user actually takes. The monitoring +host joins the headscale mesh via `infra/920`, so internal boxes are reachable +by MagicDNS name and can be polled directly rather than having to report in. + +For state with no pollable surface — ZFS pool health, UPS mains status, disk +usage, backup freshness — Gatus has **external endpoints**, a push API: + +``` +POST /api/v1/endpoints/{group}_{name}/external?success=true&error=&duration= +Authorization: Bearer +``` + +with `heartbeat.interval` to alert when nothing reports in. That is the same +shape as the generic `healthcheck_push_url` already wired into every service +role, so those scripts need a URL, a POST, and an auth header — not a rewrite. + +## Variables + +See `defaults/main.yml`. The ones that matter: + +| Variable | Default | Note | +|---|---|---| +| `gatus_version` / `gatus_image_digest` | `v5.36.0` / `sha256:c5f210d0…` | must move together | +| `gatus_bind_address` | `127.0.0.1` | **never bind publicly** — the push API shares this listener | +| `gatus_storage_type` | `sqlite` | `memory` loses all history on restart | +| `gatus_alerting` | `{}` | pass-through; any provider Gatus supports | +| `gatus_allow_icmp` | `true` | adds back `NET_RAW` for `icmp://` checks | + +`gatus_alerting` empty is valid and is the current state: every condition is +still evaluated and recorded, there is just nowhere to shout yet. + +## Verifying + +```bash +docker ps --filter name=gatus +docker logs gatus --tail 50 +curl -s localhost:8080/health +ls /opt/gatus/config/endpoints/ +``` diff --git a/ansible/roles/gatus/defaults/main.yml b/ansible/roles/gatus/defaults/main.yml new file mode 100644 index 0000000..6f6ec09 --- /dev/null +++ b/ansible/roles/gatus/defaults/main.yml @@ -0,0 +1,118 @@ +--- +# Gatus, deployed the way upstream distributes it: the container image. +# +# Upstream publishes NO binary release assets - the image is the only artefact +# they ship, and therefore the only artefact they test. Building from source is +# possible (`go build` alone is enough; the Vue dashboard is compiled in via +# `//go:embed static`, and CGO_ENABLED=0 works because the sqlite driver is +# pure-Go modernc.org/sqlite) but it produces a binary upstream never ran, and +# it means a full compile of the AWS SDK and gRPC on the smallest box in the +# estate. + +# ── Image ──────────────────────────────────────────────────────────────────── +gatus_version: "v5.36.0" +# Pinned by DIGEST, not by tag. A tag is mutable - `v5.36.0` can be repushed - +# so pinning the tag alone is a weaker promise than it looks. The digest is the +# content address: if it resolves, it is byte-for-byte the image reviewed here. +# Both must be updated together; the tag is kept only so humans can read it. +gatus_image_digest: "sha256:c5f210d095fa78e6efaa20ffeb14803f2ba4f10615e16a6d12087697149617f0" +gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" + +# ── Paths (host side) ──────────────────────────────────────────────────────── +gatus_dir: /opt/gatus +gatus_config_dir: "{{ gatus_dir }}/config" +gatus_data_dir: "{{ gatus_dir }}/data" + +# Gatus merges every *.yaml under GATUS_CONFIG_PATH and its subdirectories: +# maps deep-merge, lists append. That is why this role ships a config DIRECTORY +# rather than one file - each service contributes its own endpoint file, the +# same way each service contributes a vhost through `caddy_site`. +# +# Primitives must be defined exactly once across all files or the merge is +# ambiguous, so everything that is not a list lives in the base file and +# nowhere else. +gatus_base_config_file: "00-base.yaml" +gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" + +# ── Identity ───────────────────────────────────────────────────────────────── +# The image is FROM scratch, so it has no /etc/passwd and no user to drop to by +# name. Run it by numeric uid/gid instead, and own the data volume to match. +gatus_uid: 10001 +gatus_gid: 10001 + +# ── Web ────────────────────────────────────────────────────────────────────── +gatus_port: 8080 +# HOST-side address the container's port is published on. Gatus itself always +# binds 0.0.0.0 inside the container - see the note in config.yaml.j2. Never +# publish this on 0.0.0.0: the external-endpoint push API shares the dashboard's +# listener, and Caddy is what should be in front of both. +gatus_bind_address: "127.0.0.1" +# Pass-through, the same shape as gatus_alerting: whatever is set here is +# rendered verbatim under `ui:`, so every option upstream supports is reachable +# without adding a variable per key. See config/ui/ui.go for the full set - +# title, description, header, dashboard-heading/subheading, logo, link, favicon, +# buttons, custom-css, dark-mode, default-sort-by, default-filter-by. +gatus_ui: + title: "Status" + header: "Status" + # Group by default. The dashboard's own groupByGroup toggle starts OFF and + # remembers per browser in localStorage, so without this the ten groups render + # as one flat list for anyone who has not clicked it. + default-sort-by: group + +# ── Storage ────────────────────────────────────────────────────────────────── +# sqlite, not memory: history has to survive a restart, or the dashboard lies +# about uptime after every deploy. Path is INSIDE the container. +gatus_storage_type: sqlite +gatus_storage_path: "/data/gatus.db" +gatus_storage_caching: true + +# Per-endpoint row caps. Gatus bounds the database by COUNT, not by time, and +# trims inline on insert (storage/store/sql/sql.go, InsertEndpointResult) - so +# there is no retention job to write and no way for this to fill a disk. +# +# History depth is therefore a function of check frequency, not of days: +# 900 results is ~3 days of a 5-minute liveness check, and ~2.5 years of a daily +# disk check. Upstream's default is 100, which would have been 8 hours of +# liveness - not enough to still see a weekend incident on Monday. +# +# Note the uptime table is separate and its 30-day retention is hard-coded +# upstream (uptimeRetention), so uptime percentages top out at 30 days whatever +# this is set to. +gatus_storage_max_results: 900 +gatus_storage_max_events: 50 + +# ── Alerting ───────────────────────────────────────────────────────────────── +# Pass-through: rendered verbatim under `alerting:`, so any provider Gatus +# supports works without touching this role. Empty means "check and record, +# alert nowhere" - valid, and the default until a provider is chosen. +gatus_alerting: {} +gatus_default_alerts: [] + +# ── Security ───────────────────────────────────────────────────────────────── +# gatus_basic_auth: {username: admin, password-bcrypt-base64: "..."} +gatus_basic_auth: {} + +gatus_maintenance: {} + +# Keep serving the previous config if a contributed endpoint file is malformed, +# instead of panicking. See the note in config.yaml.j2. +gatus_skip_invalid_config_update: true +gatus_log_level: INFO + +# ── Self-check ─────────────────────────────────────────────────────────────── +# Gatus panics on a config with no endpoints, so the role always ships one. +# Turning this off is only safe once another file in endpoints/ provides one. +gatus_self_check: true + +# ── ICMP ───────────────────────────────────────────────────────────────────── +# Gatus supports icmp:// endpoints. Raw ICMP needs CAP_NET_RAW, which the +# container would get free only if it ran as root; it does not. Set false if +# you never use icmp:// checks and want the capability dropped entirely. +gatus_allow_icmp: true + +# ── Shared network ─────────────────────────────────────────────────────────── +# Gatus runs in a container, so the HOST's loopback is not reachable from it. +# Anything Gatus must talk to locally - the Signal API that sends its alerts - +# has to be on a shared docker network and addressed by service name. +gatus_network: monitoring diff --git a/ansible/roles/gatus/handlers/main.yml b/ansible/roles/gatus/handlers/main.yml new file mode 100644 index 0000000..25ad865 --- /dev/null +++ b/ansible/roles/gatus/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart gatus + ansible.builtin.command: + cmd: docker compose up -d --force-recreate + chdir: "{{ gatus_dir }}" diff --git a/ansible/roles/gatus/tasks/configure.yml b/ansible/roles/gatus/tasks/configure.yml new file mode 100644 index 0000000..ef5d7e7 --- /dev/null +++ b/ansible/roles/gatus/tasks/configure.yml @@ -0,0 +1,64 @@ +--- +- name: Assert Docker is available + ansible.builtin.command: docker --version + register: gatus_docker_check + changed_when: false + failed_when: gatus_docker_check.rc != 0 + +# Created explicitly rather than by either compose file, so neither the gatus +# stack nor the signal-api stack has to be deployed before the other. +- name: Ensure the shared monitoring network exists + ansible.builtin.command: "docker network create {{ gatus_network }}" + register: gatus_net + changed_when: "'already exists' not in gatus_net.stderr" + failed_when: + - gatus_net.rc != 0 + - "'already exists' not in gatus_net.stderr" + +- name: Create the gatus directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: + - {path: "{{ gatus_dir }}", owner: root, group: root, mode: "0755"} + # Config is root-owned and world-unreadable: it holds external-endpoint + # tokens. The container mounts it read-only and reads it as gatus_uid, so + # that id needs group access - hence the group ownership below. + - {path: "{{ gatus_config_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} + - {path: "{{ gatus_endpoints_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} + # Data is the one path the container writes to, so it must be owned by the + # numeric id the container runs as. `read_only: true` makes everything else + # in the container immutable. + - {path: "{{ gatus_data_dir }}", owner: "{{ gatus_uid }}", group: "{{ gatus_gid }}", mode: "0750"} + +- name: Write the base gatus configuration + ansible.builtin.template: + src: config.yaml.j2 + dest: "{{ gatus_config_dir }}/{{ gatus_base_config_file }}" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + notify: Restart gatus + +# Without this the container crash-loops on an empty config. See the template. +- name: Write the gatus self-check endpoint + ansible.builtin.template: + src: endpoint-self.yaml.j2 + dest: "{{ gatus_endpoints_dir }}/00-self.yaml" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + when: gatus_self_check | bool + notify: Restart gatus + +- name: Write the docker compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ gatus_dir }}/docker-compose.yml" + owner: root + group: root + mode: "0644" + notify: Restart gatus diff --git a/ansible/roles/gatus/tasks/main.yml b/ansible/roles/gatus/tasks/main.yml new file mode 100644 index 0000000..e1b67bf --- /dev/null +++ b/ansible/roles/gatus/tasks/main.yml @@ -0,0 +1,5 @@ +--- +# import, never include: dynamic includes are opaque to --list-tasks, which is +# the primary verification tool in this repo. +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml diff --git a/ansible/roles/gatus/tasks/service.yml b/ansible/roles/gatus/tasks/service.yml new file mode 100644 index 0000000..bfcca3b --- /dev/null +++ b/ansible/roles/gatus/tasks/service.yml @@ -0,0 +1,35 @@ +--- +# Pulling by digest means this either fetches exactly the reviewed image or +# fails. There is no "latest wins" path. +- name: Pull the pinned gatus image + ansible.builtin.command: + cmd: docker compose pull + chdir: "{{ gatus_dir }}" + register: gatus_pull + changed_when: "'Downloaded newer image' in gatus_pull.stderr or 'Pull complete' in gatus_pull.stderr" + +- name: Start gatus + ansible.builtin.command: + cmd: docker compose up -d --remove-orphans + chdir: "{{ gatus_dir }}" + register: gatus_up + changed_when: "'Started' in gatus_up.stderr or 'Created' in gatus_up.stderr or 'Recreated' in gatus_up.stderr" + +- name: Flush handlers so a config change is live before it is verified + ansible.builtin.meta: flush_handlers + +- name: Wait for gatus to answer + ansible.builtin.uri: + url: "http://{{ gatus_bind_address }}:{{ gatus_port }}/health" + status_code: [200, 404] + register: gatus_health + until: gatus_health.status in [200, 404] + retries: 12 + delay: 5 + +- name: Assert gatus is running + ansible.builtin.assert: + that: + - gatus_health.status in [200, 404] + fail_msg: "gatus did not come up on {{ gatus_bind_address }}:{{ gatus_port }} - check `docker logs gatus`" + success_msg: "gatus is answering on {{ gatus_bind_address }}:{{ gatus_port }}" diff --git a/ansible/roles/gatus/templates/config.yaml.j2 b/ansible/roles/gatus/templates/config.yaml.j2 new file mode 100644 index 0000000..faf61fb --- /dev/null +++ b/ansible/roles/gatus/templates/config.yaml.j2 @@ -0,0 +1,64 @@ +# {{ gatus_base_config_file }} — managed by Ansible (roles/gatus) +# +# BASE CONFIGURATION ONLY. +# +# Gatus merges every *.yaml under GATUS_CONFIG_PATH: maps are deep-merged and +# lists are appended, but a primitive defined in two files is ambiguous and +# upstream refuses it. So `web`, `storage`, `ui`, `alerting` and `security` are +# set here and MUST NOT appear in any other file in this directory. +# +# Endpoints are lists, so they append cleanly. Each service drops its own file +# into endpoints/ via the gatus_endpoint role - the same shape as caddy_site. + +web: + # 0.0.0.0 is the CONTAINER's interface, not the host's. This must not be + # 127.0.0.1: that is the container's own loopback, which docker-proxy cannot + # reach, and gatus comes up healthy while the published port refuses every + # connection. + # + # The isolation comes from the port mapping in docker-compose.yml, which + # publishes to {{ gatus_bind_address }} on the host. Caddy fronts that, and it + # matters because the external-endpoint push API shares this listener with the + # dashboard. + address: 0.0.0.0 + port: {{ gatus_port }} + +# Gatus reloads when its config changes. If the NEW config fails to parse it +# calls panic() - unless this is set, in which case it logs the error and keeps +# running on the old config. Endpoint files are contributed by other playbooks, +# so one malformed file would otherwise take the monitor down, which is the +# worst possible time to lose it. +skip-invalid-config-update: {{ gatus_skip_invalid_config_update | bool | lower }} + +storage: + type: {{ gatus_storage_type }} +{% if gatus_storage_type != 'memory' %} + path: {{ gatus_storage_path }} +{% endif %} + caching: {{ gatus_storage_caching | bool | lower }} + maximum-number-of-results: {{ gatus_storage_max_results }} + maximum-number-of-events: {{ gatus_storage_max_events }} + +ui: +{{ gatus_ui | to_nice_yaml(indent=2) | indent(2, true) }} +{% if gatus_alerting %} + +alerting: +{{ gatus_alerting | to_nice_yaml(indent=2) | indent(2, true) }} +{% else %} + +# No alerting provider is configured yet. Gatus still evaluates every condition +# and records every result; it simply has nowhere to shout. Setting +# `gatus_alerting` is the single change needed to wire one up. +{% endif %} +{% if gatus_basic_auth %} + +security: + basic: +{{ gatus_basic_auth | to_nice_yaml(indent=2) | indent(4, true) }} +{% endif %} +{% if gatus_maintenance %} + +maintenance: +{{ gatus_maintenance | to_nice_yaml(indent=2) | indent(2, true) }} +{% endif %} diff --git a/ansible/roles/gatus/templates/docker-compose.yml.j2 b/ansible/roles/gatus/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..512c9c5 --- /dev/null +++ b/ansible/roles/gatus/templates/docker-compose.yml.j2 @@ -0,0 +1,53 @@ +# Managed by Ansible (roles/gatus) +services: + gatus: + image: {{ gatus_image }} + container_name: gatus + restart: unless-stopped + + # The image is FROM scratch: no /etc/passwd, so there is no user to drop to + # by name and the default is root. Run it by numeric id instead. + user: "{{ gatus_uid }}:{{ gatus_gid }}" + + ports: + # Loopback on purpose. Caddy fronts this, and the external-endpoint push + # API is served from the same listener as the dashboard - publishing + # 0.0.0.0 would put both straight on the public internet. + - "{{ gatus_bind_address }}:{{ gatus_port }}:{{ gatus_port }}" + + environment: + GATUS_CONFIG_PATH: /config + GATUS_LOG_LEVEL: "{{ gatus_log_level }}" + + volumes: + - {{ gatus_config_dir }}:/config:ro + - {{ gatus_data_dir }}:/data + + # Hardening. The systemd unit this replaced got most of it from + # ProtectSystem/NoNewPrivileges/etc; these are the container equivalents. + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL +{% if gatus_allow_icmp %} + cap_add: + # icmp:// endpoints need raw sockets. Dropped above with ALL, added back + # explicitly so the grant is visible rather than inherited from root. + - NET_RAW +{% endif %} + + networks: + # Shared with signal-api, so alerts can be delivered by service name. + # 127.0.0.1 inside this container is the container, not the host. + - {{ gatus_network }} + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +networks: + {{ gatus_network }}: + external: true diff --git a/ansible/roles/gatus/templates/endpoint-self.yaml.j2 b/ansible/roles/gatus/templates/endpoint-self.yaml.j2 new file mode 100644 index 0000000..197ae56 --- /dev/null +++ b/ansible/roles/gatus/templates/endpoint-self.yaml.j2 @@ -0,0 +1,25 @@ +# 00-self.yaml — managed by Ansible (roles/gatus) +# +# Gatus refuses to start with no endpoints at all: +# panic: error parsing config: configuration should contain at least one +# endpoint or suite +# +# So the role ships one. Checking its own listener is not as circular as it +# looks: it proves the config directory parsed, the container is serving, and +# the storage backend accepted a write. If this row is missing from the +# dashboard, the dashboard is not telling you the truth about anything else. +# +# It is also what keeps `gatus` deployable before any service has contributed +# an endpoint file of its own. + +endpoints: + - name: gatus + group: infrastructure + url: "http://localhost:{{ gatus_port }}/health" + interval: 60s + conditions: + - "[STATUS] == 200" +{% if gatus_default_alerts %} + alerts: +{{ gatus_default_alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} diff --git a/ansible/roles/gatus_endpoint/defaults/main.yml b/ansible/roles/gatus_endpoint/defaults/main.yml new file mode 100644 index 0000000..90a2c19 --- /dev/null +++ b/ansible/roles/gatus_endpoint/defaults/main.yml @@ -0,0 +1,58 @@ +--- +# One invocation writes ONE file into Gatus's endpoints directory. Gatus merges +# every *.yaml under GATUS_CONFIG_PATH and appends lists, so each caller owns +# its own file and they compose without coordinating - the same shape as +# caddy_site, where each service contributes its own vhost. + +# Filename stem: .yaml +gatus_endpoint_name: "" + +# PULLED endpoints - Gatus makes the request and evaluates conditions. +# - {name, group, url, interval, conditions: [...], alerts: [...]} +# +# A DNS check adds `dns: {query-type, query-name}` - and note that for those, +# `url` is the RESOLVER to ask, not the name being looked up. +# A domain-expiry check is just `url: ` with a [DOMAIN_EXPIRATION] +# condition; it uses WHOIS/RDAP and needs no scheme. +gatus_endpoint_pulled: [] + +# EXTERNAL endpoints - the host pushes its own result. Gatus never reaches out, +# which is what makes this work for machines behind NAT and for state that has +# no pollable surface at all (disk usage, ZFS health, UPS mains). +# +# - {name, group, token, heartbeat, alerts: [...]} +# +# `heartbeat` is the important one: if nothing reports within that window Gatus +# alerts. That is what makes a push check detect its own failure - a dead timer +# looks exactly like a dead host, which is the correct reading. +gatus_endpoint_external: [] + +# Where the files live. Matches roles/gatus. +gatus_config_dir: /opt/gatus/config +gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" +gatus_gid: 10001 + +# Alerts attached to every endpoint in this file that does not specify its own. +# +# Gatus's provider-level `default-alert` only supplies DEFAULTS - an endpoint +# still has to opt in with `alerts: - type: signal` or it alerts on nothing at +# all. With ~90 endpoints that cannot be written by hand, so it is applied here. +# +# failure-threshold is set by the CALLER, because the right value depends on the +# check's cadence and there is no single correct default. See the note in +# infra/400_host_monitoring.yml. +gatus_endpoint_default_alerts: [] + +# UI options applied to every PULLED endpoint that does not set its own. +# +# Only pulled endpoints can carry this - an external (push) endpoint has no `ui` +# field at all, because it has no conditions to display. +# +# Note that every hide-* option already defaults to false upstream, so there is +# nothing to un-hide. The one setting that genuinely shows MORE is +# resolve-successful-conditions: by default a failing check displays the real +# value - "[STATUS] (502) == 200" - while a passing one drops it and shows only +# "[STATUS] == 200". Turning it on resolves both, so a healthy DNS check shows +# the IP it actually resolved rather than just the assertion. +gatus_endpoint_default_ui: + resolve-successful-conditions: true diff --git a/ansible/roles/gatus_endpoint/tasks/main.yml b/ansible/roles/gatus_endpoint/tasks/main.yml new file mode 100644 index 0000000..40f3448 --- /dev/null +++ b/ansible/roles/gatus_endpoint/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Assert gatus_endpoint parameters are sane + ansible.builtin.assert: + that: + - gatus_endpoint_name | length > 0 + - gatus_endpoint_pulled | length > 0 or gatus_endpoint_external | length > 0 + fail_msg: >- + gatus_endpoint needs a name and at least one of gatus_endpoint_pulled or + gatus_endpoint_external. An empty file would contribute nothing, and a + config with no endpoints at all makes Gatus refuse to start. + +- name: "Write Gatus endpoints '{{ gatus_endpoint_name }}'" + ansible.builtin.template: + src: endpoints.yaml.j2 + dest: "{{ gatus_endpoints_dir }}/{{ gatus_endpoint_name }}.yaml" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + # No handler. Gatus polls its own config every 30s + # (main.listenToConfigurationFileChanges) and reloads itself, so writing the + # file IS the deploy. A handler here would also fail whenever this role is + # used without roles/gatus loaded. diff --git a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 new file mode 100644 index 0000000..680691e --- /dev/null +++ b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 @@ -0,0 +1,60 @@ +# {{ gatus_endpoint_name }}.yaml — managed by Ansible (roles/gatus_endpoint) +# +# Contributed by a playbook, not hand-edited. Gatus appends the lists in every +# *.yaml under its config directory, so this file adds to whatever else is +# registered without knowing about it. +{% if gatus_endpoint_external %} + +external-endpoints: +{% for e in gatus_endpoint_external %} + - name: {{ e.name }} + group: {{ e.group }} + token: "{{ e.token }}" +{% if e.heartbeat is defined %} + heartbeat: + interval: {{ e.heartbeat }} +{% endif %} +{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} +{% if _alerts %} + alerts: +{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{# external endpoints have no `ui` field upstream - they carry no conditions #} +{% endfor %} +{% endif %} +{% if gatus_endpoint_pulled %} + +endpoints: +{% for e in gatus_endpoint_pulled %} + - name: {{ e.name }} + group: {{ e.group }} + url: "{{ e.url }}" + interval: {{ e.interval | default('60s') }} +{% if e.dns is defined %} + # A DNS endpoint: `url` is the RESOLVER to ask, not the thing being asked + # about. The name being queried lives in query-name, and [BODY] holds the + # resolved record for the conditions below. + dns: + query-type: {{ e.dns['query-type'] }} + query-name: {{ e.dns['query-name'] }} +{% endif %} +{% if e.client is defined %} + client: +{{ e.client | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} + conditions: +{% for c in e.conditions %} + - "{{ c }}" +{% endfor %} +{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} +{% if _alerts %} + alerts: +{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{% set _ui = e.ui | default(gatus_endpoint_default_ui) %} +{% if _ui %} + ui: +{{ _ui | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{% endfor %} +{% endif %} diff --git a/ansible/roles/healthcheck/defaults/main.yml b/ansible/roles/healthcheck/defaults/main.yml new file mode 100644 index 0000000..51534c2 --- /dev/null +++ b/ansible/roles/healthcheck/defaults/main.yml @@ -0,0 +1,54 @@ +--- +# One health check: a script, a systemd service, a timer, and an optional push. +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed -healthcheck.service +# Reporting anywhere else is optional and generic. Point healthcheck_push_url at +# Gatus, or at whatever replaces it, or at nothing. + +healthcheck_name: "" # e.g. disk-usage -> disk-usage-healthcheck +healthcheck_description: "" + +# The check itself. Pick ONE: +# healthcheck_check: a template under templates/checks/ (without .sh.j2) +# healthcheck_command: a shell one-liner that exits 0 for healthy +healthcheck_check: "" +healthcheck_command: "" + +# systemd timer. OnUnitActiveSec unless healthcheck_on_calendar is set. +healthcheck_interval: "5min" +healthcheck_on_calendar: "" +healthcheck_boot_delay: "2min" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Gatus external endpoints: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty url = check and log only, which is a valid state and not an error. +healthcheck_push_url: "" +healthcheck_push_token: "" + +# Some checks report MORE THAN ONE result - a host with four systemd services +# needs four endpoints, or a single red light cannot tell you which one died. +# Those set healthcheck_push_base to the endpoints COLLECTION and the check body +# appends each key itself, the same way check-backups.sh reports per source. +healthcheck_push_base: "" + +# Units for the systemd-units check. Each becomes its own Gatus endpoint. +healthcheck_units: [] +# Prefix for the per-unit endpoint keys, e.g. "services_vipy" -> services_vipy-caddy. +healthcheck_units_key_prefix: "" + +healthcheck_script_dir: /usr/local/bin +healthcheck_log_dir: /var/log/healthchecks + +# Per-check knobs, consumed by the templates under checks/ +healthcheck_disk_threshold: 85 # percent +healthcheck_cpu_temp_threshold: 80 # celsius +healthcheck_zfs_pool: "" +# ZFS degrades badly once a pool passes roughly 80% - allocation gets slow and +# fragmentation becomes hard to undo, and unlike a normal filesystem you cannot +# simply delete your way back to good performance. So this alarms well before +# the pool is actually out of space. +healthcheck_zfs_capacity_threshold: 80 +healthcheck_ups_name: "" diff --git a/ansible/roles/healthcheck/tasks/main.yml b/ansible/roles/healthcheck/tasks/main.yml new file mode 100644 index 0000000..f37a0d5 --- /dev/null +++ b/ansible/roles/healthcheck/tasks/main.yml @@ -0,0 +1,90 @@ +--- +- name: "Assert healthcheck '{{ healthcheck_name }}' is fully specified" + ansible.builtin.assert: + that: + - healthcheck_name | length > 0 + - healthcheck_description | length > 0 + - (healthcheck_check | length > 0) != (healthcheck_command | length > 0) + - not (healthcheck_push_url | length > 0) or (healthcheck_push_token | length > 0) + fail_msg: >- + healthcheck needs a name, a description, exactly one of healthcheck_check + or healthcheck_command, and a token whenever a push URL is set. A push URL + with no token would report to Gatus and be rejected 401 on every run. + +# Deduplicated across the whole play run. This role is included once PER CHECK, +# and a host with several checks was otherwise running apt several times to +# install a curl that was already there - 29 apt transactions estate-wide, and +# the slowest thing in the deploy by a wide margin. The fact below remembers +# what has already been ensured on this host. +- name: Install healthcheck dependencies + ansible.builtin.package: + name: "{{ healthcheck_wanted_packages }}" + state: present + vars: + healthcheck_wanted_packages: >- + {{ (healthcheck_packages | default(['curl'])) + | difference(healthcheck_installed_packages | default([])) }} + when: healthcheck_wanted_packages | length > 0 + +- name: Remember which dependencies this host already has + ansible.builtin.set_fact: + healthcheck_installed_packages: >- + {{ (healthcheck_installed_packages | default([])) + | union(healthcheck_packages | default(['curl'])) }} + +- name: Create the healthcheck log directory + ansible.builtin.file: + path: "{{ healthcheck_log_dir }}" + state: directory + owner: root + group: root + mode: "0750" + +- name: "Install the {{ healthcheck_name }} check script" + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" + owner: root + group: root + mode: "0755" + +# The token is in this unit file, so it must not be world-readable. +- name: "Install the {{ healthcheck_name }} systemd service" + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.service" + owner: root + group: root + mode: "0600" + +- name: "Install the {{ healthcheck_name }} systemd timer" + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.timer" + owner: root + group: root + mode: "0644" + +# `restarted`, not `started`: started is a no-op on an already-active timer, so +# a changed interval or a stuck timer would never be picked up. +- name: "Enable and start the {{ healthcheck_name }} timer" + ansible.builtin.systemd: + name: "{{ healthcheck_name }}-healthcheck.timer" + enabled: yes + state: restarted + daemon_reload: yes + +- name: "Run the {{ healthcheck_name }} check once now" + ansible.builtin.command: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" + environment: + HEALTHCHECK_PUSH_URL: "{{ healthcheck_push_url }}" + HEALTHCHECK_PUSH_TOKEN: "{{ healthcheck_push_token }}" + register: healthcheck_first_run + changed_when: false + failed_when: false + +- name: "Report the first {{ healthcheck_name }} result" + ansible.builtin.debug: + msg: >- + {{ healthcheck_name }}: {{ 'HEALTHY' if healthcheck_first_run.rc == 0 + else 'UNHEALTHY (rc=' ~ healthcheck_first_run.rc ~ ')' }} diff --git a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 new file mode 100644 index 0000000..cf57509 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 @@ -0,0 +1,28 @@ + # Hottest core across every thermal zone and hwmon sensor lm-sensors knows + # about. Reading the hottest rather than an average is deliberate: one core + # throttling is a real problem that an average hides. + local threshold={{ healthcheck_cpu_temp_threshold }} + local hottest=0 label="" + + while read -r t; do + [ -z "$t" ] && continue + t=${t%.*} + if [ "$t" -gt "$hottest" ]; then hottest=$t; fi + done < <(sensors -u 2>/dev/null | awk '/_input:/ && /temp/ {print $2}') + + # Fall back to the kernel thermal zones if lm-sensors reports nothing. + if [ "$hottest" -eq 0 ]; then + for z in /sys/class/thermal/thermal_zone*/temp; do + [ -r "$z" ] || continue + local milli; milli=$(cat "$z" 2>/dev/null) || continue + local c=$((milli / 1000)) + if [ "$c" -gt "$hottest" ]; then hottest=$c; label=$(cat "${z%/temp}/type" 2>/dev/null); fi + done + fi + + if [ "$hottest" -eq 0 ]; then + MESSAGE="no temperature sensors readable" + return 1 + fi + MESSAGE="${hottest}C${label:+ (${label})}" + [ "$hottest" -lt "$threshold" ] diff --git a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 new file mode 100644 index 0000000..b0e76ce --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 @@ -0,0 +1,19 @@ + # Every real filesystem must be under the threshold. tmpfs, devtmpfs, + # squashfs and overlay are excluded: they are either RAM, read-only, or + # container layers, and none of them fills up in a way an operator can act on. + local threshold={{ healthcheck_disk_threshold }} + local worst=0 worst_mount="" over="" + + while read -r pct mount; do + pct=${pct%\%} + [ -z "$pct" ] && continue + if [ "$pct" -gt "$worst" ]; then worst=$pct; worst_mount=$mount; fi + if [ "$pct" -ge "$threshold" ]; then over="${over}${over:+, }${mount} ${pct}%"; fi + done < <(df -P -x tmpfs -x devtmpfs -x squashfs -x overlay --output=pcent,target 2>/dev/null | tail -n +2) + + if [ -n "$over" ]; then + MESSAGE="over ${threshold}%: ${over}" + return 1 + fi + MESSAGE="max ${worst}% on ${worst_mount:-/}" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 new file mode 100644 index 0000000..141e478 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 @@ -0,0 +1,6 @@ + # Liveness has no test to run: the fact that this script executed at all is + # the signal. What proves the host is alive is the PUSH arriving at Gatus, + # and what detects the host being dead is the heartbeat window expiring with + # no push. So this always succeeds - the reporting is the check. + MESSAGE="up since $(uptime -p 2>/dev/null || echo unknown)" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 b/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 new file mode 100644 index 0000000..90c6193 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 @@ -0,0 +1,33 @@ + # One result PER UNIT, not one for the host. A host running four services + # needs four endpoints: a single red light would tell you "something on vipy + # is down" without saying which, and that is the question you actually have. + # + # Keys are host-qualified because unit names collide - caddy runs on four + # machines. Gatus builds the key as sanitize(group)_sanitize(name), so + # group "services" + name "vipy/caddy" becomes services_vipy-caddy. + local prefix="{{ healthcheck_units_key_prefix }}" + local down="" + + for unit in {{ healthcheck_units | join(' ') }}; do + local state substate + state=$(systemctl is-active "$unit" 2>/dev/null || true) + substate=$(systemctl show -p SubState --value "$unit" 2>/dev/null || true) + + if [ "$state" = "active" ]; then + report_key "${prefix}-${unit}" "true" "${unit} active (${substate:-running})" + else + # `failed` and `inactive` are different stories: one crashed, one was + # stopped. Both are down, but the message should say which. + local detail="${unit} is ${state:-unknown}" + [ "$state" = "failed" ] && detail="${detail} (${substate:-failed})" + report_key "${prefix}-${unit}" "false" "$detail" + down="${down}${down:+, }${unit}=${state:-unknown}" + fi + done + + if [ -n "$down" ]; then + MESSAGE="down: ${down}" + return 1 + fi + MESSAGE="all {{ healthcheck_units | length }} units active" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 new file mode 100644 index 0000000..508c860 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 @@ -0,0 +1,18 @@ + # OL means on line power. Anything else - OB (on battery), LB (low battery), + # or no answer at all - is a failure worth waking up for, because the + # hypervisor has a finite number of minutes left. + local ups="{{ healthcheck_ups_name }}" + local status charge runtime load + + status=$(upsc "${ups}@localhost" ups.status 2>/dev/null) + if [ -z "$status" ]; then + MESSAGE="cannot reach UPS ${ups} via upsd" + return 1 + fi + + charge=$(upsc "${ups}@localhost" battery.charge 2>/dev/null) + runtime=$(upsc "${ups}@localhost" battery.runtime 2>/dev/null) + load=$(upsc "${ups}@localhost" ups.load 2>/dev/null) + + MESSAGE="status=${status} charge=${charge}% runtime=${runtime}s load=${load}%" + [[ "$status" == *"OL"* ]] diff --git a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 new file mode 100644 index 0000000..ebe88b9 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 @@ -0,0 +1,63 @@ + # Five conditions, all of which have to hold. Ported from the check that + # infra/nodito/32_zfs_pool_setup_playbook.yml deployed, which was correct - + # only its reporting was tied to Uptime Kuma. + local pool="{{ healthcheck_zfs_pool }}" + local json issues="" + + json=$(zpool status -j "$pool" 2>&1) || { MESSAGE="zpool status failed: ${json}"; return 1; } + + # 1. pool state + local state + state=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].state') + [ "$state" = "ONLINE" ] || issues="${issues}${issues:+; }pool ${state}" + + # 2. every vdev and device ONLINE + local bad + bad=$(echo "$json" | jq -r --arg p "$pool" ' + .pools[$p].vdevs[] | .. | objects + | select(.state? and .state != "ONLINE") + | "\(.name // "unknown"):\(.state)"' 2>/dev/null | paste -sd, -) + [ -z "$bad" ] || issues="${issues}${issues:+; }devices ${bad}" + + # 3. resilver in progress + local fn st + fn=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.function // "NONE"') + st=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.state // "NONE"') + if [ "$fn" = "RESILVER" ] && [ "$st" = "SCANNING" ]; then + issues="${issues}${issues:+; }resilvering" + fi + + # 4. read/write/checksum errors. ZFS reports these as strings. + local errs + errs=$(echo "$json" | jq -r --arg p "$pool" ' + .pools[$p].vdevs[] | .. | objects + | select(.name? and ((.read_errors // "0" | tonumber) > 0 + or (.write_errors // "0" | tonumber) > 0 + or (.checksum_errors // "0" | tonumber) > 0)) + | "\(.name) r=\(.read_errors) w=\(.write_errors) c=\(.checksum_errors)"' 2>/dev/null | paste -sd, -) + [ -z "$errs" ] || issues="${issues}${issues:+; }errors ${errs}" + + # 5. errors from the last scrub + local scan_err + scan_err=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.errors // "0"') + if [ -n "$scan_err" ] && [ "$scan_err" != "0" ] && [ "$scan_err" != "null" ]; then + issues="${issues}${issues:+; }scan errors ${scan_err}" + fi + + # 6. Capacity. Not an error condition in `zpool status` - a 95% full pool is + # reported perfectly ONLINE - so it has to be read separately, and it is + # the failure you get warning of rather than the one you discover. + local capacity + capacity=$(zpool list -H -o capacity "$pool" 2>/dev/null | tr -dc '0-9') + if [ -z "$capacity" ]; then + issues="${issues}${issues:+; }cannot read capacity" + elif [ "$capacity" -ge {{ healthcheck_zfs_capacity_threshold }} ]; then + issues="${issues}${issues:+; }pool ${capacity}% full (>={{ healthcheck_zfs_capacity_threshold }}%)" + fi + + if [ -n "$issues" ]; then MESSAGE="$issues"; return 1; fi + + local scrub + scrub=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.start_time // "never"') + MESSAGE="${pool} ONLINE, ${capacity}% full, last scrub ${scrub}" + return 0 diff --git a/ansible/roles/healthcheck/templates/healthcheck.service.j2 b/ansible/roles/healthcheck/templates/healthcheck.service.j2 new file mode 100644 index 0000000..3ddd8aa --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description={{ healthcheck_description }} +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart={{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +Environment=HEALTHCHECK_PUSH_BASE={{ healthcheck_push_base }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..6667878 --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 @@ -0,0 +1,82 @@ +#!/bin/bash +# {{ healthcheck_name }} — {{ healthcheck_description }} +# Managed by Ansible (roles/healthcheck). Do not edit on the host. +# +# The exit code is the real answer; systemd stores it: +# systemctl is-failed {{ healthcheck_name }}-healthcheck.service +# The push below is an optional extra, and having no URL is normal. + +set -uo pipefail + +LOG_FILE="{{ healthcheck_log_dir }}/{{ healthcheck_name }}.log" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +# Set only by checks that report several results; see healthcheck_push_base. +PUSH_BASE="${HEALTHCHECK_PUSH_BASE:-}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $*" >> "$LOG_FILE"; } + +# Report to Gatus as an external endpoint. Note this is a POST with a bearer +# token, not a GET with a query string - it is not the shape Uptime Kuma used. +report() { + local success="$1" message="$2" + + # No push URL is normal, not an error: the exit code below is a complete + # answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + + local code + code=$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + --max-time 15 --retry 2 --retry-delay 3 \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null) + + if [ "$code" = "200" ]; then + log "reported success=${success}" + else + log "ERROR: report failed (HTTP ${code})" + return 1 + fi +} + +# Report to an arbitrary endpoint key under PUSH_BASE. Used by checks that +# produce one result per item rather than a single verdict. +report_key() { + local key="$1" success="$2" message="$3" + [ -n "$PUSH_BASE" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true +} + +# ── the check ──────────────────────────────────────────────────────────────── +# +# The blank line before the closing brace below is load-bearing. Jinja strips an +# included template's trailing newline, and trim_blocks (on by default in +# Ansible) then eats the newline after {% raw %}{% endif %}{% endraw %} - so without it the brace lands +# on the same line as the check body's last statement, producing `return 0}` and +# a script that dies with "syntax error: unexpected end of file". +check() { +{% if healthcheck_check %} +{% include 'checks/' ~ healthcheck_check ~ '.sh.j2' %} +{% else %} + {{ healthcheck_command }} +{% endif %} + +} + +MESSAGE="" +if check; then + log "OK${MESSAGE:+ - $MESSAGE}" + report "true" "${MESSAGE:-ok}" + exit 0 +else + log "FAILED${MESSAGE:+ - $MESSAGE}" + report "false" "${MESSAGE:-check failed}" + exit 1 +fi diff --git a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..0271735 --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Run {{ healthcheck_description }} +Requires={{ healthcheck_name }}-healthcheck.service + +[Timer] +OnBootSec={{ healthcheck_boot_delay }} +{% if healthcheck_on_calendar %} +OnCalendar={{ healthcheck_on_calendar }} +{% else %} +OnUnitActiveSec={{ healthcheck_interval }} +{% endif %} +# Run a missed occurrence on the next boot rather than silently skipping it. +Persistent=true +# Spread the pushes out so twelve hosts do not all report in the same second. +RandomizedDelaySec={{ healthcheck_randomized_delay | default('30') }} + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/mempool/README.md b/ansible/roles/mempool/README.md new file mode 100644 index 0000000..d8282cc --- /dev/null +++ b/ansible/roles/mempool/README.md @@ -0,0 +1,69 @@ +# `mempool` + +Deploys the [Mempool](https://mempool.space) block explorer as a three-container +Docker Compose stack — MariaDB, backend, frontend — on `mempool-box`, and keeps a +health check on each. + +Converted from `deploy_mempool_playbook.yml` (745 lines) under Plan 6. The +playbook is now 37 lines: this role, plus a second play that publishes the +frontend through Caddy on the edge host. + +## Phases + +| | | +|---|---| +| `docker.yml` | Docker engine: repo, key, packages, service | +| `deploy.yml` | directories, `docker-compose.yml`, pull, up, wait-for-healthy | +| `healthcheck.yml` | three check scripts, three services, three timers | + +## Three health checks, not one + +Mempool is three moving parts and knowing *which* one is down is the point, so +each gets its own check, unit and timer, driven by the `mempool_healthchecks` +list: + +| | checks | +|---|---| +| `mariadb` | `docker inspect` health status of `mempool-db` | +| `backend` | `GET /api/v1/backend-info` | +| `frontend` | `GET /` | + +Each records its answer in its exit code, which systemd keeps: +`systemctl is-failed mempool-backend-healthcheck.service`. Reporting elsewhere +is one field per check, `push_url`, and is the plug-in point for whatever +monitoring exists. Empty means check, exit honestly, report nowhere. The URLs +are credentials, so callers pass them from the vault. + +Nothing here is specific to a monitoring product. The embedded Python that +created monitors over the Uptime Kuma API, the `/tmp` credentials file, the +push-URL file read back and parsed, and three systemd `Environment=` rewrites +are gone. + +## MariaDB owns its own data directory + +`{{ mempool_mysql_dir }}` is bind-mounted into the container, which runs as uid +**999** and must create files there. The playbook this replaced declared +`owner: "{{ ansible_user }}"` (1000) on it, which had drifted from reality ever +since the containers were created — unnoticed, because the playbook had not been +run since. + +That was not academic. The first real run of this role pulled a newer +`mariadb:10.11` and recreated `mempool-db`; had the chown still been in place, +MariaDB would have come back to a directory it could not write. The role now +ensures the directory exists and leaves ownership to the container. + +## `mempool_frontend_port` lives in `services_config.yml` + +Two hosts need it: this role deploys the frontend on `mempool-box`, and the Caddy +play proxies to it from the edge host. A role default is invisible to the second +play, so the value lives in `service_settings.mempool.frontend_port` and the role +default derives from it. + +## Expect `changed=2` on a converged host + +`Pull Mempool images` and `Deploy Mempool containers with docker compose` are +bare `command:` tasks with no `changed_when`, so they always report changed. +That is the idempotent floor, not drift. Everything else reports `ok`. + +**`mariadb:10.11` is a moving tag**, so a run can pull a newer patch release and +recreate the database container. Pin it if that is not what you want. diff --git a/ansible/roles/mempool/defaults/main.yml b/ansible/roles/mempool/defaults/main.yml new file mode 100644 index 0000000..9ce8f6a --- /dev/null +++ b/ansible/roles/mempool/defaults/main.yml @@ -0,0 +1,59 @@ +# Mempool Configuration Variables + +# Version - Pinned to specific release +mempool_version: "v3.2.1" + +# Directories +mempool_dir: /opt/mempool +mempool_data_dir: "{{ mempool_dir }}/data" +mempool_mysql_dir: "{{ mempool_dir }}/mysql" + +# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) +bitcoin_host: "knots-box" +bitcoin_rpc_port: 8332 +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml + +# Network - Fulcrum Electrum server (via Tailnet Magic DNS) +fulcrum_host: "fulcrum-box" +fulcrum_port: 50001 +fulcrum_tls: "false" + +# Mempool network mode +mempool_network: "mainnet" + +# Container ports (internal) +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/mempool_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +mempool_frontend_port: 8080 +mempool_backend_port: 8999 + +# MariaDB settings +mariadb_database: "mempool" +mariadb_user: "mempool" +# Note: mariadb_mempool_password is loaded from group_vars/all/vault.yml + + + +# --- Health checks ---------------------------------------------------------- +# Three independent checks, because Mempool is three moving parts and knowing +# WHICH one is down is the whole point. Each records its answer in its exit +# code, which systemd keeps: +# systemctl is-failed mempool-backend-healthcheck.service +# +# push_url is where to report, and is the single plug-in point for whatever +# monitoring exists. Empty means check, exit honestly, report nowhere. +# The URLs are credentials, so callers pass them from the vault. +healthcheck_push_token: "" + +mempool_healthchecks: + - name: mariadb + label: MariaDB + push_url: "" + - name: backend + label: Backend + push_url: "" + - name: frontend + label: Frontend + push_url: "" diff --git a/ansible/roles/mempool/tasks/deploy.yml b/ansible/roles/mempool/tasks/deploy.yml new file mode 100644 index 0000000..d98df7e --- /dev/null +++ b/ansible/roles/mempool/tasks/deploy.yml @@ -0,0 +1,75 @@ +--- +- name: Create mempool directories + file: + path: "{{ item }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + loop: + - "{{ mempool_dir }}" + - "{{ mempool_data_dir }}" + +# MariaDB owns its own data directory. The container runs as uid 999 and has to +# create files in there; this playbook declared owner: {{ ansible_user }} (1000), +# which had been drifting from reality ever since the containers were created and +# would have broken MariaDB the first time it needed a new file. It went +# unnoticed only because the playbook had not been run since. +# +# So: ensure the directory exists, and let the container own it. On a fresh +# install the mariadb image's entrypoint sets ownership itself. +- name: Ensure the MariaDB data directory exists + file: + path: "{{ mempool_mysql_dir }}" + state: directory + +- name: Create docker-compose.yml for Mempool + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ mempool_dir }}/docker-compose.yml" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' +- name: Pull Mempool images + command: docker compose pull + args: + chdir: "{{ mempool_dir }}" + +- name: Deploy Mempool containers with docker compose + command: docker compose up -d + args: + chdir: "{{ mempool_dir }}" + +- name: Wait for MariaDB to be healthy + command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db + register: mariadb_health + until: mariadb_health.stdout == 'healthy' + retries: 30 + delay: 10 + changed_when: false + +- name: Wait for Mempool backend to start + uri: + url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + method: GET + status_code: 200 + timeout: 10 + register: backend_check + until: backend_check.status == 200 + retries: 30 + delay: 10 + ignore_errors: yes + +- name: Wait for Mempool frontend to be available + uri: + url: "http://localhost:{{ mempool_frontend_port }}" + method: GET + status_code: 200 + timeout: 10 + register: frontend_check + until: frontend_check.status == 200 + retries: 20 + delay: 5 + ignore_errors: yes + +# ═════════════════════════════════════════════════════════════════════════ diff --git a/ansible/roles/mempool/tasks/docker.yml b/ansible/roles/mempool/tasks/docker.yml new file mode 100644 index 0000000..b97e63e --- /dev/null +++ b/ansible/roles/mempool/tasks/docker.yml @@ -0,0 +1,72 @@ +--- +- name: Remove old Docker-related packages + apt: + name: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + purge: yes + autoremove: yes + +- name: Update apt cache + apt: + update_cache: yes + +- name: Install prerequisites + apt: + name: + - ca-certificates + - curl + state: present + +- name: Create directory for Docker GPG key + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + +- name: Download Docker GPG key + get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + +- name: Get Debian architecture + command: dpkg --print-architecture + register: deb_arch + changed_when: false + +- name: Add Docker repository + apt_repository: + repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + filename: docker + state: present + update_cache: yes + +- name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + +- name: Ensure Docker is started and enabled + systemd: + name: docker + enabled: yes + state: started + +- name: Add user to docker group + user: + name: "{{ ansible_user }}" + groups: docker + append: yes + diff --git a/ansible/roles/mempool/tasks/healthcheck.yml b/ansible/roles/mempool/tasks/healthcheck.yml new file mode 100644 index 0000000..1e1ac28 --- /dev/null +++ b/ansible/roles/mempool/tasks/healthcheck.yml @@ -0,0 +1,58 @@ +--- +# Three checks, one per moving part. The Uptime Kuma specifics that used to +# follow — an embedded Python script creating monitors over the API, a /tmp +# credentials file, a push-URL file read back and parsed, and three systemd +# Environment= rewrites — are gone. Where each reports is now hc.push_url. +- name: Create Mempool health check scripts + ansible.builtin.template: + src: "healthcheck-{{ hc.name }}.sh.j2" + dest: "/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh" + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" + +- name: Create systemd services for health checks + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.service" + owner: root + group: root + mode: "0600" + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" + +- name: Create systemd timers for health checks + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.timer" + owner: root + group: root + mode: '0644' + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +# Ungated on purpose: enabling a timer is deployment, not monitoring. The +# deprecation banner swept this up with the push plumbing, so Ansible stopped +# managing three timers that are in fact running on the host. +- name: Enable and start health check timers + systemd: + name: "mempool-{{ hc.name }}-healthcheck.timer" + enabled: yes + state: started + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" diff --git a/ansible/roles/mempool/tasks/main.yml b/ansible/roles/mempool/tasks/main.yml new file mode 100644 index 0000000..9feb652 --- /dev/null +++ b/ansible/roles/mempool/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: docker.yml +- ansible.builtin.import_tasks: deploy.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/mempool/templates/docker-compose.yml.j2 b/ansible/roles/mempool/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..fda2ad1 --- /dev/null +++ b/ansible/roles/mempool/templates/docker-compose.yml.j2 @@ -0,0 +1,75 @@ +# All containers use host network for Tailscale MagicDNS resolution +services: + mariadb: + image: mariadb:10.11 + container_name: mempool-db + restart: unless-stopped + network_mode: host + environment: + MYSQL_DATABASE: "{{ mariadb_database }}" + MYSQL_USER: "{{ mariadb_user }}" + MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" + MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" + volumes: + - {{ mempool_mysql_dir }}:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + mempool-backend: + image: mempool/backend:{{ mempool_version }} + container_name: mempool-backend + restart: unless-stopped + network_mode: host + environment: + # Database (localhost since all containers share host network) + DATABASE_ENABLED: "true" + DATABASE_HOST: "127.0.0.1" + DATABASE_DATABASE: "{{ mariadb_database }}" + DATABASE_USERNAME: "{{ mariadb_user }}" + DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" + # Bitcoin Core/Knots (via Tailnet MagicDNS) + CORE_RPC_HOST: "{{ bitcoin_host }}" + CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" + CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" + CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" + # Electrum (Fulcrum via Tailnet MagicDNS) + ELECTRUM_HOST: "{{ fulcrum_host }}" + ELECTRUM_PORT: "{{ fulcrum_port }}" + ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" + # Mempool settings + MEMPOOL_NETWORK: "{{ mempool_network }}" + MEMPOOL_BACKEND: "electrum" + MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" + volumes: + - {{ mempool_data_dir }}:/backend/cache + depends_on: + mariadb: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + mempool-frontend: + image: mempool/frontend:{{ mempool_version }} + container_name: mempool-frontend + restart: unless-stopped + network_mode: host + environment: + FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" + BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" + depends_on: + - mempool-backend + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s diff --git a/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 new file mode 100644 index 0000000..fe0f57c --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash +# Mempool backend health check — managed by Ansible (roles/mempool) +# The exit code is the answer; systemd keeps it. Reporting is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +BACKEND_PORT="{{ mempool_backend_port }}" + +check() { + curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 +} + +report() { + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "$1" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true +} + +if check; then report up "OK"; exit 0 +else echo "Mempool backend not responding"; report down "Mempool backend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 new file mode 100644 index 0000000..8452202 --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash +# Mempool frontend health check — managed by Ansible (roles/mempool) +# The exit code is the answer; systemd keeps it. Reporting is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +FRONTEND_PORT="{{ mempool_frontend_port }}" + +check() { + curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 +} + +report() { + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "$1" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true +} + +if check; then report up "OK"; exit 0 +else echo "Mempool frontend not responding"; report down "Mempool frontend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 b/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 new file mode 100644 index 0000000..922adab --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash +# Mempool MariaDB health check — managed by Ansible (roles/mempool) +# The exit code is the answer; systemd keeps it. Reporting is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +check() { +{% raw %} + [ "$(docker inspect --format='{{.State.Health.Status}}' mempool-db 2>/dev/null)" = "healthy" ] +{% endraw %} +} + +report() { + # No push URL is normal, not an error. The previous version logged + # "ERROR: UPTIME_KUMA_PUSH_URL not set" on every fire, once a minute. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "$1" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true +} + +if check; then report up "OK"; exit 0 +else echo "MariaDB container unhealthy"; report down "MariaDB container unhealthy"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck.service.j2 b/ansible/roles/mempool/templates/healthcheck.service.j2 new file mode 100644 index 0000000..76fa4e9 --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Mempool {{ hc.label }} Health Check +After=network.target docker.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ hc.push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/mempool/templates/healthcheck.timer.j2 b/ansible/roles/mempool/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..868cb7c --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Mempool {{ hc.name }} Health Check Timer + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/phoenixd/README.md b/ansible/roles/phoenixd/README.md new file mode 100644 index 0000000..3c147e3 --- /dev/null +++ b/ansible/roles/phoenixd/README.md @@ -0,0 +1,56 @@ +# `phoenixd` + +Deploys and runs [phoenixd](https://phoenix.acinq.co/server), an ACINQ Lightning +node, on the edge host. LNBits uses it as a wallet backend. The HTTP API stays on +loopback — phoenixd is never published through Caddy. + +Converted from `deploy_phoenixd_playbook.yml` (552 lines) under Plan 6. The +playbook is now 18 lines. + +## Phases + +| | | +|---|---| +| `install.yml` | packages, system user, directories, versioned download and install | +| `service.yml` | systemd unit, start, then first-boot checks (config written, seed created) | +| `healthcheck.yml` | check script, unit, timer | + +## The seed + +`{{ phoenixd_data_dir }}/seed.dat` **is** the funds. phoenixd is deliberately +excluded from the automated backups (Plan 5, Model C): the seed is twelve fixed +words that never change, so an automated job would only manufacture more copies +of a static secret on more machines. Write them down offline, once. + +Note the live file is mode `0644`. That is phoenixd's own doing, not this role's, +and it is worth tightening. + +## Monitoring: one variable, no product knowledge + +The check asks the node itself — the service must be active **and** +`phoenix-cli getinfo` must return a `nodeId` — and records the answer in its exit +code, which systemd keeps: + +```bash +systemctl is-failed phoenixd-healthcheck.service +``` + +That is a complete answer with no monitoring system involved. To report +elsewhere, set `healthcheck_push_url` to anything accepting an HTTP ping. Gone +from this role: the embedded Python that created monitors over the Uptime Kuma +API, the `/tmp` credentials file, the push-URL file written and parsed back, and +the systemd `Environment=` rewrite. + +### Two things the conversion fixed + +**The check used to log an error once a minute.** Its `Environment=` push URL had +been empty since the decommissioning, and the script printed +`ERROR: UPTIME_KUMA_PUSH_URL not set` on every fire — roughly 1,400 times a day. +The exit code was still correct, so nothing was broken; it was pure noise, and +noise that trains you to ignore the log. An unset push URL is now normal and +silent. + +**`Enable and start phoenixd health check timer` was guarded by +`uptime_kuma_enabled`** and so had not run since the decommissioning — while the +timer itself was still live on the host from before. Ansible had quietly stopped +managing something that was still running. Ungated. diff --git a/ansible/services/phoenixd/phoenixd_vars.yml b/ansible/roles/phoenixd/defaults/main.yml similarity index 63% rename from ansible/services/phoenixd/phoenixd_vars.yml rename to ansible/roles/phoenixd/defaults/main.yml index f67c302..2fcf1ea 100644 --- a/ansible/services/phoenixd/phoenixd_vars.yml +++ b/ansible/roles/phoenixd/defaults/main.yml @@ -35,18 +35,22 @@ phoenixd_http_bind_port: 9740 # Optional webhook for payment events. Leave empty to disable. phoenixd_webhook_url: "" -# Monitoring + + + +# --- Health check ----------------------------------------------------------- +# The check asks phoenixd itself whether it is healthy (service active AND the +# node answers getinfo with a nodeId) and records the answer in its exit code, +# which systemd keeps: +# systemctl is-failed phoenixd-healthcheck.service +# That is a complete answer with no monitoring system involved. phoenixd_healthcheck_script_path: /usr/local/bin/phoenixd-healthcheck-push.sh phoenixd_healthcheck_service_name: phoenixd-healthcheck -phoenixd_monitor_name: "Phoenixd" -# Remote access -remote_host_name: "{{ groups['edge'] | first }}" -remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" -remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" -remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" -remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" - -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/phoenixd-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/phoenixd_backup.sh" +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. +# Empty means check, log, exit honestly, report nowhere. Any endpoint that +# accepts an HTTP ping works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/phoenixd/handlers/main.yml b/ansible/roles/phoenixd/handlers/main.yml new file mode 100644 index 0000000..7779684 --- /dev/null +++ b/ansible/roles/phoenixd/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: Restart phoenixd + systemd: + name: phoenixd + state: restarted + daemon_reload: yes + +- name: Restart phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/phoenixd/tasks/healthcheck.yml b/ansible/roles/phoenixd/tasks/healthcheck.yml new file mode 100644 index 0000000..d439047 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/healthcheck.yml @@ -0,0 +1,68 @@ +--- +# Everything here answers "is phoenixd healthy" and records the answer. The +# Uptime Kuma specifics that used to follow it — an embedded Python script +# creating monitors over the API, a /tmp credentials file, a push-URL file read +# back and parsed, and a systemd Environment= rewrite — are gone. What reports +# where is now one variable, healthcheck_push_url. See the role README. +- name: Create phoenixd health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ phoenixd_healthcheck_script_path }}" + owner: root + group: root + mode: "0755" + validate: "bash -n %s" + +- name: Create phoenixd health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" + owner: root + group: root + mode: "0600" + notify: Restart phoenixd health check timer + +- name: Create phoenixd health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" + owner: root + group: root + mode: "0644" + notify: Restart phoenixd health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated on purpose. This was guarded by `uptime_kuma_enabled`, but enabling a +# timer is deployment, not monitoring — the deprecation banner swept it up with +# the push plumbing. The timer is in fact running on the host, from before the +# decommissioning, so the guard meant Ansible had stopped managing something +# that was still live. +- name: Enable and start phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + enabled: yes + state: started + +- name: Display post-install information + debug: + msg: | + ✓ phoenixd {{ phoenixd_version }} deployed + + Status: systemctl status phoenixd + Logs: journalctl -u phoenixd -f + CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo + HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) + Data dir: {{ phoenixd_data_dir }} + Health: systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service + + API password (needed to wire LNBits up to this node): + sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf + + ⚠️ BACK UP THE SEED: {{ phoenixd_data_dir }}/seed.dat + Losing it means losing the funds. phoenixd is deliberately excluded + from the automated backups (Plan 5, Model C) because the seed is 12 + fixed words — write them down offline, once: + sudo cat {{ phoenixd_data_dir }}/seed.dat diff --git a/ansible/roles/phoenixd/tasks/install.yml b/ansible/roles/phoenixd/tasks/install.yml new file mode 100644 index 0000000..89a4fa0 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/install.yml @@ -0,0 +1,125 @@ +--- +- name: Install phoenixd runtime dependencies + apt: + name: + - unzip + - curl + state: present + update_cache: yes + +# System User and Directories +- name: Create phoenixd system group + group: + name: "{{ phoenixd_group }}" + system: yes + +- name: Create phoenixd system user + user: + name: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ phoenixd_home }}" + create_home: yes + comment: "phoenixd Lightning node" + +- name: Create phoenixd home directory + file: + path: "{{ phoenixd_home }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0750" + +- name: Create phoenixd data directory + file: + path: "{{ phoenixd_data_dir }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0700" + +# Download and Install +- name: Check if phoenixd is already installed + stat: + path: "{{ phoenixd_bin_dir }}/phoenixd" + register: phoenixd_binary + +- name: Check installed phoenixd version + command: "{{ phoenixd_bin_dir }}/phoenixd --version" + register: phoenixd_installed_version + changed_when: false + failed_when: false + when: phoenixd_binary.stat.exists + +- name: Decide whether phoenixd needs installing + set_fact: + phoenixd_needs_install: >- + {{ not phoenixd_binary.stat.exists + or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} + +- name: Download phoenixd {{ phoenixd_version }} + get_url: + url: "{{ phoenixd_url }}" + dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + mode: "0644" + when: phoenixd_needs_install | bool + +- name: Create temporary extraction directory + file: + path: /tmp/phoenixd-extract + state: directory + mode: "0755" + when: phoenixd_needs_install | bool + +- name: Extract phoenixd archive + unarchive: + src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + dest: /tmp/phoenixd-extract + remote_src: yes + when: phoenixd_needs_install | bool + +- name: Locate extracted binaries + find: + paths: /tmp/phoenixd-extract + patterns: "{{ item }}" + recurse: yes + file_type: file + register: phoenixd_extracted + loop: + - phoenixd + - phoenix-cli + when: phoenixd_needs_install | bool + +- name: Fail if the archive did not contain the expected binaries + assert: + that: + - item.files | length > 0 + fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + +- name: Install phoenixd and phoenix-cli binaries + copy: + src: "{{ item.files[0].path }}" + dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" + remote_src: yes + owner: root + group: root + mode: "0755" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + notify: Restart phoenixd + +- name: Clean up phoenixd download artifacts + file: + path: "{{ item }}" + state: absent + loop: + - "/tmp/phoenixd-{{ phoenixd_version }}.zip" + - /tmp/phoenixd-extract + diff --git a/ansible/roles/phoenixd/tasks/main.yml b/ansible/roles/phoenixd/tasks/main.yml new file mode 100644 index 0000000..bc7ff05 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/phoenixd/tasks/service.yml b/ansible/roles/phoenixd/tasks/service.yml new file mode 100644 index 0000000..79bf6f5 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/service.yml @@ -0,0 +1,52 @@ +--- +- name: Build phoenixd command line arguments + set_fact: + phoenixd_args: >- + {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) + + ['--chain', phoenixd_chain] + + ['--auto-liquidity', phoenixd_auto_liquidity] + + ['--http-bind-ip', phoenixd_http_bind_ip] + + ['--http-bind-port', phoenixd_http_bind_port | string] + + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) + + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) + + ['--silent'] }} + +- name: Create phoenixd systemd service + ansible.builtin.template: + src: phoenixd.service.j2 + dest: /etc/systemd/system/phoenixd.service + owner: root + group: root + mode: "0644" + notify: Restart phoenixd + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start phoenixd + systemd: + name: phoenixd + enabled: yes + state: started + +- name: Flush handlers so phoenixd is running before we inspect its data dir + meta: flush_handlers + +# --- First boot checks --- +- name: Wait for phoenixd to write its config file + wait_for: + path: "{{ phoenixd_data_dir }}/phoenix.conf" + state: present + timeout: 120 + +- name: Check that the seed file exists + stat: + path: "{{ phoenixd_data_dir }}/seed.dat" + register: phoenixd_seed_file + +- name: Fail if phoenixd did not create a seed + assert: + that: + - phoenixd_seed_file.stat.exists + fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" diff --git a/ansible/roles/phoenixd/templates/healthcheck.service.j2 b/ansible/roles/phoenixd/templates/healthcheck.service.j2 new file mode 100644 index 0000000..6ff21ab --- /dev/null +++ b/ansible/roles/phoenixd/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=phoenixd Health Check +After=network.target phoenixd.service + +[Service] +Type=oneshot +User=root +ExecStart={{ phoenixd_healthcheck_script_path }} +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/phoenixd/templates/healthcheck.sh.j2 b/ansible/roles/phoenixd/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..a1717c5 --- /dev/null +++ b/ansible/roles/phoenixd/templates/healthcheck.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash +# phoenixd health check — managed by Ansible (roles/phoenixd) +# +# Asks the node whether it is healthy and records the answer in the exit code, +# which systemd keeps: +# systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service +# That is a complete answer on its own. Reporting anywhere else is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" + +check_phoenixd() { + # Service must be active and the node must answer getinfo. + # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, + # but not the bind address, so pass it explicitly. + systemctl is-active --quiet phoenixd && \ + {{ phoenixd_bin_dir }}/phoenix-cli \ + --http-bind-ip {{ phoenixd_http_bind_ip }} \ + --http-bind-port {{ phoenixd_http_bind_port }} \ + getinfo 2>/dev/null | grep -q '"nodeId"' +} + +report() { + local status=$1 msg=$2 + # No push URL configured is NORMAL, not an error: the exit code below still + # answers the question. The previous version logged ERROR here on every + # single fire, once a minute, which is noise that trains you to ignore it. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true +} + +if check_phoenixd; then + report "up" "OK" + exit 0 +else + echo "phoenixd is not responding" + report "down" "phoenixd not responding" + exit 1 +fi diff --git a/ansible/roles/phoenixd/templates/healthcheck.timer.j2 b/ansible/roles/phoenixd/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..ee07257 --- /dev/null +++ b/ansible/roles/phoenixd/templates/healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=phoenixd Health Check Timer + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/phoenixd/templates/phoenixd.service.j2 b/ansible/roles/phoenixd/templates/phoenixd.service.j2 new file mode 100644 index 0000000..44fdb94 --- /dev/null +++ b/ansible/roles/phoenixd/templates/phoenixd.service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=phoenixd - Lightning Network Node +Documentation=https://phoenix.acinq.co/server +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ phoenixd_user }} +Group={{ phoenixd_group }} +WorkingDirectory={{ phoenixd_home }} +Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} +ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} +Restart=always +RestartSec=30 +TimeoutStartSec=120 +TimeoutStopSec=120 +StandardOutput=journal +StandardError=journal + +# Hardening: the node only ever writes to its own data directory +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={{ phoenixd_data_dir }} + +LimitNOFILE=65535 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/signal_api/README.md b/ansible/roles/signal_api/README.md new file mode 100644 index 0000000..eadcfb7 --- /dev/null +++ b/ansible/roles/signal_api/README.md @@ -0,0 +1,133 @@ +# signal_api + +Runs [signal-cli-rest-api](https://github.com/bbernhard/signal-cli-rest-api) on +the `observability` host. Gatus uses it to deliver alerts over Signal. + +Gatus does not speak Signal — it POSTs JSON to this service, which holds the +Signal identity and does the protocol work. + +## It is never published, and that is not optional + +**This API has no authentication of any kind.** No key, no token, no basic auth. +Anything that can reach the port can send messages as your identity and read +your Signal. So the compose file publishes **no ports at all** and there is no +Caddy vhost. + +Gatus reaches it over a shared docker network (`monitoring`) by service name: +`http://signal-api:8080`. That is also *why* a shared network is needed rather +than a published port — Gatus runs in a container, so `127.0.0.1` for Gatus is +the Gatus container, not the host. + +The network is created by an explicit Ansible task in both this role and +`gatus`, so neither stack has to be deployed before the other. + +## MODE, and why `native` + +Upstream offers `normal`, `native`, `json-rpc` and `json-rpc-native`. The +json-rpc modes keep a resident JVM daemon and upstream describes them as +"increased memory". + +**This VPS has 464 MB of RAM**, already running Gatus and Caddy. A resident JVM +is not affordable. `native` runs a precompiled GraalVM binary per request — no +daemon, no resident cost — and alerts are rare enough that paying startup cost +per alert is the right trade. + +## Linking the device — a one-time manual step + +Ansible cannot scan a QR code, so this is manual. **Do not use +`/v1/qrcodelink`** — it is broken in `native` mode. + +### The trap + +`GET /v1/qrcodelink?device_name=...` returns: + +```json +{"error":"Couldn't create QR code: no data to encode"} +``` + +The linking itself is fine: running the binary directly inside the container +emits a perfectly good provisioning URI. + +``` +$ docker exec signal-api signal-cli-native link -n gatus +sgnl://linkdevice?uuid=...&pub_key=... +``` + +It is the REST wrapper that fails to capture that output in `native` mode. + +**Do not "fix" this by switching MODE to `normal` or `json-rpc`.** That puts a +JVM in the path of *every alert* on a 464 MB host, permanently degrading the +running system to work around a step performed once. Generate the QR yourself +instead. + +### The procedure + +**`docker exec` runs as root, but the service runs as uid 1000.** Without +`--config`, signal-cli writes the linked account to `/root/.local/share/signal-cli` +— the container's ephemeral layer, NOT the mounted volume. It looks like it +worked (`Associated with: +34…`), `/v1/accounts` keeps returning `[]`, and the +account is destroyed on the next `docker compose up`. Always pass `--config`. + +1. Start the link and capture the URI. It must keep running while you scan: + + docker exec signal-api sh -c "rm -f /tmp/link.uri; \ + nohup signal-cli-native --config /home/.local/share/signal-cli \ + link -n gatus > /tmp/link.uri 2>/tmp/link.log & echo started" + sleep 10 + docker exec signal-api cat /tmp/link.uri + + Do **not** add `setsid`, and do **not** background `docker exec` itself from + the host — the first stops the URI appearing, the second is killed when the + Ansible task returns. The output is block-buffered because stdout is a file, + so the URI appears only after several seconds; `stdbuf` does not help, as the + buffering is GraalVM's, not libc's. + +2. Render the QR on your own machine and scan it: + + qrencode -o /tmp/qr.png -s 12 -m 4 "sgnl://linkdevice?uuid=...&pub_key=..." + +3. Phone: Signal → Settings → Linked devices → **+** → scan. Provisioning links + expire in a couple of minutes, so generate and scan in one sitting. + +4. Confirm — this must list the number, not `[]`: + + docker exec signal-api curl -s http://localhost:8080/v1/accounts + +5. Send a test message: + + docker exec signal-api curl -s -X POST -H "Content-Type: application/json" \ + -d '{"message":"test","number":"+34…","recipients":["+34…"]}' \ + http://localhost:8080/v2/send + +Alerts are sent **from your own number**, so sending to yourself lands in Note +to Self. If the device is ever unlinked from the phone, alerts stop silently — +which is why this service is itself monitored. + +### If the phone says "network error" + +The phone is not the problem. `chat.signal.org` resolves to AWS Global +Accelerator **dualstack** addresses with the AAAA records first, this container +has no IPv6 address at all, and this host's IPv6 path is broken — the same edge +that returned a bogus 404 for the Go tarball. signal-cli reaches for an +unreachable IPv6 address and dies with `Link request error: Connection closed!`, +while the phone can only report a failed handshake. + +That is what `gai.conf` (mounted at `/etc/gai.conf`) fixes. If linking starts +failing again, check it is still mounted and that `getent ahosts chat.signal.org` +returns an IPv4 address first. + +## Backups + +Deliberately **not** backed up. The data directory holds Signal private keys, +and the recovery path is to link again from the phone — which takes a minute and +does not depend on any stored artefact. Backing it up would copy a credential +off the host to buy nothing. + +## Verifying + +```bash +docker ps --filter name=signal-api +docker exec signal-api curl -fsS http://localhost:8080/v1/health +docker exec signal-api curl -fsS http://localhost:8080/v1/accounts +docker logs signal-api --tail 50 +``` diff --git a/ansible/roles/signal_api/defaults/main.yml b/ansible/roles/signal_api/defaults/main.yml new file mode 100644 index 0000000..6bb7d4f --- /dev/null +++ b/ansible/roles/signal_api/defaults/main.yml @@ -0,0 +1,39 @@ +--- +# signal-cli-rest-api: the transport Gatus uses to send Signal messages. +# +# Gatus does not speak Signal. It POSTs JSON to this service, which holds the +# actual Signal identity and does the protocol work. + +# Pinned by digest for the same reason as Gatus: a tag is mutable. +# Upstream publishes no versioned tags worth pinning to, so this pins the +# DIGEST that `latest` resolved to when this was reviewed. `latest` is a moving +# target; a digest is a content address, and `docker compose pull` either +# fetches exactly this image or fails. +signal_api_image_digest: "sha256:2399d449123cdad56c4d859277e3b9127e1a00c4d2ab4601c239882609286cf8" +signal_api_image: "bbernhard/signal-cli-rest-api@{{ signal_api_image_digest }}" + +signal_api_dir: /opt/signal-api +signal_api_data_dir: "{{ signal_api_dir }}/data" + +# MODE matters on this host. Upstream offers normal / native / json-rpc / +# json-rpc-native. json-rpc keeps a resident JVM daemon and upstream describes it +# as "increased memory" - this VPS has 464MB total and already runs Gatus and +# Caddy, so a resident JVM is not affordable. `native` runs a precompiled +# GraalVM binary per request: no daemon, no resident cost, and alerts are rare +# enough that paying startup per alert is the right trade. +signal_api_mode: native + +# Port INSIDE the shared docker network. Never published to the host: this API +# has NO AUTHENTICATION of any kind. Anyone who can reach it can send messages +# as you and read your Signal. +signal_api_port: 8080 + +# Both this and Gatus join this network so Gatus can reach the API by service +# name. Gatus runs in a container, so the host's loopback is NOT reachable from +# it - this is why a shared network is required rather than a published port. +signal_api_network: monitoring +signal_api_service_name: signal-api + +# The uid the upstream image drops to (`setpriv --reuid=1000`). The data +# directory must be owned by it or signal-cli cannot write the account. +signal_api_uid: 1000 diff --git a/ansible/roles/signal_api/tasks/main.yml b/ansible/roles/signal_api/tasks/main.yml new file mode 100644 index 0000000..1aa92b4 --- /dev/null +++ b/ansible/roles/signal_api/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Assert Docker is available + ansible.builtin.command: docker --version + register: signal_docker_check + changed_when: false + +# Created explicitly rather than by either compose file, so neither stack has to +# be deployed before the other and neither owns it. +- name: Ensure the shared monitoring network exists + ansible.builtin.command: "docker network create {{ signal_api_network }}" + register: signal_net + changed_when: "'already exists' not in signal_net.stderr" + failed_when: + - signal_net.rc != 0 + - "'already exists' not in signal_net.stderr" + +- name: Create the signal-api directory + ansible.builtin.file: + path: "{{ signal_api_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +# Owned by the container's uid, NOT root. +# +# The image drops to uid 1000 (`setpriv --reuid=1000`), and a root-owned 0700 +# directory cannot be traversed by uid 1000 - signal-cli then fails to write the +# account and linking silently never completes, leaving a 39-byte accounts.json +# with no accounts and the API returning "Failed to read local accounts list". +# +# 0700 on uid 1000 is still private: only that uid and root can read the Signal +# private keys, which is the property actually wanted. +- name: Create the signal-api data directory owned by the container user + ansible.builtin.file: + path: "{{ signal_api_data_dir }}" + state: directory + owner: "{{ signal_api_uid }}" + group: "{{ signal_api_uid }}" + mode: "0700" + +- name: Write the IPv4-preference resolver config + ansible.builtin.template: + src: gai.conf.j2 + dest: "{{ signal_api_dir }}/gai.conf" + owner: root + group: root + mode: "0644" + +- name: Write the docker compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ signal_api_dir }}/docker-compose.yml" + owner: root + group: root + mode: "0644" + +- name: Pull the pinned signal-api image + ansible.builtin.command: + cmd: docker compose pull + chdir: "{{ signal_api_dir }}" + register: signal_pull + changed_when: "'Downloaded newer image' in signal_pull.stderr or 'Pull complete' in signal_pull.stderr" + +- name: Start signal-api + ansible.builtin.command: + cmd: docker compose up -d --remove-orphans + chdir: "{{ signal_api_dir }}" + register: signal_up + changed_when: "'Started' in signal_up.stderr or 'Created' in signal_up.stderr or 'Recreated' in signal_up.stderr" + +- name: Wait for the API to answer + ansible.builtin.command: + cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/health" + register: signal_health + until: signal_health.rc == 0 + retries: 12 + delay: 5 + changed_when: false + +- name: Report whether an account is linked yet + ansible.builtin.command: + cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/accounts" + register: signal_accounts + changed_when: false + failed_when: false + +- name: Show the linking status + ansible.builtin.debug: + msg: >- + {{ 'Linked account(s): ' ~ signal_accounts.stdout + if (signal_accounts.stdout | default('[]') | trim) not in ['[]', '', 'null'] + else 'NO ACCOUNT LINKED YET - this is a one-time manual step, see the role README.' }} diff --git a/ansible/roles/signal_api/templates/docker-compose.yml.j2 b/ansible/roles/signal_api/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..9825299 --- /dev/null +++ b/ansible/roles/signal_api/templates/docker-compose.yml.j2 @@ -0,0 +1,47 @@ +# Managed by Ansible (roles/signal_api) +services: + {{ signal_api_service_name }}: + image: {{ signal_api_image }} + container_name: {{ signal_api_service_name }} + restart: unless-stopped + + environment: + MODE: "{{ signal_api_mode }}" + + volumes: + # Prefer IPv4. See gai.conf.j2 - without this, signal-cli reaches for + # chat.signal.org's IPv6 address, which is unreachable from here, and + # linking fails with an opaque "network error" on the phone. + - {{ signal_api_dir }}/gai.conf:/etc/gai.conf:ro + # Holds the Signal identity: the linked-device keys and registration + # state. Lose this and the device must be linked again by scanning a new + # QR code from the phone. It is also the most sensitive thing on this + # host - anyone with these keys can send and read Signal as you. + - {{ signal_api_data_dir }}:/home/.local/share/signal-cli + + networks: + - {{ signal_api_network }} + + # NO PORTS. Deliberately. + # + # This API has no authentication whatsoever - no key, no token, nothing. + # Publishing it, even on 127.0.0.1, would expose "send a Signal message as + # this identity" to anything that can reach the host. Gatus talks to it over + # the shared docker network by service name instead, which is why no port is + # published and why there is no Caddy vhost. + + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8080/v1/health"] + interval: 60s + timeout: 5s + retries: 3 + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +networks: + {{ signal_api_network }}: + external: true diff --git a/ansible/roles/signal_api/templates/gai.conf.j2 b/ansible/roles/signal_api/templates/gai.conf.j2 new file mode 100644 index 0000000..9b2cca4 --- /dev/null +++ b/ansible/roles/signal_api/templates/gai.conf.j2 @@ -0,0 +1,17 @@ +# Managed by Ansible (roles/signal_api) +# +# Prefer IPv4 over IPv6 in getaddrinfo. +# +# chat.signal.org resolves to AWS Global Accelerator dualstack addresses, and +# DNS returns the AAAA records first. This container has NO IPv6 address at all, +# and this host's IPv6 path is unreliable anyway - the same edge that made +# Google's IPv6 endpoint return a confident 404 for the Go tarball. +# +# signal-cli would connect to the AAAA address, fail, and report +# Link request error: Connection closed! +# while the phone showed a bare "network error" - a failure with no obvious +# cause on either end. +# +# This line flips the precedence so IPv4-mapped addresses sort first, which is +# the standard glibc fix. It does NOT disable IPv6; it only changes the order. +precedence ::ffff:0:0/96 100 diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml deleted file mode 100644 index c9bd7ca..0000000 --- a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Bitcoin Knots Configuration Variables - -# Version - REQUIRED: Specify exact version/tag to build -bitcoin_knots_version: "v29.2.knots20251110" # Must specify exact version/tag -bitcoin_knots_version_short: "29.2.knots20251110" # Version without 'v' prefix (for tarball URLs) - -# Directories -bitcoin_knots_dir: /opt/bitcoin-knots -bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" -bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets -bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) -bitcoin_conf_dir: /etc/bitcoin - -# Network -bitcoin_rpc_port: 8332 -bitcoin_p2p_port: 8333 -bitcoin_rpc_bind: "0.0.0.0" - -# Build options -bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores -bitcoin_build_prefix: /usr/local - -# Configuration options -bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) -bitcoin_max_connections: 125 -# dbcache will be calculated as 90% of host RAM automatically in playbook - -# ZMQ Configuration -bitcoin_zmq_enabled: true -bitcoin_zmq_bind: "tcp://0.0.0.0" -bitcoin_zmq_port_rawblock: 28332 -bitcoin_zmq_port_rawtx: 28333 -bitcoin_zmq_port_hashblock: 28334 -bitcoin_zmq_port_hashtx: 28335 - -# Service user -bitcoin_user: bitcoin -bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index 1ae5823..d67006b 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -1,757 +1,25 @@ +--- +# Bitcoin Knots: full node built from source, with PGP signature and SHA256 +# verification of the release tarball. The build is guarded by a binary-exists +# check, so a converged host skips the whole 30-60 minute compile. +# +# The chain lives in bitcoin_large_data_dir (/mnt/knots_data, ~875 GB). Nothing +# here touches it; the only `state: absent` tasks target the build tree under +# /opt/bitcoin-knots and run only when the binary is missing. - name: Build and Deploy Bitcoin Knots from Source hosts: bitcoin become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./bitcoin_knots_vars.yml vars: - bitcoin_repo_url: "https://github.com/bitcoinknots/bitcoin.git" - bitcoin_sigs_base_url: "https://raw.githubusercontent.com/bitcoinknots/guix.sigs/knots" - bitcoin_version_major: "{{ bitcoin_knots_version_short | regex_replace('^(\\d+)\\..*', '\\1') }}" - bitcoin_source_tarball_url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Calculate 90% of system RAM for dbcache - set_fact: - bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" - changed_when: false - - - name: Display calculated dbcache value - debug: - msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" - - - - name: Install build dependencies - apt: - name: - - build-essential - - libtool - - autotools-dev - - automake - - pkg-config - - bsdmainutils - - python3 - - python3-pip - - libevent-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-test-dev - - libboost-thread-dev - - libboost-chrono-dev - - libboost-program-options-dev - - libboost-dev - - libssl-dev - - libdb-dev - - libminiupnpc-dev - - libzmq3-dev - - libnatpmp-dev - - libsqlite3-dev - - git - - curl - - wget - - cmake - state: present - update_cache: yes - - - name: Create bitcoin group - group: - name: "{{ bitcoin_group }}" - system: yes - state: present - - - name: Create bitcoin user - user: - name: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ bitcoin_data_dir }}" - create_home: yes - state: present - - - name: Create bitcoin-knots directory - file: - path: "{{ bitcoin_knots_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create bitcoin-knots source directory - file: - path: "{{ bitcoin_knots_source_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create bitcoin data directory (for config, logs, wallets) - file: - path: "{{ bitcoin_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - - - name: Create bitcoin large data directory (for blockchain) - file: - path: "{{ bitcoin_large_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - - - name: Create bitcoin config directory - file: - path: "{{ bitcoin_conf_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Check if bitcoind binary already exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_binary_exists - changed_when: false - - - name: Install gnupg for signature verification - apt: - name: gnupg - state: present - when: not bitcoind_binary_exists.stat.exists - - - name: Import Luke Dashjr's Bitcoin Knots signing key - command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 - register: key_import - changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" - when: not bitcoind_binary_exists.stat.exists - failed_when: key_import.rc != 0 - - - name: Display imported key fingerprint - command: gpg --fingerprint 90C8019E36C2E964 - register: key_fingerprint - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Download SHA256SUMS file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - - - name: Download SHA256SUMS.asc signature file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - - - name: Verify PGP signature on SHA256SUMS file - command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS - register: sha256sums_verification - changed_when: false - failed_when: false # Don't fail here - check for 'Good signature' in next task - when: not bitcoind_binary_exists.stat.exists - - - - name: Display SHA256SUMS verification result - debug: - msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" - when: not bitcoind_binary_exists.stat.exists - - - name: Fail if SHA256SUMS signature verification failed - fail: - msg: "SHA256SUMS signature verification failed. Aborting build." - when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) - - - name: Remove any existing tarball to force fresh download - file: - path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Download Bitcoin Knots source tarball - get_url: - url: "{{ bitcoin_source_tarball_url }}" - dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - mode: '0644' - validate_certs: yes - force: yes - when: not bitcoind_binary_exists.stat.exists - - - name: Calculate SHA256 checksum of downloaded tarball - command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - register: tarball_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Extract expected checksum from SHA256SUMS file - shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' - register: expected_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - failed_when: expected_checksum.stdout == "" - - - name: Display checksum comparison - debug: - msg: - - "Expected: {{ expected_checksum.stdout | trim }}" - - "Actual: {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists - - - name: Verify tarball checksum matches SHA256SUMS - fail: - msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] - - - name: Remove existing source directory if it exists (to force fresh extraction) - file: - path: "{{ bitcoin_knots_source_dir }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Remove extracted directory if it exists (from previous runs) - file: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Extract verified source tarball - unarchive: - src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - dest: "{{ bitcoin_knots_dir }}" - remote_src: yes - when: not bitcoind_binary_exists.stat.exists - - - name: Check if extracted directory exists - stat: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - register: extracted_dir_stat - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Rename extracted directory to expected name - command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" - when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists - - - name: Check if CMakeLists.txt exists - stat: - path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" - register: cmake_exists - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Create CMake build directory - file: - path: "{{ bitcoin_knots_source_dir }}/build" - state: directory - mode: '0755' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - - - name: Configure Bitcoin Knots build with CMake - command: > - cmake - -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} - -DBUILD_BITCOIN_WALLET=OFF - -DCMAKE_BUILD_TYPE=Release - -DWITH_ZMQ=ON - .. - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: configure_result - changed_when: true - - - name: Verify CMake enabled ZMQ - shell: | - set -e - cd "{{ bitcoin_knots_source_dir }}/build" - cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: zmq_check - changed_when: false - - - name: Fail if CMakeLists.txt not found - fail: - msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." - when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) - - - name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) - command: cmake --build . -j{{ bitcoin_build_jobs }} - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - async: 3600 - poll: 0 - register: build_result - changed_when: true - - - name: Check build status - async_status: - jid: "{{ build_result.ansible_job_id }}" - register: build_job_result - until: build_job_result.finished - retries: 120 - delay: 60 - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined - - - name: Fail if build failed - fail: - msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) - - - name: Install Bitcoin Knots binaries - command: cmake --install . - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - changed_when: true - - - name: Verify bitcoind binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_installed - changed_when: false - - - name: Verify bitcoin-cli binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" - register: bitcoin_cli_installed - changed_when: false - - - name: Fail if binaries not found - fail: - msg: "Bitcoin Knots binaries not found after installation" - when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists - - - name: Create bitcoin.conf configuration file - copy: - dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" - content: | - # Bitcoin Knots Configuration - # Generated by Ansible - - # Data directory (blockchain storage) - datadir={{ bitcoin_large_data_dir }} - - # RPC Configuration - server=1 - rpcuser={{ bitcoin_rpc_user }} - rpcpassword={{ bitcoin_rpc_password }} - rpcbind={{ bitcoin_rpc_bind }} - rpcport={{ bitcoin_rpc_port }} - rpcallowip=0.0.0.0/0 - - # Network Configuration - listen=1 - port={{ bitcoin_p2p_port }} - maxconnections={{ bitcoin_max_connections }} - - # Performance - dbcache={{ bitcoin_dbcache_mb }} - - # Transaction Index (optional) - {% if bitcoin_enable_txindex %} - txindex=1 - {% endif %} - - # Logging (to journald via systemd) - logtimestamps=1 - printtoconsole=1 - - # ZMQ Configuration - {% if bitcoin_zmq_enabled | default(false) %} - zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} - zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} - zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} - zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} - {% endif %} - - # Security - disablewallet=1 - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0640' - notify: Restart bitcoind - - - name: Create systemd service file for bitcoind - copy: - dest: /etc/systemd/system/bitcoind.service - content: | - [Unit] - Description=Bitcoin Knots daemon - After=network.target - - [Service] - Type=simple - User={{ bitcoin_user }} - Group={{ bitcoin_group }} - ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf - Restart=always - RestartSec=10 - TimeoutStopSec=600 - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoind - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start bitcoind service - systemd: - name: bitcoind - enabled: yes - state: started - - - name: Wait for bitcoind RPC to be available - uri: - url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" - method: POST - body_format: json - body: - jsonrpc: "1.0" - id: "healthcheck" - method: "getblockchaininfo" - params: [] - user: "{{ bitcoin_rpc_user }}" - password: "{{ bitcoin_rpc_password }}" - status_code: 200 - timeout: 10 - register: rpc_check - until: rpc_check.status == 200 - retries: 30 - delay: 5 - ignore_errors: yes - - - name: Display RPC connection status - debug: - msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Bitcoin Knots health check and push script - when: uptime_kuma_enabled | default(false) - copy: - dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh - content: | - #!/bin/bash - # - # Bitcoin Knots Health Check and Push to Uptime Kuma - # Checks if bitcoind RPC is responding and pushes status to Uptime Kuma - # - - RPC_HOST="{{ bitcoin_rpc_bind }}" - RPC_PORT={{ bitcoin_rpc_port }} - RPC_USER="{{ bitcoin_rpc_user }}" - RPC_PASSWORD="{{ bitcoin_rpc_password }}" - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - # Check if bitcoind RPC is responding - check_bitcoind() { - local response - response=$(curl -s --max-time 30 \ - --user "${RPC_USER}:${RPC_PASSWORD}" \ - --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ - --header 'Content-Type: application/json' \ - "http://${RPC_HOST}:${RPC_PORT}" 2>&1) - - if [ $? -eq 0 ]; then - # Check if response contains a non-null error - # Successful responses have "error": null, failures have "error": {...} - if echo "$response" | grep -q '"error":null\|"error": null'; then - return 0 - else - return 1 - fi - else - return 1 - fi - } - - # Push status to Uptime Kuma - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - if ! curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then - echo "ERROR: Failed to push to Uptime Kuma" - return 1 - fi - } - - # Main health check - if check_bitcoind; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "bitcoind RPC not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Install curl for health check script - apt: - name: curl - state: present - - - name: Create systemd timer for Bitcoin Knots health check - copy: - dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer - content: | - [Unit] - Description=Bitcoin Knots Health Check Timer - Requires=bitcoind.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for Bitcoin Knots health check - when: uptime_kuma_enabled | default(false) - copy: - dest: /etc/systemd/system/bitcoin-knots-healthcheck.service - content: | - [Unit] - Description=Bitcoin Knots Health Check and Push to Uptime Kuma - After=network.target bitcoind.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin Knots health check timer - systemd: - name: bitcoin-knots-healthcheck.timer - enabled: yes - state: started - - - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_bitcoin_knots_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for new monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "Bitcoin Knots" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_bitcoin_knots_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Extract push URL from monitor setup output - set_fact: - uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" - delegate_to: localhost - become: no - when: monitor_setup.stdout is defined - - - name: Display extracted push URL - debug: - msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" - when: uptime_kuma_push_url | default('') != '' - - - name: Set push URL in systemd service environment - lineinfile: - path: /etc/systemd/system/bitcoin-knots-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" - state: present - insertafter: '^\[Service\]' - when: uptime_kuma_push_url | default('') != '' - - - name: Reload systemd daemon after push URL update - systemd: - daemon_reload: yes - when: uptime_kuma_push_url | default('') != '' - - - name: Restart health check timer to pick up new environment - systemd: - name: bitcoin-knots-healthcheck.timer - state: restarted - when: uptime_kuma_push_url | default('') != '' - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_bitcoin_knots_monitor.py - - /tmp/ansible_config.yml - - handlers: - - name: Restart bitcoind - when: uptime_kuma_enabled | default(false) - systemd: - name: bitcoind - state: restarted - + # Preserves the push URL this check has been reporting to. The role knows + # nothing about Uptime Kuma — this is just "a URL that accepts a ping". + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_bitcoin-knots/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - bitcoin_knots - name: Setup public Bitcoin P2P forwarding on the edge host hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./bitcoin_knots_vars.yml - vars: - bitcoin_tailscale_hostname: "knots-box" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - name: Expose Bitcoin P2P through a socket proxy ansible.builtin.include_role: @@ -759,134 +27,9 @@ vars: socket_proxy_name: bitcoin-p2p socket_proxy_description: "Bitcoin P2P" - socket_proxy_listen_port: "{{ bitcoin_p2p_port }}" - socket_proxy_upstream_host: "{{ bitcoin_tailscale_hostname }}" - # These four were added by hand on vipy and were NOT in this playbook; - # writing the unit without them would have dropped FreeBind, which lets - # the socket bind before the address is up. + socket_proxy_listen_port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}" + socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" socket_proxy_documentation: "https://github.com/bitcoin/bitcoin" socket_proxy_free_bind: true socket_proxy_timeout_stop_sec: 5 socket_proxy_log_to_journal: true - - - name: Display public endpoint - when: uptime_kuma_enabled | default(false) - debug: - msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public P2P - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_bitcoin_p2p_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_bitcoin_p2p_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for TCP monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_bitcoin_p2p_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ bitcoin_p2p_port }} - monitor_name: "Bitcoin Knots P2P Public" - mode: '0644' - - - name: Run Uptime Kuma TCP monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up TCP monitor temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_bitcoin_p2p_tcp_monitor.py - - /tmp/ansible_bitcoin_p2p_config.yml - diff --git a/ansible/services/datum-gateway/datum_gateway_vars.yml b/ansible/services/datum-gateway/datum_gateway_vars.yml deleted file mode 100644 index 48bb5b7..0000000 --- a/ansible/services/datum-gateway/datum_gateway_vars.yml +++ /dev/null @@ -1,43 +0,0 @@ -# DATUM Gateway Configuration Variables -# https://github.com/OCEAN-xyz/datum_gateway - -# Version - pin to a specific tag -datum_gateway_version: "v0.4.1beta" - -# Directories -datum_gateway_dir: /opt/datum-gateway -datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" -datum_gateway_config_dir: /etc/datum-gateway -datum_gateway_log_dir: /var/log/datum-gateway - -# Binary -datum_gateway_bin_path: /usr/local/bin/datum_gateway - -# Ports -datum_gateway_stratum_port: 23334 # Miners connect here via Stratum v1 -datum_gateway_api_port: 7152 # Web dashboard / API - -# Stratum settings -datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) - -# Service user -datum_gateway_user: datum -datum_gateway_group: datum - -# Build options -datum_gateway_build_jobs: 4 - -# Bitcoin node connection -# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. -# datum_bitcoin_rpc_url should include http:// and port. -datum_bitcoin_rpc_url: "http://127.0.0.1:8332" -# Note: bitcoin_rpc_user and bitcoin_rpc_password come from infra_secrets.yml - -# Mining config -datum_coinbase_tag_primary: "DATUM" -datum_coinbase_tag_secondary: "BY ORDER OF BIP110" - -datum_pool_pass_workers: true -datum_pool_pass_full_users: true -datum_pooled_mining_only: true - diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 0c53e9d..2e88a27 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -1,667 +1,49 @@ --- -# DATUM Gateway Deployment Playbook +# DATUM Gateway: solo/pooled mining gateway, built from source on knots-box. # -# Deploys DATUM Gateway (https://github.com/OCEAN-xyz/datum_gateway) on the -# Bitcoin Knots host so it has direct localhost RPC access to bitcoind. -# -# What this does: -# 1. Installs build deps and compiles datum_gateway from source -# 2. Creates a dedicated system user and config/log directories -# 3. Writes /etc/datum-gateway/config.json from vars/secrets -# 4. Patches bitcoin.conf with the required blockmaxsize/blocknotify lines -# 5. Creates and enables a systemd service -# 6. Creates a push-monitor health check script + systemd timer -# 7. Registers a push monitor in Uptime Kuma -# -# Separate play: adds a Caddy reverse proxy on vipy for the dashboard. -# -# Stratum port (23334) is bound on knots_box_local. Expose it to miners via -# a firewall rule, Tailscale, or a socket proxy on vipy — not handled here. -# -# Required secrets in infra_secrets.yml: -# datum_mining_address - Bitcoin address for block rewards -# datum_gateway_admin_password - Password for the /api admin endpoint -# bitcoin_rpc_user - Shared with the bitcoin-knots deployment -# bitcoin_rpc_password - Shared with the bitcoin-knots deployment - -- name: Deploy DATUM Gateway on knots_box_local +# This is HALF OF A SYSTEM. The Bitcoin Knots node on the same host feeds it via +# `blocknotify=killall -USR1 datum_gateway` in bitcoin.conf (see +# roles/bitcoin_knots/README.md). Changing either config means thinking about +# both. Interrupting Stratum costs mining shares, so check for connected miners +# before restarting: +# ss -tn state established '( sport = :23334 )' +- name: Deploy DATUM Gateway on the bitcoin host hosts: bitcoin become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./datum_gateway_vars.yml vars: - datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" - datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + # Preserves the push URL this check reports to. The role knows nothing about + # Uptime Kuma — this is just "a URL that accepts a ping". + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_datum-gateway/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - datum_gateway - tasks: - # =========================================== - # Build Dependencies - # =========================================== - - name: Install DATUM Gateway build dependencies - apt: - name: - - cmake - - build-essential - - git - - libjansson-dev - - libmicrohttpd-dev - - libsodium-dev - - libcurl4-openssl-dev - # Runtime-only (netcat for health check) - - netcat-openbsd - state: present - update_cache: yes - - # =========================================== - # System User and Directories - # =========================================== - - name: Create datum system user - user: - name: "{{ datum_gateway_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ datum_gateway_dir }}" - create_home: no - comment: "DATUM Gateway" - - - name: Create DATUM Gateway directories - file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ datum_gateway_group }}" - mode: "{{ item.mode }}" - loop: - - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - - # =========================================== - # Build from Source - # =========================================== - - name: Clone DATUM Gateway repository at {{ datum_gateway_version }} - git: - repo: https://github.com/OCEAN-xyz/datum_gateway.git - dest: "{{ datum_gateway_source_dir }}" - version: "{{ datum_gateway_version }}" - force: yes - register: git_clone - - - name: Configure cmake build - command: cmake . -DCMAKE_BUILD_TYPE=Release - args: - chdir: "{{ datum_gateway_source_dir }}" - - - name: Compile datum_gateway - command: make -j{{ datum_gateway_build_jobs }} - args: - chdir: "{{ datum_gateway_source_dir }}" - - - name: Install datum_gateway binary - copy: - src: "{{ datum_gateway_source_dir }}/datum_gateway" - dest: "{{ datum_gateway_bin_path }}" - remote_src: yes - owner: root - group: root - mode: "0755" - notify: Restart datum-gateway - - # =========================================== - # Configuration - # =========================================== - - name: Write DATUM Gateway config.json - copy: - dest: "{{ datum_gateway_config_dir }}/config.json" - content: | - { - "bitcoind": { - "rpcuser": "{{ bitcoin_rpc_user }}", - "rpcpassword": "{{ bitcoin_rpc_password }}", - "rpcurl": "{{ datum_bitcoin_rpc_url }}", - "notify_fallback": true - }, - "stratum": { - "listen_port": {{ datum_gateway_stratum_port }}, - "vardiff_min": {{ datum_vardiff_min }} - }, - "mining": { - "pool_address": "{{ datum_mining_address }}", - "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", - "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" - }, - "api": { - "admin_password": "{{ datum_gateway_admin_password }}", - "listen_port": {{ datum_gateway_api_port }}, - "modify_conf": false - }, - "logger": { - "log_to_console": true, - "log_to_file": true, - "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", - "log_rotate_daily": true, - "log_level_console": 2, - "log_level_file": 1 - }, - "datum": { - "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, - "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, - "pooled_mining_only": {{ datum_pooled_mining_only | lower }} - } - } - owner: "{{ datum_gateway_user }}" - group: "{{ datum_gateway_group }}" - mode: "0640" - notify: Restart datum-gateway - - # =========================================== - # Systemd Service - # =========================================== - - name: Create datum-gateway systemd service - copy: - dest: /etc/systemd/system/datum-gateway.service - content: | - [Unit] - Description=DATUM Gateway - Bitcoin Mining Gateway - Documentation=https://github.com/OCEAN-xyz/datum_gateway - After=network.target bitcoind.service - Wants=bitcoind.service - - [Service] - User={{ datum_gateway_user }} - Group={{ datum_gateway_group }} - Type=simple - ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json - Restart=on-failure - RestartSec=10 - StandardOutput=journal - StandardError=journal - - # Prevent config from being read by other users - ReadWritePaths={{ datum_gateway_log_dir }} - ReadOnlyPaths={{ datum_gateway_config_dir }} - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - notify: Restart datum-gateway - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start datum-gateway - systemd: - name: datum-gateway - enabled: yes - state: started - - # =========================================== - # Health Check Script + Systemd Timer - # =========================================== - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create DATUM Gateway health check script - when: uptime_kuma_enabled | default(false) - copy: - dest: /usr/local/bin/datum-gateway-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - STRATUM_PORT={{ datum_gateway_stratum_port }} - - check_datum() { - # Service must be active and stratum port must be listening - systemctl is-active --quiet datum-gateway && \ - nc -z 127.0.0.1 "${STRATUM_PORT}" - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_datum; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "DATUM Gateway not responding" - exit 1 - fi - owner: root - group: root - mode: "0755" - - - name: Create datum-gateway health check systemd service - copy: - dest: /etc/systemd/system/datum-gateway-healthcheck.service - content: | - [Unit] - Description=DATUM Gateway Health Check - After=network.target datum-gateway.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - - - name: Create datum-gateway health check systemd timer - copy: - dest: /etc/systemd/system/datum-gateway-healthcheck.timer - content: | - [Unit] - Description=DATUM Gateway Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - - - name: Enable and start datum-gateway health check timer - when: uptime_kuma_enabled | default(false) - systemd: - name: datum-gateway-healthcheck.timer - enabled: yes - state: started - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for DATUM Gateway - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_datum_gateway_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - - try: - import socketio.exceptions - except ImportError: - pass - - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_datum_gateway_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - push_url = None - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - else: - print(f"Creating push monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - # socketio timeout: add_monitor may have succeeded server-side - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - - api.disconnect() - - if push_url: - print(f"PUSH_URL={push_url}") - with open('/tmp/datum_gateway_push_url.txt', 'w') as f: - f.write(push_url) - - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for push monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_datum_gateway_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "DATUM Gateway" - mode: "0644" - - - name: Run Uptime Kuma push monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_datum_gateway_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URL from file - when: uptime_kuma_enabled | default(false) - slurp: - src: /tmp/datum_gateway_push_url.txt - delegate_to: localhost - become: no - register: push_url_file - ignore_errors: yes - - - name: Parse push URL - set_fact: - datum_push_url: "{{ push_url_file.content | b64decode | trim }}" - when: push_url_file.content is defined - - - name: Update health check service with push URL - lineinfile: - path: /etc/systemd/system/datum-gateway-healthcheck.service - regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" - line: "Environment=UPTIME_KUMA_PUSH_URL={{ datum_push_url }}" - when: datum_push_url is defined - notify: Restart datum-gateway health check timer - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_datum_gateway_monitor.py - - /tmp/ansible_datum_gateway_config.yml - - /tmp/datum_gateway_push_url.txt - - handlers: - - name: Restart datum-gateway - systemd: - name: datum-gateway - state: restarted - daemon_reload: yes - - - name: Restart datum-gateway health check timer - when: uptime_kuma_enabled | default(false) - systemd: - name: datum-gateway-healthcheck.timer - state: restarted - daemon_reload: yes - - -# =========================================== -# Caddy Reverse Proxy for DATUM Dashboard (on vipy) -# =========================================== -- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on the edge host +- name: Configure Caddy reverse proxy for the DATUM Gateway dashboard on the edge host hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./datum_gateway_vars.yml - vars: - datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" - datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - name: Publish the DATUM Gateway dashboard through Caddy ansible.builtin.include_role: name: caddy_site vars: caddy_site_name: datum-gateway - caddy_site_domain: "{{ datum_gateway_domain }}" - caddy_site_upstream: "knots-box:{{ datum_gateway_api_port }}" + caddy_site_domain: "{{ subdomains.datum_gateway }}.{{ root_domain }}" + caddy_site_upstream: "{{ hostvars['knots_box_local'].ansible_host }}:{{ hostvars['knots_box_local'].datum_gateway_api_port }}" caddy_site_resolvers: "100.100.100.100" caddy_site_basic_auth: - user: "{{ datum_dashboard_username }}" hash: "{{ datum_dashboard_password_hash }}" - # The role validates the site fragment on its own. This re-validates the - # whole assembled Caddyfile, which is the only thing that catches a - # conflict between this site and another. Kept from the hand-rolled - # version; the other nine services never had it. + # The role validates its own site fragment; this re-validates the whole + # assembled Caddyfile, which is the only thing that catches a conflict + # between two sites. - name: Validate the assembled Caddyfile ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile changed_when: false - - name: Display DATUM Gateway dashboard URL - when: uptime_kuma_enabled | default(false) - debug: - msg: "DATUM Gateway dashboard: https://{{ datum_gateway_domain }}" - - # =========================================== - # Uptime Kuma HTTP Monitor for Public Dashboard - # =========================================== - - name: Create Uptime Kuma HTTP monitor setup script for DATUM dashboard - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_datum_http_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_datum_http_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - else: - print(f"Creating HTTP monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for HTTP monitor - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_datum_http_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ datum_gateway_domain }}" - monitor_name: "DATUM Gateway Dashboard" - mode: "0644" - - - name: Run Uptime Kuma HTTP monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_datum_http_monitor.py - delegate_to: localhost - become: no - register: http_monitor_setup - changed_when: "'SUCCESS' in http_monitor_setup.stdout" - ignore_errors: yes - - - name: Display HTTP monitor setup output - debug: - msg: "{{ http_monitor_setup.stdout_lines }}" - when: http_monitor_setup.stdout is defined - - - name: Clean up HTTP monitor temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_datum_http_monitor.py - - /tmp/ansible_datum_http_config.yml - - -# =========================================== -# Stratum Port Forwarding on vipy via systemd-socket-proxyd -# Miners connect to vipy:23334; traffic is forwarded to knots-box:23334 -# over the Tailscale network, matching the Bitcoin P2P proxy pattern. -# =========================================== - name: Setup public Stratum port forwarding on the edge host hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./datum_gateway_vars.yml - vars: - datum_tailscale_hostname: "knots-box" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - name: Expose the DATUM Stratum port through a socket proxy ansible.builtin.include_role: @@ -669,134 +51,8 @@ vars: socket_proxy_name: datum-stratum socket_proxy_description: "DATUM Stratum" - socket_proxy_listen_port: "{{ datum_gateway_stratum_port }}" - socket_proxy_upstream_host: "{{ datum_tailscale_hostname }}" - # Matches the UFW comment already on vipy; the derived default would - # have said "DATUM Stratum" and rewritten the rule. + socket_proxy_listen_port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}" + socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" + # Matches the UFW comment already on the edge host; the derived default + # would say "DATUM Stratum" and rewrite the rule. socket_proxy_ufw_comment: "DATUM Gateway Stratum public access" - - - name: Display public Stratum endpoint - when: uptime_kuma_enabled | default(false) - debug: - msg: "DATUM Stratum public endpoint: {{ ansible_host }}:{{ datum_gateway_stratum_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public Stratum - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Stratum - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_datum_stratum_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_datum_stratum_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for Stratum TCP monitor - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_datum_stratum_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ datum_gateway_stratum_port }} - monitor_name: "DATUM Stratum (public)" - mode: "0644" - - - name: Run Uptime Kuma TCP monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_datum_stratum_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up Stratum TCP monitor temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_datum_stratum_tcp_monitor.py - - /tmp/ansible_datum_stratum_config.yml - diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index bdc8428..04081ff 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -1,409 +1,13 @@ +--- - name: Install Forgejo Runner on Debian 13 hosts: ci_runner become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./forgejo_runner_vars.yml vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - healthcheck_interval_seconds: 60 - healthcheck_timeout_seconds: 90 - healthcheck_retries: 1 - healthcheck_script_dir: /opt/forgejo-runner-healthcheck - healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" - healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" - healthcheck_service_name: forgejo-runner-healthcheck - - tasks: - # ── 1. Assert Docker is available ────────────────────────────────── - - name: Check if Docker is installed - command: docker --version - register: docker_check - changed_when: false - failed_when: docker_check.rc != 0 - - - name: Fail if Docker is not available - assert: - that: - - docker_check.rc == 0 - fail_msg: > - Docker is not installed or not in PATH. - Please install Docker before running this playbook. - - # ── 2. Download forgejo-runner binary ────────────────────────────── - - name: Download forgejo-runner binary - get_url: - url: "{{ forgejo_runner_url }}" - dest: "{{ forgejo_runner_bin_path }}" - mode: '0755' - - # ── 3. Create runner system user ─────────────────────────────────── - - name: Create runner system user - user: - name: "{{ forgejo_runner_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ forgejo_runner_dir }}" - create_home: no - groups: docker - append: yes - comment: 'Forgejo Runner' - - # ── 4. Create working directory ──────────────────────────────────── - - name: Create forgejo-runner working directory - file: - path: "{{ forgejo_runner_dir }}" - state: directory - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - mode: '0750' - - # ── 5. Generate default config ───────────────────────────────────── - - name: Check if config already exists - stat: - path: "{{ forgejo_runner_config_path }}" - register: config_stat - - - name: Generate default config - shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not config_stat.stat.exists - - - name: Set config file ownership - file: - path: "{{ forgejo_runner_config_path }}" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not config_stat.stat.exists - - # ── 6. Register runner ───────────────────────────────────────────── - - name: Check if runner is already registered - stat: - path: "{{ forgejo_runner_dir }}/.runner" - register: runner_stat - - - name: Register runner with Forgejo instance - command: > - {{ forgejo_runner_bin_path }} register --no-interactive - --instance {{ forgejo_instance_url }} - --token {{ forgejo_runner_registration_token }} - --name forgejo-runner-box - --labels "{{ forgejo_runner_labels }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not runner_stat.stat.exists - - - name: Set runner registration file ownership - file: - path: "{{ forgejo_runner_dir }}/.runner" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not runner_stat.stat.exists - - # ── 7. Create systemd service ────────────────────────────────────── - - name: Create forgejo-runner systemd service - copy: - dest: /etc/systemd/system/forgejo-runner.service - content: | - [Unit] - Description=Forgejo Runner - Documentation=https://forgejo.org/docs/latest/admin/actions/ - After=docker.service - Requires=docker.service - - [Service] - Type=simple - User={{ forgejo_runner_user }} - Group={{ forgejo_runner_user }} - WorkingDirectory={{ forgejo_runner_dir }} - ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} - Restart=on-failure - RestartSec=10 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - # ── 8. Reload systemd, enable and start ──────────────────────────── - - name: Reload systemd - systemd: - daemon_reload: yes - - - name: Enable and start forgejo-runner service - systemd: - name: forgejo-runner - enabled: yes - state: started - - # ── 9. Verify runner is active ───────────────────────────────────── - - name: Verify forgejo-runner is active - command: systemctl is-active forgejo-runner - register: runner_active - changed_when: false - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Assert runner is running - when: uptime_kuma_enabled | default(false) - assert: - that: - - runner_active.stdout == "active" - fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" - - # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── - - name: Create Uptime Kuma push monitor setup script - when: uptime_kuma_enabled | default(false) - copy: - dest: /tmp/setup_forgejo_runner_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma push monitor setup - when: uptime_kuma_enabled | default(false) - command: > - {{ ansible_playbook_python }} - /tmp/setup_forgejo_runner_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "services" - "forgejo-runner-healthcheck" - "Forgejo Runner healthcheck - ping every {{ healthcheck_interval_seconds }}s" - "{{ healthcheck_timeout_seconds }}" - "{{ healthcheck_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - when: uptime_kuma_enabled | default(false) - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL - when: uptime_kuma_enabled | default(false) - set_fact: - uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - - name: Create healthcheck script directory - file: - path: "{{ healthcheck_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create forgejo-runner healthcheck script - when: uptime_kuma_enabled | default(false) - copy: - dest: "{{ healthcheck_script_path }}" - content: | - #!/bin/bash - - # Forgejo Runner Healthcheck Script - # Checks if forgejo-runner is active and pings Uptime Kuma on success - - LOG_FILE="{{ healthcheck_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_push_url }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - main() { - if systemctl is-active --quiet forgejo-runner; then - log_message "forgejo-runner is active, sending ping" - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=forgejo-runner%20is%20active" 2>&1) - http_code=$(echo "$response" | tail -n1) - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Ping sent successfully (HTTP $http_code)" - else - log_message "ERROR: Failed to send ping (HTTP $http_code)" - exit 1 - fi - else - log_message "ERROR: forgejo-runner is not active" - exit 1 - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create healthcheck systemd service - copy: - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" - content: | - [Unit] - Description=Forgejo Runner Healthcheck - After=network.target - - [Service] - Type=oneshot - ExecStart={{ healthcheck_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create healthcheck systemd timer - copy: - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" - content: | - [Unit] - Description=Run Forgejo Runner Healthcheck every minute - Requires={{ healthcheck_service_name }}.service - - [Timer] - OnBootSec=30sec - OnUnitActiveSec={{ healthcheck_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd for healthcheck units - systemd: - daemon_reload: yes - - - name: Enable and start healthcheck timer - systemd: - name: "{{ healthcheck_service_name }}.timer" - enabled: yes - state: started - - - name: Test healthcheck script - command: "{{ healthcheck_script_path }}" - register: healthcheck_test - changed_when: false - - - name: Verify healthcheck script works - assert: - that: - - healthcheck_test.rc == 0 - fail_msg: "Healthcheck script failed to execute properly" - - - name: Display deployment summary - debug: - msg: | - Forgejo Runner deployed successfully! - - Runner Name: forgejo-runner-box - Instance: {{ forgejo_instance_url }} - Working Directory: {{ forgejo_runner_dir }} - Service: forgejo-runner.service ({{ runner_active.stdout }}) - - Healthcheck Monitor: forgejo-runner-healthcheck - Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s - Timeout: {{ healthcheck_timeout_seconds }}s - - - name: Clean up temporary monitor setup script - when: uptime_kuma_enabled | default(false) - file: - path: /tmp/setup_forgejo_runner_monitor.py - state: absent - delegate_to: localhost - become: no + # Preserves the push URL this host has been reporting to all along, so the + # move to a role changes no behaviour. The role itself knows nothing about + # Uptime Kuma — this is just "a URL that accepts a ping", and whatever + # replaces it sets the same variable. + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_forgejo-runner/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - forgejo_runner diff --git a/ansible/services/forgejo-runner/forgejo_runner_vars.yml b/ansible/services/forgejo-runner/forgejo_runner_vars.yml deleted file mode 100644 index e618fca..0000000 --- a/ansible/services/forgejo-runner/forgejo_runner_vars.yml +++ /dev/null @@ -1,9 +0,0 @@ -forgejo_runner_version: "6.3.1" -forgejo_runner_arch: "linux-amd64" -forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" -forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" -forgejo_runner_user: "runner" -forgejo_runner_dir: "/opt/forgejo-runner" -forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" -forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" -forgejo_instance_url: "https://forgejo.contrapeso.xyz" diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index db78e95..f908586 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -2,14 +2,10 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./forgejo_vars.yml vars: forgejo_subdomain: "{{ subdomains.forgejo }}" forgejo_domain: "{{ forgejo_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure required packages are installed @@ -94,123 +90,3 @@ caddy_site_name: forgejo caddy_site_domain: "{{ forgejo_domain }}" caddy_site_upstream: "localhost:{{ forgejo_port }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Uptime Kuma monitor setup script for Forgejo - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_forgejo_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ forgejo_domain }}/api/healthz" - monitor_name: "Forgejo" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_forgejo_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_forgejo_monitor.py - - /tmp/ansible_config.yml - diff --git a/ansible/services/forgejo/forgejo_vars.yml b/ansible/services/forgejo/forgejo_vars.yml index 2a24133..7bba0ed 100644 --- a/ansible/services/forgejo/forgejo_vars.yml +++ b/ansible/services/forgejo/forgejo_vars.yml @@ -9,7 +9,7 @@ forgejo_url: "https://codeberg.org/forgejo/forgejo/releases/download/v{{ forgejo forgejo_bin_path: "/usr/local/bin/forgejo" forgejo_user: "git" -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access remote_host_name: "{{ groups['edge'] | first }}" @@ -18,6 +18,3 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/forgejo-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/forgejo_backup.sh" diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml index d6aafef..5329e2a 100644 --- a/ansible/services/forgejo/setup_backup_forgejo.yml +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -9,7 +9,6 @@ hosts: edge become: yes vars_files: - - ../../group_vars/all/main.yml - ./forgejo_vars.yml tasks: @@ -23,3 +22,7 @@ backup_source_stop_service: forgejo backup_source_retention_days: 2 backup_source_on_calendar: "*-*-* 02:30:00" + # Reported to Gatus as backup-dump_forgejo. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_forgejo/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml deleted file mode 100644 index c3ff8d6..0000000 --- a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml +++ /dev/null @@ -1,122 +0,0 @@ ---- -- name: Configure local backup for Forgejo from remote - hosts: control - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./forgejo_vars.yml - vars: - remote_data_path: "{{ forgejo_data_dir }}" - remote_config_path: "{{ forgejo_config_dir }}" - forgejo_service_name: "forgejo" - gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" - gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" - - tasks: - - name: Debug Forgejo backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "remote_config_path='{{ remote_config_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - "gpg_recipient={{ gpg_recipient }}" - - "gpg_key_id={{ gpg_key_id }}" - - - name: Ensure local backup directory exists - ansible.builtin.file: - path: "{{ local_backup_dir }}" - state: directory - mode: "0755" - - - name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: "0755" - - - name: Create Forgejo backup script - ansible.builtin.copy: - dest: "{{ backup_script_path }}" - mode: "0750" - content: | - #!/bin/bash - set -euo pipefail - - if [ -z "{{ gpg_recipient }}" ]; then - echo "GPG recipient is not configured. Aborting." - exit 1 - fi - - TIMESTAMP=$(date +'%Y-%m-%d') - ENCRYPTED_BACKUP="{{ local_backup_dir }}/forgejo-backup-$TIMESTAMP.tar.gz.gpg" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - echo "Stopping Forgejo service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop {{ forgejo_service_name }}" - - echo "Creating encrypted backup archive..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo tar -czf - {{ remote_data_path }} {{ remote_config_path }}" | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" - - echo "Starting Forgejo service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start {{ forgejo_service_name }}" - - # Rotate old backups (keep 3 days) - # Calculate cutoff date (3 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '3 days ago' +'%Y-%m-%d') - for backup_file in "{{ local_backup_dir }}"/forgejo-backup-*.tar.gz.gpg; do - if [ -f "$backup_file" ]; then - # Extract date from filename: forgejo-backup-YYYY-MM-DD.tar.gz.gpg - file_date=$(basename "$backup_file" | sed -n 's/forgejo-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$backup_file" - fi - fi - done - - echo "Backup completed successfully" - - - name: Ensure cronjob for Forgejo backup exists - ansible.builtin.cron: - name: "Forgejo backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run Forgejo backup script to create initial backup - ansible.builtin.command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check if backup file exists - stat: - path: "{{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg" - register: backup_file_stat - - - name: Verify backup file exists - assert: - that: - - backup_file_stat.stat.exists - - backup_file_stat.stat.isreg - fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" - success_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists" - - - name: Verify backup file is not empty - assert: - that: - - backup_file_stat.stat.size > 0 - fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" - success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 01e9f17..1b7b963 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -1,556 +1,21 @@ +--- +# Fulcrum: Electrum server indexing the Bitcoin Knots node. +# The index takes days to rebuild, so nothing here touches its data directory +# beyond asserting that it exists. - name: Deploy Fulcrum Electrum Server hosts: electrum become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./fulcrum_vars.yml vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Calculate 75% of system RAM for db_mem - set_fact: - fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" - changed_when: false - - - name: Display calculated db_mem value - debug: - msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" - - - name: Display Fulcrum version to install - debug: - msg: "Installing Fulcrum version {{ fulcrum_version }}" - - - name: Install required packages - apt: - name: - - curl - - wget - - openssl - state: present - update_cache: yes - - - name: Create fulcrum group - group: - name: "{{ fulcrum_group }}" - system: yes - state: present - - - name: Create fulcrum user - user: - name: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - system: yes - shell: /usr/sbin/nologin - home: /home/{{ fulcrum_user }} - create_home: yes - state: present - - - name: Create Fulcrum database directory (heavy data on special mount) - file: - path: "{{ fulcrum_db_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - - - name: Create Fulcrum config directory - file: - path: "{{ fulcrum_config_dir }}" - state: directory - owner: root - group: "{{ fulcrum_group }}" - mode: '0755' - - - name: Create Fulcrum lib directory (for banner and other data files) - file: - path: "{{ fulcrum_lib_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - - # =========================================== - # SSL Certificate Generation - # =========================================== - - name: Check if SSL certificate already exists - stat: - path: "{{ fulcrum_ssl_cert_path }}" - register: fulcrum_ssl_cert_exists - when: fulcrum_ssl_enabled | default(false) - - - name: Generate self-signed SSL certificate for Fulcrum - command: > - openssl req -x509 -newkey rsa:4096 - -keyout {{ fulcrum_ssl_key_path }} - -out {{ fulcrum_ssl_cert_path }} - -sha256 -days {{ fulcrum_ssl_cert_days }} - -nodes - -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" - args: - creates: "{{ fulcrum_ssl_cert_path }}" - when: fulcrum_ssl_enabled | default(false) - notify: Restart fulcrum - - - name: Set SSL certificate permissions - file: - path: "{{ fulcrum_ssl_cert_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) - - - name: Set SSL key permissions - file: - path: "{{ fulcrum_ssl_key_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0600' - when: fulcrum_ssl_enabled | default(false) - - - name: Check if Fulcrum binary already exists - stat: - path: "{{ fulcrum_binary_path }}" - register: fulcrum_binary_exists - changed_when: false - - - name: Download Fulcrum binary tarball - get_url: - url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - mode: '0644' - when: not fulcrum_binary_exists.stat.exists - - - name: Extract Fulcrum binary - unarchive: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp" - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - - - name: Install Fulcrum binary - copy: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" - dest: "{{ fulcrum_binary_path }}" - owner: root - group: root - mode: '0755' - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - - - name: Verify Fulcrum binary installation - command: "{{ fulcrum_binary_path }} --version" - register: fulcrum_version_check - changed_when: false - - - name: Display Fulcrum version - debug: - msg: "{{ fulcrum_version_check.stdout_lines }}" - - - name: Create Fulcrum banner file - copy: - dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" - content: | - counterinfra - - PER ASPERA AD ASTRA - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - - - name: Create Fulcrum configuration file - copy: - dest: "{{ fulcrum_config_dir }}/fulcrum.conf" - content: | - # Fulcrum Configuration - # Generated by Ansible - - # Bitcoin Core/Knots RPC settings - bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} - rpcuser = {{ bitcoin_rpc_user }} - rpcpassword = {{ bitcoin_rpc_password }} - - # Fulcrum server general settings - datadir = {{ fulcrum_db_dir }} - tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} - peering = {{ 'true' if fulcrum_peering else 'false' }} - zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} - - # SSL/TLS Configuration - {% if fulcrum_ssl_enabled | default(false) %} - ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} - cert = {{ fulcrum_ssl_cert_path }} - key = {{ fulcrum_ssl_key_path }} - {% endif %} - - # Anonymize client IP addresses and TxIDs in logs - anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} - - # Max RocksDB Memory in MiB - db_mem = {{ fulcrum_db_mem_mb }}.0 - - # Banner - banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0640' - notify: Restart fulcrum - - - name: Create systemd service file for Fulcrum - copy: - dest: /etc/systemd/system/fulcrum.service - content: | - # MiniBolt: systemd unit for Fulcrum - # /etc/systemd/system/fulcrum.service - - [Unit] - Description=Fulcrum - After=network.target - - StartLimitBurst=2 - StartLimitIntervalSec=20 - - [Service] - ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf - - User={{ fulcrum_user }} - Group={{ fulcrum_group }} - - # Process management - #################### - Type=simple - KillSignal=SIGINT - TimeoutStopSec=300 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum service - systemd: - name: fulcrum - enabled: yes - state: started - - - name: Wait for Fulcrum to start - wait_for: - port: "{{ fulcrum_tcp_port }}" - host: "{{ fulcrum_tcp_bind }}" - delay: 5 - timeout: 30 - ignore_errors: yes - - - name: Check Fulcrum service status - systemd: - name: fulcrum - register: fulcrum_service_status - changed_when: false - - - name: Display Fulcrum service status - debug: - msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Fulcrum health check and push script - when: uptime_kuma_enabled | default(false) - copy: - dest: /usr/local/bin/fulcrum-healthcheck-push.sh - content: | - #!/bin/bash - # - # Fulcrum Health Check and Push to Uptime Kuma - # Checks if Fulcrum TCP port is responding and pushes status to Uptime Kuma - # - - FULCRUM_HOST="{{ fulcrum_tcp_bind }}" - FULCRUM_PORT={{ fulcrum_tcp_port }} - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - # Check if Fulcrum TCP port is responding - check_fulcrum() { - # Try to connect to TCP port - timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null - return $? - } - - # Push status to Uptime Kuma - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - if ! curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then - echo "ERROR: Failed to push to Uptime Kuma" - return 1 - fi - } - - # Main health check - if check_fulcrum; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Fulcrum TCP port not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create systemd timer for Fulcrum health check - copy: - dest: /etc/systemd/system/fulcrum-healthcheck.timer - content: | - [Unit] - Description=Fulcrum Health Check Timer - Requires=fulcrum.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for Fulcrum health check - when: uptime_kuma_enabled | default(false) - copy: - dest: /etc/systemd/system/fulcrum-healthcheck.service - content: | - [Unit] - Description=Fulcrum Health Check and Push to Uptime Kuma - After=network.target fulcrum.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum health check timer - systemd: - name: fulcrum-healthcheck.timer - enabled: yes - state: started - - - name: Create Uptime Kuma push monitor setup script for Fulcrum - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_fulcrum_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for new monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "Fulcrum" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_fulcrum_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Extract push URL from monitor setup output - set_fact: - uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" - delegate_to: localhost - become: no - when: monitor_setup.stdout is defined - - - name: Display extracted push URL - debug: - msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" - when: uptime_kuma_push_url | default('') != '' - - - name: Set push URL in systemd service environment - lineinfile: - path: /etc/systemd/system/fulcrum-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" - state: present - insertafter: '^\[Service\]' - when: uptime_kuma_push_url | default('') != '' - - - name: Reload systemd daemon after push URL update - systemd: - daemon_reload: yes - when: uptime_kuma_push_url | default('') != '' - - - name: Restart health check timer to pick up new environment - systemd: - name: fulcrum-healthcheck.timer - state: restarted - when: uptime_kuma_push_url | default('') != '' - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_fulcrum_monitor.py - - /tmp/ansible_config.yml - - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz - - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux - - handlers: - - name: Restart fulcrum - when: uptime_kuma_enabled | default(false) - systemd: - name: fulcrum - state: restarted - + # Preserves the push URL this check has been configured with. The role knows + # nothing about Uptime Kuma — this is just "a URL that accepts a ping". + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_fulcrum/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - fulcrum - name: Setup public Fulcrum SSL forwarding on the edge host hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./fulcrum_vars.yml - vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - name: Expose Fulcrum SSL through a socket proxy ansible.builtin.include_role: @@ -558,128 +23,5 @@ vars: socket_proxy_name: fulcrum-ssl socket_proxy_description: "Fulcrum SSL" - socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" - socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" - - - name: Display public endpoint - when: uptime_kuma_enabled | default(false) - debug: - msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public SSL Port - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_fulcrum_ssl_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_fulcrum_ssl_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for TCP monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_fulcrum_ssl_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ fulcrum_ssl_port }} - monitor_name: "Fulcrum SSL Public" - mode: '0644' - - - name: Run Uptime Kuma TCP monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up TCP monitor temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_fulcrum_ssl_tcp_monitor.py - - /tmp/ansible_fulcrum_ssl_config.yml - - + socket_proxy_listen_port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ hostvars['fulcrum_box_local'].ansible_host }}" diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml deleted file mode 100644 index 8486b14..0000000 --- a/ansible/services/fulcrum/fulcrum_vars.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Fulcrum Configuration Variables - -# Version - Pinned to specific release -fulcrum_version: "2.1.0" # Fulcrum version to install - -# Directories -fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) -fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) -fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk -fulcrum_binary_path: /usr/local/bin/Fulcrum - -# Network - Bitcoin RPC connection -# Bitcoin Knots is on a different host (knots_box_local) -# Using RPC user/password authentication (credentials from infra_secrets.yml) -bitcoin_rpc_host: "192.168.1.140" # Bitcoin Knots RPC host (IP of knots_box_local) -bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml - -# Network - Fulcrum server -fulcrum_tcp_port: 50001 -fulcrum_ssl_port: 50002 -# Binding address for Fulcrum TCP/SSL server: -# - "127.0.0.1" = localhost only (use when Caddy is on the same box) -# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) -# - Specific IP = bind to specific network interface -fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) -fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port -# If Caddy is on a different box, set this to the IP address that Caddy will use to connect - -# SSL/TLS Configuration -fulcrum_ssl_enabled: true -fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" -fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" -fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert - -# Port forwarding configuration (for public access via VPS) -fulcrum_tailscale_hostname: "fulcrum-box" - -# Performance -# db_mem will be calculated as 75% of available RAM automatically in playbook -fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache - -# Configuration options -fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs -fulcrum_peering: false # Disable peering with other Fulcrum servers -fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications - -# Service user -fulcrum_user: fulcrum -fulcrum_group: fulcrum - diff --git a/ansible/services/gatus/deploy_gatus_playbook.yml b/ansible/services/gatus/deploy_gatus_playbook.yml new file mode 100644 index 0000000..d6a5ac1 --- /dev/null +++ b/ansible/services/gatus/deploy_gatus_playbook.yml @@ -0,0 +1,106 @@ +--- +# Gatus: health checks, status page and alerting for the whole estate. +# +# Built from source and run under systemd - upstream publishes no binaries, and +# their Dockerfile shows the runtime needs nothing but the static binary and a +# CA bundle. See roles/gatus/README.md. +- name: Deploy Gatus on the observability host + hosts: observability + become: yes + vars: + gatus_alerting: + signal: + # NOTE: the key is `api-url`, not `url` as upstream's own README table + # says - see alerting/provider/signal/signal.go. Gatus appends /v2/send + # itself if the suffix is missing. + # + # Reached by service name over the shared docker network. Gatus runs in + # a container, so 127.0.0.1 here would be the Gatus container, not the + # host - and the Signal API deliberately publishes no ports because it + # has no authentication. + api-url: "http://signal-api:8080" + number: "{{ signal_number }}" + recipients: "{{ signal_recipients }}" + default-alert: + # Overridden per group by the registration playbooks; these are the + # values that apply if a caller sets nothing. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + # An ongoing outage should not become an ongoing phone buzz. + minimum-reminder-interval: 6h + + roles: + - gatus + +# The dashboard is bound to loopback; Caddy publishes it. +# +# Auth is done HERE, at the edge, and not with Gatus's own `security.basic`. +# Gatus's security middleware protects exactly four routes (api/api.go): +# +# /api/v1/endpoints/statuses +# /api/v1/endpoints/:key/statuses +# /api/v1/suites/statuses +# /api/v1/suites/:key/statuses +# +# Everything else is registered on the UNPROTECTED router, including +# /api/v1/config, every badge, and - the part that matters - +# /api/v1/endpoints/:key/uptimes/:duration and .../response-times/:duration/history, +# which return real per-endpoint data to anyone who can guess a key. Keys are +# just "_". So Gatus's own auth makes the dashboard render empty +# while leaving the data readable, which is worse than it looks. +# +# The one route that must NOT sit behind basic auth is the external-endpoint +# push API. It authenticates with `Authorization: Bearer `, and basic +# auth wants `Authorization: Basic <...>` - same header, two schemes, and the +# push clients lose. It is not actually unauthenticated: the handler 401s on a +# missing prefix, an empty token, or a token that does not match that endpoint's +# own. Upstream's comment on the route says exactly that. +- name: Publish the Gatus status page through Caddy + hosts: observability + become: yes + tasks: + - name: Require the dashboard credentials to be set + ansible.builtin.assert: + that: + - gatus_dashboard_username is defined + - gatus_dashboard_username | length > 0 + - gatus_dashboard_password_hash is defined + - gatus_dashboard_password_hash.startswith('$2') + fail_msg: >- + gatus_dashboard_username and gatus_dashboard_password_hash must be in + the vault. Generate the hash on the observability host, which runs + Caddy natively, so the bcrypt cost and format match what verifies it: + caddy hash-password --plaintext 'your-password' + then: ansible-vault edit group_vars/all/vault.yml + + - name: Configure the Caddy vhost for Gatus + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: gatus + caddy_site_domain: "{{ subdomains.gatus }}.{{ root_domain }}" + # caddy_site_body rather than caddy_site_upstream + caddy_site_basic_auth, + # because that pair applies auth to the whole site with no way to carve + # out the push path. `handle` blocks are mutually exclusive and first + # match wins, so the push API gets a route of its own. + caddy_site_body: | + @push { + path /api/v1/endpoints/*/external + method POST + } + + # Push API: Bearer-authenticated by Gatus itself. No basic auth here, + # or the Authorization header collides. + handle @push { + reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} + } + + # Everything else: the dashboard, the config endpoint, the badges and + # the uptime/response-time history. + handle { + basic_auth { + {{ gatus_dashboard_username }} {{ gatus_dashboard_password_hash }} + } + reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} + } diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 527e4c2..e181f24 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -2,16 +2,11 @@ hosts: vpn_control become: no vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./headscale_vars.yml vars: headscale_subdomain: "{{ subdomains.headscale }}" headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" headscale_base_domain: "tailnet.{{ root_domain }}" - headscale_namespace: "{{ service_settings.headscale.namespace }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Install required packages @@ -258,125 +253,6 @@ # All API operations require a valid Bearer token in the Authorization header reverse_proxy * http://localhost:{{ headscale_port }} - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Uptime Kuma monitor setup script for Headscale - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_headscale_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ headscale_domain }}/health" - monitor_name: "Headscale" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_headscale_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_headscale_monitor.py - - /tmp/ansible_config.yml - handlers: - name: Restart headscale become: yes diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index 05fc040..c3fb948 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -1,5 +1,5 @@ # Headscale service configuration -# (subdomain and caddy_sites_dir now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) headscale_port: 8080 headscale_grpc_port: 50443 @@ -10,7 +10,7 @@ headscale_version: "0.26.1" # Data directory headscale_data_dir: /var/lib/headscale -# Namespace now configured in services_config.yml under service_settings.headscale.namespace +# Namespace is headscale_namespace in group_vars/all/main.yml # Remote access remote_host_name: "{{ groups['vpn_control'] | first }}" @@ -19,9 +19,5 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/headscale-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/headscale_backup.sh" - # Headplane (headscale admin UI), proxied at /admin* behind Caddy basic auth headplane_port: 3000 diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml index 9c72d1d..15ba5b7 100644 --- a/ansible/services/headscale/setup_backup_headscale.yml +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -3,7 +3,6 @@ hosts: vpn_control become: yes vars_files: - - ../../group_vars/all/main.yml - ./headscale_vars.yml tasks: @@ -19,3 +18,7 @@ backup_source_stop_service: headscale backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_headscale. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_headscale/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml deleted file mode 100644 index acc82ce..0000000 --- a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml +++ /dev/null @@ -1,75 +0,0 @@ -- name: Configure local backup for Headscale from remote - hosts: control - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./headscale_vars.yml - vars: - remote_data_path: "{{ headscale_data_dir }}" - remote_config_path: "/etc/headscale" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "remote_config_path='{{ remote_config_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - # Stop headscale service for consistent backup - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop headscale" - - # Backup data directory - rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/data/" - - # Backup config directory - rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_config_path }}/ "$BACKUP_DIR/config/" - - # Start headscale service again - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start headscale" - - # Rotate old backups (keep 14 days) - find "{{ local_backup_dir }}" -maxdepth 1 -type d -name '20*' -mtime +13 -exec rm -rf {} \; - - - name: Ensure cronjob for backup exists - cron: - name: "Headscale backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index 65bdfbf..7b89f33 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -2,14 +2,10 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./lnbits_vars.yml vars: lnbits_subdomain: "{{ subdomains.lnbits }}" lnbits_domain: "{{ lnbits_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create lnbits directory @@ -155,123 +151,3 @@ caddy_site_upstream: "localhost:{{ lnbits_port }}" caddy_site_headers_up: X-Forwarded-Host: "{{ lnbits_domain }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Uptime Kuma monitor setup script for LNBits - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_lnbits_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ lnbits_domain }}/api/v1/health" - monitor_name: "LNBits" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_lnbits_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_lnbits_monitor.py - - /tmp/ansible_config.yml - diff --git a/ansible/services/lnbits/lnbits_vars.yml b/ansible/services/lnbits/lnbits_vars.yml index eabd6fc..466ce78 100644 --- a/ansible/services/lnbits/lnbits_vars.yml +++ b/ansible/services/lnbits/lnbits_vars.yml @@ -3,7 +3,7 @@ lnbits_dir: /opt/lnbits lnbits_data_dir: "{{ lnbits_dir }}/data" lnbits_port: 8765 -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access remote_host_name: "{{ groups['edge'] | first }}" @@ -12,6 +12,3 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/lnbits-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/lnbits_backup.sh" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml index 46abf31..2da0ac0 100644 --- a/ansible/services/lnbits/setup_backup_lnbits.yml +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -9,7 +9,6 @@ hosts: edge become: yes vars_files: - - ../../group_vars/all/main.yml - ./lnbits_vars.yml tasks: @@ -23,3 +22,7 @@ backup_source_stop_service: lnbits backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:20:00" + # Reported to Gatus as backup-dump_lnbits. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_lnbits/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml deleted file mode 100644 index 48296cb..0000000 --- a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml +++ /dev/null @@ -1,104 +0,0 @@ -- name: Configure local backup for LNBits from remote - hosts: control - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./lnbits_vars.yml - vars: - remote_data_path: "{{ lnbits_data_dir }}" - remote_lnbits_dir: "{{ lnbits_dir }}/lnbits" - gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" - gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - "gpg_recipient={{ gpg_recipient }}" - - "gpg_key_id={{ gpg_key_id }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - ENCRYPTED_BACKUP="{{ local_backup_dir }}/lnbits-backup-$TIMESTAMP.tar.gz.gpg" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - # Stop LNBits service before backup - echo "Stopping LNBits service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop lnbits.service" - - # Create encrypted backup on the fly - # First, create a tar archive of the data directory and pipe it through gpg - echo "Creating backup..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "cd {{ remote_data_path }} && tar -czf - ." | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" - - # Also backup the .env file separately (smaller, might need quick access) - $SSH_CMD {{ remote_user }}@{{ remote_host }} "cat {{ remote_lnbits_dir }}/.env" | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "{{ local_backup_dir }}/lnbits-env-$TIMESTAMP.gpg" - - # Start LNBits service after backup - echo "Starting LNBits service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start lnbits.service" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for backup_file in "{{ local_backup_dir }}"/lnbits-backup-*.tar.gz.gpg; do - if [ -f "$backup_file" ]; then - # Extract date from filename: lnbits-backup-YYYY-MM-DD.tar.gz.gpg - file_date=$(basename "$backup_file" | sed -n 's/lnbits-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$backup_file" - fi - fi - done - for env_file in "{{ local_backup_dir }}"/lnbits-env-*.gpg; do - if [ -f "$env_file" ]; then - # Extract date from filename: lnbits-env-YYYY-MM-DD.gpg - file_date=$(basename "$env_file" | sed -n 's/lnbits-env-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$env_file" - fi - fi - done - - echo "Backup completed successfully" - - - name: Ensure cronjob for backup exists - cron: - name: "LNBits backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index 04e99cf..8b21b85 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -2,9 +2,6 @@ hosts: memos become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" @@ -65,14 +62,6 @@ owner: root group: root - - name: Clean up temporary files - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/memos.tar.gz - - /tmp/memos - - name: Create memos environment file copy: dest: "{{ memos_config_dir }}/memos.env" @@ -139,18 +128,7 @@ msg: "Memos is running on port {{ memos_port }}. Access via Tailscale at http://{{ memos_tailscale_hostname }}:{{ memos_port }}" handlers: - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - name: Restart memos - when: uptime_kuma_enabled | default(false) systemd: name: memos state: restarted @@ -160,14 +138,10 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Publish Memos through Caddy (via Tailscale) @@ -178,117 +152,3 @@ caddy_site_domain: "{{ memos_domain }}" caddy_site_upstream: "{{ memos_tailscale_hostname }}:{{ memos_port }}" caddy_site_resolvers: "100.100.100.100" - - - name: Create Uptime Kuma monitor setup script for Memos - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_memos_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_memos_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_memos_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ memos_domain }}/healthz" - monitor_name: "Memos" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_memos_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_memos_monitor.py - - /tmp/ansible_memos_config.yml diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index 99618db..94c6de7 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -12,15 +12,7 @@ memos_url: "https://github.com/usememos/memos/releases/download/v{{ memos_versio memos_tailscale_hostname: "memos-box" memos_tailscale_ip: "100.64.0.4" -# (caddy_sites_dir and subdomain in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access (for backup from lapy via Tailscale) -backup_host: "{{ memos_tailscale_hostname }}" -backup_user: "counterweight" -backup_key_file: "~/.ssh/counterganzua" -backup_port: 22 - -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/memos_backup.sh" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml index 2131924..53ab85d 100644 --- a/ansible/services/memos/setup_backup_memos.yml +++ b/ansible/services/memos/setup_backup_memos.yml @@ -7,7 +7,6 @@ hosts: memos become: yes vars_files: - - ../../group_vars/all/main.yml - ./memos_vars.yml tasks: @@ -24,3 +23,7 @@ backup_source_stop_service: memos backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_memos. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_memos/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml deleted file mode 100644 index c32c332..0000000 --- a/ansible/services/memos/setup_backup_memos_to_lapy.yml +++ /dev/null @@ -1,106 +0,0 @@ -- name: Configure local backup for Memos from memos-box - hosts: control - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./memos_vars.yml - vars: - backup_data_path: "{{ memos_data_dir }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "backup_host={{ backup_host }}" - - "backup_user={{ backup_user }}" - - "backup_data_path='{{ backup_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if backup_key_file %} - SSH_CMD="ssh -i {{ backup_key_file }} -p {{ backup_port }}" - {% else %} - SSH_CMD="ssh -p {{ backup_port }}" - {% endif %} - - rsync -az -e "$SSH_CMD" --rsync-path="sudo rsync" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for dir in "{{ local_backup_dir }}"/20*; do - if [ -d "$dir" ]; then - dir_date=$(basename "$dir") - if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then - rm -rf "$dir" - fi - fi - done - - - name: Ensure cronjob for backup exists - cron: - name: "Memos backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 15 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check backup directory exists and contains files - stat: - path: "{{ local_backup_dir }}/{{ today_date.stdout }}" - register: backup_dir_stat - - - name: Verify backup directory exists - assert: - that: - - backup_dir_stat.stat.exists - - backup_dir_stat.stat.isdir - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" - success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" - - - name: Check if backup directory contains files - find: - paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" - recurse: yes - register: backup_files - - - name: Verify backup directory is not empty - assert: - that: - - backup_files.files | length > 0 - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" - success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" - diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index e9cab0e..0042fb0 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -1,620 +1,27 @@ +--- - name: Deploy Mempool Block Explorer with Docker hosts: mempool become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./mempool_vars.yml vars: - mempool_subdomain: "{{ subdomains.mempool }}" - mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - # =========================================== - # Docker Installation (from 910_docker_playbook.yml) - # =========================================== - - name: Remove old Docker-related packages - apt: - name: - - docker.io - - docker-doc - - docker-compose - - podman-docker - - containerd - - runc - state: absent - purge: yes - autoremove: yes - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install prerequisites - apt: - name: - - ca-certificates - - curl - state: present - - - name: Create directory for Docker GPG key - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - - - name: Download Docker GPG key - get_url: - url: https://download.docker.com/linux/debian/gpg - dest: /etc/apt/keyrings/docker.asc - mode: '0644' - - - name: Get Debian architecture - command: dpkg --print-architecture - register: deb_arch - changed_when: false - - - name: Add Docker repository - apt_repository: - repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" - filename: docker - state: present - update_cache: yes - - - name: Install Docker packages - apt: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present - update_cache: yes - - - name: Ensure Docker is started and enabled - systemd: - name: docker - enabled: yes - state: started - - - name: Add user to docker group - user: - name: "{{ ansible_user }}" - groups: docker - append: yes - - # =========================================== - # Mempool Deployment - # =========================================== - - name: Create mempool directories - file: - path: "{{ item }}" - state: directory - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0755' - loop: - - "{{ mempool_dir }}" - - "{{ mempool_data_dir }}" - - "{{ mempool_mysql_dir }}" - - - name: Create docker-compose.yml for Mempool - copy: - dest: "{{ mempool_dir }}/docker-compose.yml" - content: | - # All containers use host network for Tailscale MagicDNS resolution - services: - mariadb: - image: mariadb:10.11 - container_name: mempool-db - restart: unless-stopped - network_mode: host - environment: - MYSQL_DATABASE: "{{ mariadb_database }}" - MYSQL_USER: "{{ mariadb_user }}" - MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" - MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" - volumes: - - {{ mempool_mysql_dir }}:/var/lib/mysql - healthcheck: - test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - mempool-backend: - image: mempool/backend:{{ mempool_version }} - container_name: mempool-backend - restart: unless-stopped - network_mode: host - environment: - # Database (localhost since all containers share host network) - DATABASE_ENABLED: "true" - DATABASE_HOST: "127.0.0.1" - DATABASE_DATABASE: "{{ mariadb_database }}" - DATABASE_USERNAME: "{{ mariadb_user }}" - DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" - # Bitcoin Core/Knots (via Tailnet MagicDNS) - CORE_RPC_HOST: "{{ bitcoin_host }}" - CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" - CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" - CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" - # Electrum (Fulcrum via Tailnet MagicDNS) - ELECTRUM_HOST: "{{ fulcrum_host }}" - ELECTRUM_PORT: "{{ fulcrum_port }}" - ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" - # Mempool settings - MEMPOOL_NETWORK: "{{ mempool_network }}" - MEMPOOL_BACKEND: "electrum" - MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" - MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" - volumes: - - {{ mempool_data_dir }}:/backend/cache - depends_on: - mariadb: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - mempool-frontend: - image: mempool/frontend:{{ mempool_version }} - container_name: mempool-frontend - restart: unless-stopped - network_mode: host - environment: - FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" - BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" - depends_on: - - mempool-backend - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0644' - - - name: Pull Mempool images - command: docker compose pull - args: - chdir: "{{ mempool_dir }}" - - - name: Deploy Mempool containers with docker compose - command: docker compose up -d - args: - chdir: "{{ mempool_dir }}" - - - name: Wait for MariaDB to be healthy - command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db - register: mariadb_health - until: mariadb_health.stdout == 'healthy' - retries: 30 - delay: 10 - changed_when: false - - - name: Wait for Mempool backend to start - uri: - url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - method: GET - status_code: 200 - timeout: 10 - register: backend_check - until: backend_check.status == 200 - retries: 30 - delay: 10 - ignore_errors: yes - - - name: Wait for Mempool frontend to be available - uri: - url: "http://localhost:{{ mempool_frontend_port }}" - method: GET - status_code: 200 - timeout: 10 - register: frontend_check - until: frontend_check.status == 200 - retries: 20 - delay: 5 - ignore_errors: yes - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Display deployment status - when: uptime_kuma_enabled | default(false) - debug: - msg: - - "Mempool deployment complete!" - - "Frontend: http://localhost:{{ mempool_frontend_port }}" - - "Backend API: http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - - "Backend check: {{ 'OK' if backend_check.status == 200 else 'Still initializing...' }}" - - "Frontend check: {{ 'OK' if frontend_check.status == 200 else 'Still initializing...' }}" - - # =========================================== - # Health Check Scripts for Uptime Kuma Push Monitors - # =========================================== - - name: Create Mempool MariaDB health check script - when: uptime_kuma_enabled | default(false) - copy: - dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - check_container() { - local status=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db 2>/dev/null) - [ "$status" = "healthy" ] - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_container; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "MariaDB container unhealthy" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create Mempool backend health check script - when: uptime_kuma_enabled | default(false) - copy: - dest: /usr/local/bin/mempool-backend-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - BACKEND_PORT={{ mempool_backend_port }} - - check_backend() { - curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_backend; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Backend API not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create Mempool frontend health check script - when: uptime_kuma_enabled | default(false) - copy: - dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - FRONTEND_PORT={{ mempool_frontend_port }} - - check_frontend() { - curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_frontend; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Frontend not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - # =========================================== - # Systemd Timers for Health Checks - # =========================================== - - name: Create systemd services for health checks - copy: - dest: "/etc/systemd/system/mempool-{{ item.name }}-healthcheck.service" - content: | - [Unit] - Description=Mempool {{ item.label }} Health Check - After=network.target docker.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/mempool-{{ item.name }}-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - loop: - - { name: "mariadb", label: "MariaDB" } - - { name: "backend", label: "Backend" } - - { name: "frontend", label: "Frontend" } - - - name: Create systemd timers for health checks - copy: - dest: "/etc/systemd/system/mempool-{{ item }}-healthcheck.timer" - content: | - [Unit] - Description=Mempool {{ item }} Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - loop: - - mariadb - - backend - - frontend - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start health check timers - when: uptime_kuma_enabled | default(false) - systemd: - name: "mempool-{{ item }}-healthcheck.timer" - enabled: yes - state: started - loop: - - mariadb - - backend - - frontend - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for Mempool - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_mempool_monitors.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_mempool_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitors_to_create = config['monitors'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - results = {} - for monitor_name in monitors_to_create: - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - results[monitor_name] = f"{url}/api/push/{push_token}" - print(f"Push URL ({monitor_name}): {results[monitor_name]}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - results[monitor_name] = f"{url}/api/push/{push_token}" - print(f"Push URL ({monitor_name}): {results[monitor_name]}") - - api.disconnect() - print("SUCCESS") - - # Write results to file for Ansible to read - with open('/tmp/mempool_push_urls.yml', 'w') as f: - yaml.dump(results, f) - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_mempool_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitors: - - "Mempool MariaDB" - - "Mempool Backend" - - "Mempool Frontend" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_mempool_monitors.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URLs from file - when: uptime_kuma_enabled | default(false) - slurp: - src: /tmp/mempool_push_urls.yml - delegate_to: localhost - become: no - register: push_urls_file - ignore_errors: yes - - - name: Parse push URLs - set_fact: - push_urls: "{{ push_urls_file.content | b64decode | from_yaml }}" - when: push_urls_file.content is defined - ignore_errors: yes - - - name: Update MariaDB health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-mariadb-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool MariaDB'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool MariaDB'] is defined - - - name: Update Backend health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-backend-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Backend'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool Backend'] is defined - - - name: Update Frontend health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-frontend-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Frontend'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool Frontend'] is defined - - - name: Reload systemd after push URL updates - systemd: - daemon_reload: yes - when: push_urls is defined - - - name: Restart health check timers - systemd: - name: "mempool-{{ item }}-healthcheck.timer" - state: restarted - loop: - - mariadb - - backend - - frontend - when: push_urls is defined - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_mempool_monitors.py - - /tmp/ansible_mempool_config.yml - - /tmp/mempool_push_urls.yml + # Preserves the three push URLs these checks have been reporting to all + # along, so the move to a role changes no behaviour. The role knows nothing + # about Uptime Kuma — these are just "URLs that accept a ping", and whatever + # replaces it sets the same values. + mempool_healthchecks: + - {name: mariadb, label: MariaDB, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-mariadb/external"} + - {name: backend, label: Backend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-backend/external"} + - {name: frontend, label: Frontend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-frontend/external"} + # One token for all three components: they run on the same host, so the + # blast radius is already that host. + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - mempool - name: Configure Caddy reverse proxy for Mempool on the edge host hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./mempool_vars.yml vars: - mempool_subdomain: "{{ subdomains.mempool }}" - mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - + mempool_domain: "{{ subdomains.mempool }}.{{ root_domain }}" tasks: - name: Publish Mempool through Caddy (via Tailscale) ansible.builtin.include_role: @@ -622,124 +29,5 @@ vars: caddy_site_name: mempool caddy_site_domain: "{{ mempool_domain }}" - caddy_site_upstream: "mempool-box:{{ mempool_frontend_port }}" + caddy_site_upstream: "{{ hostvars['mempool_box_local'].ansible_host }}:{{ hostvars['mempool_box_local'].mempool_frontend_port }}" caddy_site_resolvers: "100.100.100.100" - - - name: Display Mempool URL - when: uptime_kuma_enabled | default(false) - debug: - msg: "Mempool is now available at https://{{ mempool_domain }}" - - # =========================================== - # Uptime Kuma HTTP Monitor for Public Endpoint - # =========================================== - - name: Create Uptime Kuma HTTP monitor setup script for Mempool - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_mempool_http_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_mempool_http_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating HTTP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for HTTP monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_mempool_http_config.yml - content: | - uptime_kuma_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ mempool_domain }}" - monitor_name: "Mempool Public Access" - mode: '0644' - - - name: Run Uptime Kuma HTTP monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_mempool_http_monitor.py - delegate_to: localhost - become: no - register: http_monitor_setup - changed_when: "'SUCCESS' in http_monitor_setup.stdout" - ignore_errors: yes - - - name: Display HTTP monitor setup output - debug: - msg: "{{ http_monitor_setup.stdout_lines }}" - when: http_monitor_setup.stdout is defined - - - name: Clean up HTTP monitor temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_mempool_http_monitor.py - - /tmp/ansible_mempool_http_config.yml - diff --git a/ansible/services/mempool/mempool_vars.yml b/ansible/services/mempool/mempool_vars.yml deleted file mode 100644 index d3051c3..0000000 --- a/ansible/services/mempool/mempool_vars.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Mempool Configuration Variables - -# Version - Pinned to specific release -mempool_version: "v3.2.1" - -# Directories -mempool_dir: /opt/mempool -mempool_data_dir: "{{ mempool_dir }}/data" -mempool_mysql_dir: "{{ mempool_dir }}/mysql" - -# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) -bitcoin_host: "knots-box" -bitcoin_rpc_port: 8332 -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml - -# Network - Fulcrum Electrum server (via Tailnet Magic DNS) -fulcrum_host: "fulcrum-box" -fulcrum_port: 50001 -fulcrum_tls: "false" - -# Mempool network mode -mempool_network: "mainnet" - -# Container ports (internal) -mempool_frontend_port: 8080 -mempool_backend_port: 8999 - -# MariaDB settings -mariadb_database: "mempool" -mariadb_user: "mempool" -# Note: mariadb_mempool_password is loaded from infra_secrets.yml - - diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index 7379d5f..58f0348 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -2,18 +2,23 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../infra_secrets.yml - - ../../services_config.yml - ./ntfy_emergency_app_vars.yml vars: ntfy_emergency_app_subdomain: "{{ subdomains.ntfy_emergency_app }}" ntfy_emergency_app_domain: "{{ ntfy_emergency_app_subdomain }}.{{ root_domain }}" - ntfy_service_domain: "{{ subdomains.ntfy }}.{{ root_domain }}" - ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" + # ⚠ UNRESOLVED: this app sends its notifications to an ntfy server, and the + # server it points at ran on watchtower, which is being destroyed. This was + # derived from subdomains.ntfy, which is now gone with it. + # + # Until an ntfy server exists again this URL is dead, and the app fails + # silently at exactly the moment it matters - it is an EMERGENCY notifier. + # Three ways out, none of them automatic: + # * stand ntfy up somewhere else (the monitoring VPS, or vipy) + # * point this at the public ntfy.sh + # * retire the app + ntfy_emergency_app_ntfy_url: "{{ ntfy_service_url }}" ntfy_emergency_app_ntfy_user: "{{ ntfy_username | default('') }}" ntfy_emergency_app_ntfy_password: "{{ ntfy_password | default('') }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create ntfy-emergency-app directory @@ -55,127 +60,3 @@ caddy_site_name: ntfy-emergency-app caddy_site_domain: "{{ ntfy_emergency_app_domain }}" caddy_site_upstream: "localhost:{{ ntfy_emergency_app_port }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_ntfy_emergency_app_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ ntfy_emergency_app_domain }}" - monitor_name: "ntfy-emergency-app" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_ntfy_emergency_app_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_ntfy_emergency_app_monitor.py - - /tmp/ansible_config.yml diff --git a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml index a3bb480..bfa4bd1 100644 --- a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml +++ b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml @@ -2,7 +2,7 @@ ntfy_emergency_app_dir: /opt/ntfy-emergency-app ntfy_emergency_app_port: 3000 -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # ntfy configuration ntfy_emergency_app_topic: "emergencia" @@ -14,3 +14,9 @@ remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_h remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" + +# Where the emergency notifications are sent. This pointed at ntfy.contrapeso.xyz +# on watchtower; that host is being destroyed, so this MUST be repointed before +# the app can work again. Left at the old value so the break is visible rather +# than silently defaulted to something plausible. +ntfy_service_url: "https://ntfy.contrapeso.xyz" diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml deleted file mode 100644 index 61fafe1..0000000 --- a/ansible/services/ntfy/deploy_ntfy_playbook.yml +++ /dev/null @@ -1,100 +0,0 @@ -- name: Deploy ntfy and configure Caddy reverse proxy - hosts: monitoring - become: yes - vars_files: - - ../../infra_vars.yml - - ../../infra_secrets.yml - - ../../services_config.yml - - ./ntfy_vars.yml - vars: - ntfy_subdomain: "{{ subdomains.ntfy }}" - ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" - - tasks: - - name: Ensure /etc/apt/keyrings exists - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - - - name: Download and dearmor ntfy GPG key - shell: curl -fsSL https://archive.heckel.io/apt/pubkey.txt | gpg --dearmor -o /etc/apt/keyrings/archive.heckel.io.gpg - args: - creates: /etc/apt/keyrings/archive.heckel.io.gpg - - - name: Add ntfy APT repository - copy: - dest: /etc/apt/sources.list.d/archive.heckel.io.list - content: | - deb [arch=amd64 signed-by=/etc/apt/keyrings/archive.heckel.io.gpg] https://archive.heckel.io/apt debian main - mode: '0644' - - - name: Update APT cache - apt: - update_cache: yes - - - name: Install ntfy - apt: - name: ntfy - state: present - - - name: Ensure ntfy cache directories exist - file: - path: "{{ item }}" - state: directory - owner: ntfy - group: ntfy - mode: '0755' - loop: - - /var/cache/ntfy - - /var/cache/ntfy/attachments - - - name: Deploy ntfy configuration file - copy: - dest: /etc/ntfy/server.yml - content: | - base-url: "http://{{ ntfy_domain }}" - listen-http: ":{{ ntfy_port }}" - cache-file: "/var/cache/ntfy/cache.db" - attachment-cache-dir: "/var/cache/ntfy/attachments" - behind-proxy: true - auth-file: "/var/lib/ntfy/user.db" - auth-default-access: "deny-all" - owner: root - group: root - mode: '0644' - notify: Restart ntfy - - - name: Enable and start ntfy service - systemd: - name: ntfy - enabled: yes - state: started - - - name: Create ntfy admin user - shell: | - (echo "{{ ntfy_password }}"; echo "{{ ntfy_password }}") | ntfy user add --role=admin "{{ ntfy_username }}" - - - name: Publish ntfy through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: ntfy - caddy_site_domain: "{{ ntfy_domain }}, http://{{ ntfy_domain }}" - # Raw body: ntfy needs a plain-HTTP listener for its CLI/app clients, - # with only GETs to the docs and topic paths redirected to HTTPS. - caddy_site_body: | - reverse_proxy 127.0.0.1:{{ ntfy_port }} - - @httpget { - protocol http - method GET - path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) - } - redir @httpget https://{host}{uri} - - handlers: - - name: Restart ntfy - systemd: - name: ntfy - state: restarted \ No newline at end of file diff --git a/ansible/services/ntfy/ntfy_vars.yml b/ansible/services/ntfy/ntfy_vars.yml deleted file mode 100644 index 5ebec37..0000000 --- a/ansible/services/ntfy/ntfy_vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -ntfy_port: 6674 - -# ntfy_topic now lives in services_config.yml under service_settings.ntfy.topic \ No newline at end of file diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index 96d030f..21d8b43 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -2,14 +2,10 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./personal_blog_vars.yml vars: personal_blog_subdomain: "{{ subdomains.personal_blog }}" personal_blog_domain: "{{ personal_blog_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure user is in www-data group @@ -57,123 +53,3 @@ caddy_site_name: personal-blog caddy_site_domain: "{{ personal_blog_domain }}" caddy_site_root: "{{ personal_blog_web_root }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Uptime Kuma monitor setup script for Personal Blog - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_personal_blog_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ personal_blog_domain }}" - monitor_name: "Personal Blog" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_personal_blog_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_personal_blog_monitor.py - - /tmp/ansible_config.yml - diff --git a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml index 99f9b34..bd9b715 100644 --- a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml +++ b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml @@ -2,7 +2,6 @@ hosts: control gather_facts: no vars_files: - - ../../infra_vars.yml - ./personal_blog_vars.yml vars: bashrc_path: "{{ lookup('env', 'HOME') }}/.bashrc" diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index 8b30a1f..b9cf243 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -1,552 +1,15 @@ --- -# phoenixd Deployment Playbook -# -# Deploys phoenixd (https://phoenix.acinq.co/server), the server version of the -# Phoenix Lightning wallet, on vipy so LNBits can use it as a wallet backend -# over loopback. -# -# What this does: -# 1. Downloads the pinned phoenixd release and installs phoenixd + phoenix-cli -# 2. Creates a dedicated system user and a 0700 data directory -# 3. Creates and enables a systemd service -# 4. Creates a push-monitor health check script + systemd timer -# 5. Registers a push monitor in Uptime Kuma -# -# The HTTP API stays bound to 127.0.0.1 and is NOT proxied by Caddy: phoenixd -# holds funds and its API is protected by a single password. Anything that needs -# it either runs on this host or reaches it over the Tailscale mesh. -# -# ⚠️ After the first run, back up {{ phoenixd_data_dir }}/seed.dat. Losing it -# means losing the funds. See setup_backup_phoenixd_to_lapy.yml. - +# phoenixd: Lightning node on the edge host, used by LNBits as a wallet backend. +# Never exposed through Caddy — the HTTP API stays on loopback. - name: Deploy phoenixd on the edge host hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./phoenixd_vars.yml vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - # =========================================== - # Prerequisites - # =========================================== - - name: Install phoenixd runtime dependencies - apt: - name: - - unzip - - curl - state: present - update_cache: yes - - # =========================================== - # System User and Directories - # =========================================== - - name: Create phoenixd system group - group: - name: "{{ phoenixd_group }}" - system: yes - - - name: Create phoenixd system user - user: - name: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ phoenixd_home }}" - create_home: yes - comment: "phoenixd Lightning node" - - - name: Create phoenixd home directory - file: - path: "{{ phoenixd_home }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0750" - - - name: Create phoenixd data directory - file: - path: "{{ phoenixd_data_dir }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0700" - - # =========================================== - # Download and Install - # =========================================== - - name: Check if phoenixd is already installed - stat: - path: "{{ phoenixd_bin_dir }}/phoenixd" - register: phoenixd_binary - - - name: Check installed phoenixd version - command: "{{ phoenixd_bin_dir }}/phoenixd --version" - register: phoenixd_installed_version - changed_when: false - failed_when: false - when: phoenixd_binary.stat.exists - - - name: Decide whether phoenixd needs installing - set_fact: - phoenixd_needs_install: >- - {{ not phoenixd_binary.stat.exists - or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} - - - name: Download phoenixd {{ phoenixd_version }} - get_url: - url: "{{ phoenixd_url }}" - dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - mode: "0644" - when: phoenixd_needs_install | bool - - - name: Create temporary extraction directory - file: - path: /tmp/phoenixd-extract - state: directory - mode: "0755" - when: phoenixd_needs_install | bool - - - name: Extract phoenixd archive - unarchive: - src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - dest: /tmp/phoenixd-extract - remote_src: yes - when: phoenixd_needs_install | bool - - - name: Locate extracted binaries - find: - paths: /tmp/phoenixd-extract - patterns: "{{ item }}" - recurse: yes - file_type: file - register: phoenixd_extracted - loop: - - phoenixd - - phoenix-cli - when: phoenixd_needs_install | bool - - - name: Fail if the archive did not contain the expected binaries - assert: - that: - - item.files | length > 0 - fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - - - name: Install phoenixd and phoenix-cli binaries - copy: - src: "{{ item.files[0].path }}" - dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" - remote_src: yes - owner: root - group: root - mode: "0755" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - notify: Restart phoenixd - - - name: Clean up phoenixd download artifacts - file: - path: "{{ item }}" - state: absent - loop: - - "/tmp/phoenixd-{{ phoenixd_version }}.zip" - - /tmp/phoenixd-extract - - # =========================================== - # Systemd Service - # =========================================== - - name: Build phoenixd command line arguments - set_fact: - phoenixd_args: >- - {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) - + ['--chain', phoenixd_chain] - + ['--auto-liquidity', phoenixd_auto_liquidity] - + ['--http-bind-ip', phoenixd_http_bind_ip] - + ['--http-bind-port', phoenixd_http_bind_port | string] - + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) - + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) - + ['--silent'] }} - - - name: Create phoenixd systemd service - copy: - dest: /etc/systemd/system/phoenixd.service - content: | - [Unit] - Description=phoenixd - Lightning Network Node - Documentation=https://phoenix.acinq.co/server - After=network-online.target - Wants=network-online.target - - [Service] - Type=simple - User={{ phoenixd_user }} - Group={{ phoenixd_group }} - WorkingDirectory={{ phoenixd_home }} - Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} - ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} - Restart=always - RestartSec=30 - TimeoutStartSec=120 - TimeoutStopSec=120 - StandardOutput=journal - StandardError=journal - - # Hardening: the node only ever writes to its own data directory - NoNewPrivileges=true - PrivateTmp=true - ProtectSystem=strict - ProtectHome=read-only - ReadWritePaths={{ phoenixd_data_dir }} - - LimitNOFILE=65535 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - notify: Restart phoenixd - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start phoenixd - systemd: - name: phoenixd - enabled: yes - state: started - - - name: Flush handlers so phoenixd is running before we inspect its data dir - meta: flush_handlers - - # =========================================== - # First Boot Checks - # =========================================== - - name: Wait for phoenixd to write its config file - wait_for: - path: "{{ phoenixd_data_dir }}/phoenix.conf" - state: present - timeout: 120 - - - name: Check that the seed file exists - stat: - path: "{{ phoenixd_data_dir }}/seed.dat" - register: phoenixd_seed_file - - - name: Fail if phoenixd did not create a seed - assert: - that: - - phoenixd_seed_file.stat.exists - fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" - - # =========================================== - # Health Check Script + Systemd Timer - # =========================================== - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create phoenixd health check script - when: uptime_kuma_enabled | default(false) - copy: - dest: "{{ phoenixd_healthcheck_script_path }}" - content: | - #!/bin/bash - # Checks phoenixd and pushes the result to Uptime Kuma. - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" - - check_phoenixd() { - # Service must be active and the node must answer getinfo. - # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, - # but not the bind address, so pass it explicitly. - systemctl is-active --quiet phoenixd && \ - {{ phoenixd_bin_dir }}/phoenix-cli \ - --http-bind-ip {{ phoenixd_http_bind_ip }} \ - --http-bind-port {{ phoenixd_http_bind_port }} \ - getinfo 2>/dev/null | grep -q '"nodeId"' - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_phoenixd; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "phoenixd not responding" - exit 1 - fi - owner: root - group: root - mode: "0755" - - - name: Create phoenixd health check systemd service - copy: - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" - content: | - [Unit] - Description=phoenixd Health Check - After=network.target phoenixd.service - - [Service] - Type=oneshot - User=root - ExecStart={{ phoenixd_healthcheck_script_path }} - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - - - name: Create phoenixd health check systemd timer - copy: - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" - content: | - [Unit] - Description=phoenixd Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - - - name: Enable and start phoenixd health check timer - when: uptime_kuma_enabled | default(false) - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - enabled: yes - state: started - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for phoenixd - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_phoenixd_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_phoenixd_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - push_url = None - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - else: - print(f"Creating push monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - # socketio timeout: add_monitor may have succeeded server-side - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - - api.disconnect() - - if push_url: - print(f"PUSH_URL={push_url}") - with open('/tmp/phoenixd_push_url.txt', 'w') as f: - f.write(push_url) - - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for push monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_phoenixd_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "{{ phoenixd_monitor_name }}" - mode: "0644" - - - name: Run Uptime Kuma push monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_phoenixd_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URL from file - when: uptime_kuma_enabled | default(false) - slurp: - src: /tmp/phoenixd_push_url.txt - delegate_to: localhost - become: no - register: push_url_file - ignore_errors: yes - - - name: Parse push URL - set_fact: - phoenixd_push_url: "{{ push_url_file.content | b64decode | trim }}" - when: push_url_file.content is defined - - - name: Update health check service with push URL - lineinfile: - path: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" - regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" - line: "Environment=UPTIME_KUMA_PUSH_URL={{ phoenixd_push_url }}" - when: phoenixd_push_url is defined - notify: Restart phoenixd health check timer - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_phoenixd_monitor.py - - /tmp/ansible_phoenixd_config.yml - - /tmp/phoenixd_push_url.txt - - # =========================================== - # Post-install Notes - # =========================================== - - name: Display post-install information - debug: - msg: | - ✓ phoenixd {{ phoenixd_version }} deployed - - Status: systemctl status phoenixd - Logs: journalctl -u phoenixd -f - CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo - HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) - Data dir: {{ phoenixd_data_dir }} - - API password (needed to wire LNBits up to this node): - sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf - - ⚠️ BACK UP THE SEED NOW: {{ phoenixd_data_dir }}/seed.dat - Losing it means losing the funds. Run - services/phoenixd/setup_backup_phoenixd_to_lapy.yml and also keep - the 12 words somewhere offline. - - handlers: - - name: Restart phoenixd - systemd: - name: phoenixd - state: restarted - daemon_reload: yes - - - name: Restart phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - state: restarted - daemon_reload: yes + # phoenixd's health check has never reported anywhere since the Uptime Kuma + # decommissioning — its systemd Environment= was left empty. Leaving it empty + # preserves that; the check still runs and its exit code is still the answer. + # Set this to plug in whatever monitoring replaces it. + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_phoenixd/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - phoenixd diff --git a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml deleted file mode 100644 index 7474af1..0000000 --- a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- -# Backs up the phoenixd seed and config from vipy to Lapy, gpg encrypted. -# -# Only seed.dat and phoenix.conf are backed up, on purpose: -# - seed.dat is what actually recovers the funds. phoenixd keeps its channel -# state with the ACINQ peer, so a wallet is restored from the seed alone. -# - restoring a *stale* channel database to a running Lightning node is -# dangerous (it can trigger a force close and a penalty), so we do not keep -# copies of phoenix.db around to be tempted by. -# - phoenix.conf holds the http api password, which is what LNBits and any -# other local consumer authenticate with. -# -# Because both files are static, phoenixd does not need to be stopped. - -- name: Configure local backup for phoenixd from remote - hosts: control - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./phoenixd_vars.yml - vars: - remote_data_path: "{{ phoenixd_data_dir }}" - gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" - gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" - - tasks: - - name: Debug phoenixd backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - "gpg_recipient={{ gpg_recipient }}" - - "gpg_key_id={{ gpg_key_id }}" - - - name: Ensure local backup directory exists - ansible.builtin.file: - path: "{{ local_backup_dir }}" - state: directory - mode: "0700" - - - name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: "0755" - - - name: Create phoenixd backup script - ansible.builtin.copy: - dest: "{{ backup_script_path }}" - mode: "0750" - content: | - #!/bin/bash - set -euo pipefail - - if [ -z "{{ gpg_recipient }}" ]; then - echo "GPG recipient is not configured. Aborting." - exit 1 - fi - - TIMESTAMP=$(date +'%Y-%m-%d') - ENCRYPTED_BACKUP="{{ local_backup_dir }}/phoenixd-backup-$TIMESTAMP.tar.gz.gpg" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - # seed.dat + phoenix.conf only, see the header of the playbook. - echo "Creating encrypted backup archive..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} \ - "sudo tar -czf - -C {{ remote_data_path }} seed.dat phoenix.conf" | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" - - chmod 600 "$ENCRYPTED_BACKUP" - - # Rotate old backups (keep 14 days) - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for backup_file in "{{ local_backup_dir }}"/phoenixd-backup-*.tar.gz.gpg; do - if [ -f "$backup_file" ]; then - # Extract date from filename: phoenixd-backup-YYYY-MM-DD.tar.gz.gpg - file_date=$(basename "$backup_file" | sed -n 's/phoenixd-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$backup_file" - fi - fi - done - - echo "Backup completed successfully" - - - name: Ensure cronjob for phoenixd backup exists - ansible.builtin.cron: - name: "phoenixd backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 15 - hour: "9" - - - name: Run phoenixd backup script to create initial backup - ansible.builtin.command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check if backup file exists - stat: - path: "{{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg" - register: backup_file_stat - - - name: Verify backup file exists - assert: - that: - - backup_file_stat.stat.exists - - backup_file_stat.stat.isreg - fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" - success_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists" - - - name: Verify backup file is not empty - assert: - that: - - backup_file_stat.stat.size > 0 - fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" - success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" - - - name: Remind about the offline seed copy - debug: - msg: | - These encrypted backups are only as safe as your GPG key. - Also write the 12 words down offline once: - ssh {{ remote_user }}@{{ remote_host }} "sudo cat {{ remote_data_path }}/seed.dat" diff --git a/ansible/services/signal-api/deploy_signal_api_playbook.yml b/ansible/services/signal-api/deploy_signal_api_playbook.yml new file mode 100644 index 0000000..d4690f9 --- /dev/null +++ b/ansible/services/signal-api/deploy_signal_api_playbook.yml @@ -0,0 +1,44 @@ +--- +# The Signal transport for Gatus alerts. +# +# Deliberately NOT published and NOT fronted by Caddy: the API has no +# authentication of any kind, so it is reachable only from the shared docker +# network that Gatus is on. See roles/signal_api/README.md, including the +# one-time manual step to link the device. +- name: Deploy the Signal API on the observability host + hosts: observability + become: yes + roles: + - signal_api + + # post_tasks, not a second play: the registration below needs the role's + # defaults (service name, port) in scope, and a separate play would not have + # them. + post_tasks: + # Monitored, because a dead alert transport is the worst kind of dead: every + # check could be failing and nothing would tell you. Gatus polls it over the + # shared network - the same path the alerts take - so this proves the actual + # delivery route rather than merely that a container is running. + # + # Deliberately NOT backed up: the data directory holds Signal private keys, + # and the recovery path is to link the device again from the phone. Backing + # it up would copy a credential off the host to buy nothing. + - name: Register the signal-api health endpoint with Gatus + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: signal-api + gatus_endpoint_pulled: + - name: signal-api + group: infrastructure + url: "http://{{ signal_api_service_name }}:{{ signal_api_port }}/v1/health" + interval: 5m + # Deliberately NOT alerted via Signal: if this endpoint is down, + # Signal is exactly what cannot deliver the alert. It is visible on + # the dashboard, and its failure shows up indirectly as every other + # alert going missing. + conditions: + # /v1/health answers 204 No Content, not 200 - checked live. Any + # 2xx is asserted rather than the exact code, so an upstream + # change from 204 to 200 does not read as an outage. + - "[STATUS] < 300" diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 74e87d8..282fc2e 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -2,14 +2,10 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./vaultwarden_vars.yml vars: vaultwarden_subdomain: "{{ subdomains.vaultwarden }}" vaultwarden_domain: "{{ vaultwarden_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create vaultwarden directory @@ -87,128 +83,3 @@ caddy_site_name: vaultwarden caddy_site_domain: "{{ vaultwarden_domain }}" caddy_site_upstream: "localhost:{{ vaultwarden_port }}" - - # ═════════════════════════════════════════════════════════════════════════ - # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. - # - # Every task below is inert: uptime_kuma_enabled is false in - # group_vars/all/main.yml, so they all skip and the deployment above still - # runs normally. Kept because the health-check logic is the durable part — - # when a replacement exists, rewire the push transport and flip the flag. - # - # What was being monitored: archive/uptime_kuma/MONITORS.md - # ═════════════════════════════════════════════════════════════════════════ - - name: Create Uptime Kuma monitor setup script for Vaultwarden - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_vaultwarden_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ vaultwarden_domain }}/alive" - monitor_name: "Vaultwarden" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - when: uptime_kuma_enabled | default(false) - command: python3 /tmp/setup_vaultwarden_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - when: uptime_kuma_enabled | default(false) - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_vaultwarden_monitor.py - - /tmp/ansible_config.yml - diff --git a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml index eebb214..bccc2cd 100644 --- a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml +++ b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml @@ -2,7 +2,6 @@ hosts: edge become: yes vars_files: - - ../../infra_vars.yml - ./vaultwarden_vars.yml tasks: diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml index 8430475..d0dfdfa 100644 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -6,7 +6,6 @@ hosts: edge become: yes vars_files: - - ../../group_vars/all/main.yml - ./vaultwarden_vars.yml tasks: @@ -23,3 +22,7 @@ backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:10:00" + # Reported to Gatus as backup-dump_vaultwarden. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_vaultwarden/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml deleted file mode 100644 index 096f9d3..0000000 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml +++ /dev/null @@ -1,105 +0,0 @@ -- name: Configure local backup for Vaultwarden from remote - hosts: control - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./vaultwarden_vars.yml - vars: - remote_data_path: "{{ vaultwarden_data_dir }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for dir in "{{ local_backup_dir }}"/20*; do - if [ -d "$dir" ]; then - dir_date=$(basename "$dir") - if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then - rm -rf "$dir" - fi - fi - done - - - name: Ensure cronjob for backup exists - cron: - name: "Vaultwarden backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check backup directory exists and contains files - stat: - path: "{{ local_backup_dir }}/{{ today_date.stdout }}" - register: backup_dir_stat - - - name: Verify backup directory exists - assert: - that: - - backup_dir_stat.stat.exists - - backup_dir_stat.stat.isdir - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" - success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" - - - name: Check if backup directory contains files - find: - paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" - recurse: yes - register: backup_files - - - name: Verify backup directory is not empty - assert: - that: - - backup_files.files | length > 0 - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" - success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" diff --git a/ansible/services/vaultwarden/vaultwarden_vars.yml b/ansible/services/vaultwarden/vaultwarden_vars.yml index e60f1d4..9edc2cd 100644 --- a/ansible/services/vaultwarden/vaultwarden_vars.yml +++ b/ansible/services/vaultwarden/vaultwarden_vars.yml @@ -3,7 +3,7 @@ vaultwarden_dir: /opt/vaultwarden vaultwarden_data_dir: "{{ vaultwarden_dir }}/data" vaultwarden_port: 8222 -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access remote_host_name: "{{ groups['edge'] | first }}" @@ -12,6 +12,3 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/vaultwarden-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/vaultwarden_backup.sh" diff --git a/ansible/services_config.yml b/ansible/services_config.yml deleted file mode 100644 index a20c853..0000000 --- a/ansible/services_config.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Centralized Services Configuration -# Subdomains and Caddy settings for all services - -# Edit these subdomains to match your preferences -subdomains: - # Monitoring Services (on watchtower) - ntfy: ntfy - # DEPRECATED 2026-09-11 — Uptime Kuma is decommissioned and this subdomain no - # longer resolves to anything. Kept only because the deprecated monitoring - # blocks still template it into uptime_kuma_api_url. See archive/uptime_kuma/. - uptime_kuma: uptime - - # VPN Infrastructure (on spacey) - headscale: headscale - - # Core Services (on vipy) - vaultwarden: vault - forgejo: forgejo - lnbits: wallet - - # Secondary Services (on vipy) - ntfy_emergency_app: avisame - personal_blog: pablohere - - # Memos (on memos-box) - memos: memos - - # Mempool Block Explorer (on mempool_box, proxied via vipy) - mempool: mempool - - # DATUM Gateway dashboard (on knots_box, proxied via vipy) - datum_gateway: datum - -# Caddy configuration -caddy_sites_dir: /etc/caddy/sites-enabled - -# Service-specific settings shared across playbooks -service_settings: - ntfy: - topic: alerts - headscale: - namespace: counter-net diff --git a/ansible/site.yml b/ansible/site.yml new file mode 100644 index 0000000..35f5867 --- /dev/null +++ b/ansible/site.yml @@ -0,0 +1,90 @@ +--- +# Everything, in the order it has to happen. +# +# This file is a TABLE OF CONTENTS, not a second source of truth. It says what +# runs and in what order. It does NOT say which hosts get what — that stays on +# the `hosts:` line inside each playbook, exactly where it is today. Nothing +# moves; this file only makes the set readable in one place. +# +# What runs on a host? ansible-playbook site.yml --limit --list-hosts +# Who gets thing Y? the `hosts:` line in Y's own playbook +# What is a host? ansible-inventory --graph +# +# Run a slice with --limit, or run one playbook directly as before. Nothing here +# changes how any individual playbook behaves. + +# ── Baseline: every managed machine ───────────────────────────────────────── +- import_playbook: infra/01_user_and_access_setup_playbook.yml +- import_playbook: infra/02_firewall_and_fail2ban_playbook.yml +- import_playbook: infra/900_install_rsync.yml +- import_playbook: infra/920_join_headscale_mesh.yml +# Idempotent and kept permanently: guarantees a rebuilt or restored host cannot +# quietly bring the Uptime-Kuma-era monitoring back. +- import_playbook: infra/409_remove_legacy_monitoring.yml + +# ── Monitoring ────────────────────────────────────────────────────────────── +# Gatus first: the three plays below register endpoints with it, and registering +# against a host that is not serving yet would simply fail. +- import_playbook: services/gatus/deploy_gatus_playbook.yml +# The Signal transport for Gatus alerts. Shares a docker network with Gatus and +# publishes no ports - the API has no authentication. Needs a one-time manual +# device link; see roles/signal_api/README.md. +- import_playbook: services/signal-api/deploy_signal_api_playbook.yml +- import_playbook: infra/400_host_monitoring.yml +- import_playbook: infra/401_service_monitoring.yml +- import_playbook: infra/402_public_monitoring.yml +# Registers where the per-service probes report. The probes themselves are +# deployed by each service's own playbook further down; the endpoints must exist +# before the first push arrives. +- import_playbook: infra/403_service_probe_registration.yml +# 910_docker says `hosts: managed`, but only 5 of 11 managed hosts have or need +# Docker. Left out until it has a [docker] group — see the note in PLAN_7. + +# ── The hypervisor ────────────────────────────────────────────────────────── +- import_playbook: infra/nodito/31_proxmox_community_repos_playbook.yml +- import_playbook: infra/nodito/32_zfs_pool_setup_playbook.yml +- import_playbook: infra/nodito/34_nut_ups_setup_playbook.yml + +# ── Reverse proxy, before anything that registers a vhost ─────────────────── +- import_playbook: services/caddy_playbook.yml + +# ── Services ──────────────────────────────────────────────────────────────── +- import_playbook: services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +- import_playbook: services/fulcrum/deploy_fulcrum_playbook.yml +- import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml +- import_playbook: services/mempool/deploy_mempool_playbook.yml +- import_playbook: services/memos/deploy_memos_playbook.yml +- import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml +- import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml +- import_playbook: services/headscale/deploy_headscale_playbook.yml +- import_playbook: services/vaultwarden/deploy_vaultwarden_playbook.yml +- import_playbook: services/forgejo/deploy_forgejo_playbook.yml +- import_playbook: services/lnbits/deploy_lnbits_playbook.yml +- import_playbook: services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +- import_playbook: services/personal-blog/deploy_personal_blog_playbook.yml + +# ── Backups: each source dumps itself, the box pulls ──────────────────────── +- import_playbook: services/headscale/setup_backup_headscale.yml +- import_playbook: services/vaultwarden/setup_backup_vaultwarden.yml +- import_playbook: services/forgejo/setup_backup_forgejo.yml +- import_playbook: services/lnbits/setup_backup_lnbits.yml +- import_playbook: services/memos/setup_backup_memos.yml +- import_playbook: playbooks/backups.yml + +# Deliberately not here. Every playbook in the repo is either imported above or +# listed below, so this file accounts for all of them: +# +# infra/910_docker_playbook.yml says `hosts: managed`, but Docker is on 5 +# of 11 managed hosts and those 5 are exactly +# the ones that need it. Running it would +# install Docker on the Bitcoin node and the +# hypervisor. Needs a [docker] group first. +# +# infra/nodito/30_proxmox_bootstrap one-shot: bare-metal bootstrap, run once +# infra/nodito/33_..._cloud_template one-shot: builds the VM template +# +# +# services/vaultwarden/disable_ deliberate manual actions, not convergence +# vaultwarden_sign_ups_playbook.yml +# services/personal-blog/setup_ +# deploy_alias_lapy.yml diff --git a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml b/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml similarity index 97% rename from ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml rename to archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml index 2d3d22a..97d6c9c 100644 --- a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml +++ b/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml @@ -12,17 +12,13 @@ # What was being monitored: archive/uptime_kuma/MONITORS.md # ═════════════════════════════════════════════════════════════════════════════ - name: Setup ntfy as Uptime Kuma Notification Channel - hosts: monitoring + hosts: observability become: no vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./ntfy_vars.yml vars: ntfy_subdomain: "{{ subdomains.ntfy }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" uptime_kuma_subdomain: "{{ subdomains.uptime_kuma }}" ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" ntfy_server_url: "https://{{ ntfy_domain }}" diff --git a/tofu/nodito/.terraform.lock.hcl b/tofu/nodito/.terraform.lock.hcl new file mode 100644 index 0000000..fadff38 --- /dev/null +++ b/tofu/nodito/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/telmate/proxmox" { + version = "3.0.2-rc05" + constraints = "3.0.2-rc05" + hashes = [ + "h1:QfHovHn8h9uJXdJ+urOuiD7R46OXmdZQiRcCBaV6AD4=", + "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f", + "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc", + "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321", + "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3", + "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98", + "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb", + "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e", + "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9", + "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d", + "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3", + "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e", + "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269", + "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1", + "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123", + ] +} diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md index 3a0b18f..bff6585 100644 --- a/tofu/nodito/README.md +++ b/tofu/nodito/README.md @@ -27,16 +27,17 @@ This directory lets you declare VMs on the `nodito` Proxmox node and apply with - The Ansible template exists: `debian-13-cloud-init` (VMID 9001 by default). ### Provider Auth -Create a `terraform.tfvars` (copy from `terraform.tfvars.example`) and set: +Credentials are the only thing not in git. Copy `terraform.tfvars.example` to +`terraform.tfvars` (gitignored) and set: - `proxmox_api_url` (e.g. `https://nodito:8006/api2/json`) - `proxmox_api_token_id` (e.g. `root@pam!tofu`) - `proxmox_api_token_secret` -- `ssh_authorized_keys` (your public key content) -Alternatively, you can export env vars and reference them in a tfvars file. +Alternatively, export them as `TF_VAR_proxmox_api_token_secret` etc. ### Declare VMs -Edit `terraform.tfvars` and fill the `vms` map. Example entry: +VMs are declared in `vms.auto.tfvars`, which is committed. `*.auto.tfvars` is +loaded automatically, so it needs no `-var-file`. Example entry: ``` vms = { web1 = { @@ -54,15 +55,24 @@ All VM disks are created on `zfs_storage_name` (defaults to `proxmox-tank-1`). N ### Usage ``` tofu init -tofu plan -var-file=terraform.tfvars -tofu apply -var-file=terraform.tfvars +tofu plan +tofu apply ``` +`terraform.tfvars` and `vms.auto.tfvars` are both auto-loaded. > VMs are created once and then protected: the module sets `lifecycle.prevent_destroy = true` and ignores subsequent config changes. After the initial apply, manage day‑2 changes directly in Proxmox (or remove the lifecycle block if you need OpenTofu to own ongoing updates). ### Notes - Clones are full clones by default (`full_clone = true`). - Cloud-init injects `cloud_init_user` and `ssh_authorized_keys`. +- `.terraform.lock.hcl` is committed: it pins the provider hashes, which matters + because `versions.tf` tracks a release candidate (`3.0.2-rc05`). +- State is local (`terraform.tfstate`, gitignored) and has no remote backend, so + it exists only on the machine that last ran `tofu apply`. +- The map is not a complete inventory of nodito: state also manages + `bastion-box` (1100) and `nonkeiwaisi-box` (3300), which are not declared in + `vms.auto.tfvars`. `tofu plan` is clean today, but relaxing the `lifecycle` + block without first declaring them would put them up for destruction. - Disks use `scsi0` on ZFS with `discard` enabled. diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example index cc88b3f..37b6d7a 100644 --- a/tofu/nodito/terraform.tfvars.example +++ b/tofu/nodito/terraform.tfvars.example @@ -1,35 +1,7 @@ -proxmox_api_url = "https://nodito:8006/api2/json" -proxmox_api_token_id = "root@pam!tofu" -proxmox_api_token_secret = "REPLACE_ME" - -proxmox_node = "nodito" -zfs_storage_name = "proxmox-tank-1" -template_name = "debian-13-cloud-init" -cloud_init_user = "counterweight" - -# paste your ~/.ssh/id_ed25519.pub or similar -ssh_authorized_keys = <