diff --git a/.gitignore b/.gitignore index 471cd6f..9ca7b8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,13 @@ # OpenTofu / Terraform .terraform/ .tofu/ +.terraform.lock.hcl +.tofu.lock.hcl terraform.tfstate terraform.tfstate.* crash.log - -# Provider credentials only. Non-secret infra config (the vms map) is committed -# as *.auto.tfvars, and *.lock.hcl is committed on purpose so provider hashes -# are pinned. -terraform.tfvars -terraform.tfvars.json -*secrets.auto.tfvars +*.tfvars +*.tfvars.json venv/* .env diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index f77ae48..0be0162 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -6,6 +6,7 @@ root_domain: contrapeso.xyz # Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the # playbooks are kept deliberately — the check logic is meant to be rewired to # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. +uptime_kuma_enabled: false # age recipient for all backup artefacts age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" @@ -13,54 +14,3 @@ age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr # Public key small-backups-box pulls with # Authorised on each source host for an unprivileged, dedicated user only backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" - - -# ───────────────────────────────────────────────────────────────────────────── -# Subdomains. Global because the edge host proxies for services that live on -# other machines, so no single inventory group covers the readers. Combine with -# root_domain above to build an FQDN. -# -# Moved here from services_config.yml, which 30 plays had to remember to name in -# vars_files: - a file everyone must opt into is a file someone will forget. -# ───────────────────────────────────────────────────────────────────────────── -subdomains: - # Monitoring - gatus: status - - # VPN infrastructure (spacey) - headscale: headscale - - # Core services (vipy) - vaultwarden: vault - forgejo: forgejo - lnbits: wallet - - # Secondary services (vipy) - ntfy_emergency_app: avisame - personal_blog: pablohere - - # Memos (memos-box) - memos: memos - - # Mempool block explorer (mempool-box, proxied via vipy) - mempool: mempool - - # DATUM Gateway dashboard (knots-box, proxied via vipy) - datum_gateway: datum - -# Read by plays across several groups, so global rather than group_vars/. -headscale_namespace: counter-net - -# ───────────────────────────────────────────────────────────────────────────── -# Domains whose registration expiry is monitored (infra/402_public_monitoring). -# -# Registration renewal is a manual act at the registrar, and losing a domain is -# not recoverable in the way losing a host is - so these are checked daily and -# alarm with two weeks of runway. -# -# root_domain is the estate's own domain; the rest are domains we own that are -# served from it or from a host in the inventory. -# ───────────────────────────────────────────────────────────────────────────── -monitored_domains: - - "{{ root_domain }}" - - arbret.com diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index 76eb7bf..d1f18ec 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,164 +1,60 @@ $ANSIBLE_VAULT;1.1;AES256 -34383033613438623432656438313763613639313139626432303637383363643339613166626665 -6537663336353661373437303636613364643736663031300a623461353135623537343763383333 -37323731646262613436646231656263396532616132316237353762376363383062646132303435 -3435316666316235660a323163613932616661303935336633656332353766393536393736356539 -61396164386266303464373262373862363361363365363235383363613335663933313062663565 -39613466353161356462656539613335613536363532333431393935303430373435373737363537 -62383266323339373062313134316464303263313830646161333530383736336637333865373764 -31313165396366653763343466643561383236643836616662383439666165343139326232326364 -32633133373863616163343365656231353939366435346534383462396134653064663566343566 -66343464373865383864313264646635316233336133346365396634626261613561373965643339 -30353831393037316334313632376537393361336366326561343832373531303139323937613531 -34386164303430316162363536386137316330336638303365393462386465633862663932376563 -34323862636435306563333065383134343362333733623066656439613339376666353331373665 -32353938663364313036373237323062393430306661633732356434333232376338313835663330 -32306536393666323464663238306562613936376236613935616664653865373330626365626330 -64613066653632646638306338393331376430383165316363323437373362613866313962663432 -62356663636630356239613831353066303530363966396135376134633936313530643262353965 -38383330613962313935366633353635363638303937316362303866343433353437363334396431 -32393134626133353564643233616632376633303934653065613262353436333533376137303862 -35303861383833326537376433336634633730373833313036643365373431666438653033653930 -33346230396661636333613031353133353631306331666362386366633062666534303539316238 -39366634656333646634643032393738333330636230663366623265666533356534623465316665 -35336535326237343231653661613736626239316262386363666264613636326332366232333964 -38353635616662663133376534306366643037613732343233373336653166316330643638353539 -35353532643534613864333839393365313439343663333337373639633832383630316164306132 -63636162373034363839656638353534613733663737626432636436373164303239316436313566 -66623466396663353136323565643262383865323830616232373466373431373261306537646463 -61353966623062356438626635663461396165343366393132393965383734633632666335313030 -37333631373239303231396264393031373162373936303462363736313538343562343737356332 -63663762633032356130303964386664366137373432353533386566613037613163383963623732 -32343765376535343630656633333362393765666131653632613361363163383038323138333537 -37373333316466316562613439356365316163336366623435666166653835616531653563343664 -30303464663838646333373837363961356465613234636138633162366636633030383337643131 -62636236663066316630396132333934353139383465363034363033656161383663656462313939 -62353738396132343734343363636164323163616237613861643862303433633934656165343263 -35313737373333386335663731643664393630653839396464376639306231653934386336353630 -61393735336662663435613431386361373561363531643232303831363163636139653538393931 -66313965646562616237383439343637623835303065333730613865666638383131616261346463 -31396464306537613135653830653138323539393731626264326335336432666333333735646534 -30616439393464326631633635363466303336613135346231626232313361303666323661616638 -64363937343231363536353034363966326132333734386638613737636130646232363666643633 -61353563636466356630346232613761636432303430336461663434333636623962643764336639 -35303565613861323431393036303534303061666437366538306236373930313439353330313632 -61363363303562316664303065613863613339626632643931386438326330373938613762303334 -33316430363931393262373661623137633835656136313235613666613932313236343966306331 -36343031303632323432366337633637336335343638393564313738386162386164613339343662 -62383939663265373265613932633265626263623939303638383838646531343433393864363235 -37313761396436623833376435393537376136373162393465343764326533666533393061643965 -61306365346562333737313962633764313232323161623861336566343735353737633539656162 -34323939306661373662663331653333346264323930663636633134666532623438323537383165 -66343033363962633766303331623866306462623235373838616565653066646132633034363535 -31613462356361613535383963643662373162363363653334393937313039666266366363653537 -64356166396139393262313565303731346534646462333638316661383139346362333364613466 -38303561316263636136303431393366343936653161336238613439366266346136636465303337 -61373634376464333037303862616335623031656133636165636264386265643261373735663266 -39396265366138363430343035356337303438346165366361316230656239326633653537626339 -39613334316563376133343336343563363564653237633764393532336437303334363830396635 -31333963666462313830333163646465356337643263636462356363613266323630393866626535 -35336436663063333364353663616564626535396133386461326433323936323161613633353539 -66323935626433353162666335613839346561343264323763303034613366663233613037343538 -66376236373462383430313030373235373337653338333033306431616562353435656433356365 -37653063363435366532306334373438306632306435653334396630343863623938666335656164 -37386634366430383762316435396237313362313965353634613266616532633465646464313662 -35303433363236393465396161346431306231666432666531626136623966663864376663373562 -37653239633732346536323265643434616136326561666330633435383264333937353366653736 -61353430633363323263323163646639663330663632343038623964396433626337353232373334 -32343938356539303938666137323831646562393233323733336464316165623232326132316362 -61313631626234336164656533616231393663363364623130313439623466396231383537656562 -35393163323131383838356130643863633736613635306635306135666261623731613239303962 -61373432336638353339646665653561653738353731373032366361313562306466373365653333 -35306236336331333961613864346165613763383266613236343066623432303766333434333331 -66376263363132616132366165393235626336323665373135653462653533376138346632393363 -35396165383966326635333134663138366136336430393532373935373264616530613762386430 -37643862316232613261343430616438333831623835663239656531356666613032653837373531 -30386232643537306530323936386561646265616339376265353763663833623539653831656634 -63626564633964343663656435636635656562666236386639353332356566316631643431353330 -32643238623932643732353332633535643738663066383830376432346463393464333236386163 -64376165333266323166613564313061643832656239333165393035343836376434626162643061 -31373861393534636236383765313834356638613332626666653762343537363339383939666263 -61366230623930643534353565316165656132396664393138326139613536343738396330653735 -37623463343834626133313231393130646134363331393238663065643930383237326439343831 -32393536613361616637373761393361303936363863333838666633343063386235316237323531 -62633362316436623761616366383336613765373362393161303765613464643632666462343565 -63616430396164346263666436336463623638313130656231643131336663336565616135373433 -39326533643733333564373434333738343634383033353036646534646632383730613161393130 -62653836376435663435353739373536366466646332653030653332353761626466353631663163 -38613734383634663662376464653733306564313761336566336336656366653536643834653631 -35333963306562393234623039613338643266333762333236366331333431343830393864623830 -34323439386133613035363861623332663933313734313637623739613163663730316134303434 -35363239363836666463366334356435666632396132306163303037663430383535393934323537 -30353266373864346662333035393761646632313032383738346439343232663666343833333565 -35363564623038376666363436663830356165323234653230636264303066666230643632356237 -66633734346633366533623966393462313438333965623033343337323135353730613561363236 -30636333613034626162393137323762306432663761393766303163393536656338366664643432 -34636630323965396166356538623165663335333739323034333237306531393734306138343436 -36613937373932633234303336373530663533646635396664343935326334333239646534343336 -35396666353938326337646431353965653836323731313861613031356561376262363965623934 -65336164623134366365303062393161353965363937646363396564313138323938656330333964 -64663862363766613630333531623463616331363533343962653164376665303463336134346662 -66376238663632663932656230323638323135653666363736643065336330636236323064386361 -37646631303138616565383665663634333338613336346438366333636165366635313337373635 -62656236653961393061373262326534393339363236353336613339656462336661663739396261 -35363636653237316436656662633462306333313336313865333037636661383731336466666536 -36646537653431666261353232653233653930346538383438633466346636656432383963333765 -62346363633265306665366630383831303434336139343837616361666539656262633735363064 -30376138386531336430613563366666333964663231363465326230316233323532366632663930 -63653164373931353861653833326130643365373530663039363166366331303134366336613438 -63633336363539636265396331653630633162316366316130326531363463616464376635376538 -32376638353337343639633434613663666437313564333264383766623561393933333561663235 -37666163626538353736383435373530663438363636643665626266316338336532623235353337 -61653762383630616266353438386137653137616532633066623165383663646437343565366237 -33333463613539346136613135386466656135616136663036313661323738363066633038656139 -38353230313861666463383737303038613731316334333031643438343463666164633536313031 -39613663376436643165333362636462303230656336333234363336633262336363643736336231 -66316630613762383832613836333263303038306235663662323162633932653733626130356665 -65316630346534356261373961353130393334396365376237353439326261336339393336303030 -63323439666439313831626639323735653730356565643938376564316337613333616664643865 -30633530383865663731383635366566613565393632336337306334663937396135303539363536 -65656331386532306438383031313164626237393762303636663831666239336637666465633630 -63383961336333663537666561376532376631343132363638633364616531306363623432396332 -31343864626335613131653762363338393738343531626330663663376531336230656365363661 -64363530333965386439376161313534333435393966656231613161633034303964653564363436 -63396131623562303962383137666439373132633330363237353031306561306362666561376435 -66383535333233663736363762323662363264393365643135323834366565653932313863353437 -38363536323739336165356561613335653464363637636234313637383066633865353239393961 -31613532373361383062393238303430353662383930373764373237383035633364323661623661 -62653661623234366666616532663563316234663137616338393036333937633963323635366165 -61613032363531666338353132346365323661626265626461396362646338373536306362303338 -31646564343434633433663039366533656665643235336136333134366231376336343465336539 -37363139346239666232313366613432646265626564313838346364306133306434313337626562 -32346430316532623037633838303363346130623636386134313566383333613565316135373264 -38393261313435323433326130303032393538333963646430646366653363613830613534353832 -32643434613964613239363966376138376661636335373130356430336631333461623734383535 -35356461326136636234663738343738303064386536336462303632303461303733333666383036 -38383563393739343134363038623638393766316533373163336439336238613734656132623163 -32626564666430663339316163393934323133306238353562323866633738363737643937376435 -66306430623334643766623564363239346361393666663766306637313265313833396435626234 -35366239616132623863343365663833363934316362636638303536656631633364646235366566 -39386637613735373761643339396132323031323438316633363464336636316534643435323861 -62663732366430646538626663313035616235643537643234356434323635373962343633336266 -30323031306438656537626430356231393466313334633934623632663661306361313330633339 -33396234386536336331616161363132343765306562303932313963623037366633383765663134 -62636466636433666530363930633765626531613539363832313361666565323635326334326533 -36336439363836663033643262336139613437623633616138656564393032393263393132303031 -31626336663039633362366663306662333432643261613464303939326562653562346239383261 -37373966383635626433363831653936333964626262363839383936356634633233336365623765 -62313031353538636464303234383865613932323164386336316362323731303263346637346534 -32353734343130343364646161643431353230336364366330363261326334613936633234646264 -39623163346663613431323630303034383761663835373565663166366239303130386139396236 -62393565353761636636366462353038366664663430616332326465373364396264323064666536 -61653035633562316634336263636332363733663666626131383161656535383133613939623034 -66363632613733306336323165336534346337386163656631343332323636353539353737316131 -34616336343865656565646338653037333838663736333330376330663834373138363739633064 -61646439636365323838376131663266636235333062616532613936616339633661303634323531 -63356335386534353639323566376134373565353137333134363761666532366431633634333430 -39326662633765626230326130386333393463323433366162363432613234336634663439313339 -33653737333830336264343839396563316462613032376335356634383436353962613636333866 -32333032623635323430356636623138366635386533646133626164316438393937626462313239 -31636635363064363763396566306234363965346438653738333961623435303233396634643763 -65343763386464366636383734336335646464306262623363303934393734333666636635356365 -65353931303862343765343665373139376530656263356365366333323135383838383030613166 -30303963383833643937636539656164336537383337346436303534303035313935323633663765 -333464336235373230316337333632326138 +61356165613635386631393135656434646436303665313031346566323336313138353433316463 +3363323534613064643132663335623238366431393062340a346538396662306537663163623366 +38626166383933616331623231373137306562623637313263333237633661663436666266616433 +3862346438643638650a306634333535653633613534646630386131333236366538333765323333 +38656163303837303732663561373232393132343331663164656262393730326434373731333636 +36613538646431396536363936336562616431656665653965373864633366663836353434626434 +34383932373461333564303439623565383661646365386665393831383463663662356536356236 +36373964666236626465366161636135393734356536633466383262326537343833636630343738 +62353066316131613737373162643363653662656261363465386364323962656537373061373032 +33343763353464383438363438343965653532393831343930393562633630383932653862623637 +38386239353237356631646436356166373961333464396639383538383662326534343339313330 +65373339356364636634616532363832386631323062363530313861336238353261353334306235 +62616338396431316537346638656365356564346666366366343638356261623664393263323937 +30343261363562383332323462336435376664386134646562643836363834313237373631353731 +35653535663864643266313332356635363262363533663232656531373130336539633066376139 +32316665393831663035623962656364363831333563366135636164346335383738363336663566 +63356532643563393939383635386462663561386434323939303431653438653131363538383034 +33623933333464363032656636643033326162626163353633343062633966343332383138363963 +36383831663562616533316436366566323061386535343538393861383462333166343562316633 +30623665623035393537393965626363323132656433313339396233356666346634316332616336 +37363635363330323230373332326565343530653335383437373230366563366237633665626331 +66623336626230663361636439316337393865383035326136653264666438666566646132353036 +38616264313833316536623238633339373466613866626366383835656638623863323838653030 +63303938376164653966356435386333363731656666313234663535666165646233313137343563 +36386437393139656438333262383437656666343831313239323961373637653163643664356565 +62346431393133656530316262303763646165643336396661666431383436323562336137653031 +62626538373839613734396366653065306534636630346338316237616161613037616364356431 +38643965376133616161336633383664326230383435363334353137303162663738313331346238 +39356161616533616134356231323530306338333162343363353531303263636632613036386638 +63633136386232306234323936303563646466313935326631396565383432386130656638616266 +32383334363237336539396665336366643764633131643663643137376438323666326435626461 +64346633636431393137633537306431646564386565303933636434386462346630626537346438 +32623333666133303061646564366366326665363163396262633164323631636337346130303239 +38373936663337356134666132303165393365663763396362623434633737373538653566646134 +62316164396438303532616266313062326666633130656338653139376634306664633031333037 +31636166306565353334633435656233336233363664306264626237623366336161303134353433 +63383739666462623336386537346662633666626466393039653439346436653937633537396436 +35393339383066326630353066623132333034656539363561346462626265363263303535343961 +39316461616630326539613731303039613736393633373338646266323938326162373831346336 +30656130343463366534323030646238313465306266383034623065636665623366333063383736 +38373063393837306462303564643962373334343139626338623935336435643730646532633630 +66363730386636633639346463363365343239373265303738353732653633653437636130636664 +39386365353334653765343335303263363461313965383664326563333734626533376436626530 +63353163323637303730353564383733653365613635353764333266393532653663326533646132 +39313763373735323835626437306435373238653432393936643165663663656665316132653330 +35383038353532656434366336346235363563636264303734633138323963396562306232646236 +38366536306561653937336333373434336164663336613839353439356435333833396363636437 +38323934613735643363656233333037336465336564313966623063376566663030303230323262 +65643564666534326234306164343365383632333061316238623565353538313538396364313337 +35336230313339643736653238386231343661623337306236383665356632366236356335323530 +66373831616239376231636361333430343433303233393066323865663434643433303832373262 +36366534646161323130393931626362626139663139643263366639656531613436313533363130 +65643736363833333939613566663339623964333262643863333030623138633464386238613934 +31323639333234336264313663376465323737353766643839303665313737336534386665363034 +30326439666433373232306136306365343764643434306561343339353132346430646436343362 +32333765616262363930353435616563333736313533653339656231316230346166363335356638 +37333561366339613136306438306130343230663732333862663838396463623661303961336433 +65646332373139303462303633346432366530643130366133363937653739653036366136373434 +343864373734666431326239373866313734 diff --git a/ansible/host_vars/forgejo_runner_local/main.yml b/ansible/host_vars/forgejo_runner_local/main.yml deleted file mode 100644 index 6e852ce..0000000 --- a/ansible/host_vars/forgejo_runner_local/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - forgejo-runner diff --git a/ansible/host_vars/fulcrum_box_local/main.yml b/ansible/host_vars/fulcrum_box_local/main.yml deleted file mode 100644 index aa189b0..0000000 --- a/ansible/host_vars/fulcrum_box_local/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -# fulcrum-box: the Electrum server. -# -# Read by the fulcrum role here and by the socket-proxy play on the edge host, -# which publishes the port. See host_vars/knots_box_local/main.yml for why this -# lives in host_vars rather than in the role's defaults. -fulcrum_ssl_port: 50002 - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - fulcrum diff --git a/ansible/host_vars/knots_box_local/main.yml b/ansible/host_vars/knots_box_local/main.yml deleted file mode 100644 index 3866541..0000000 --- a/ansible/host_vars/knots_box_local/main.yml +++ /dev/null @@ -1,26 +0,0 @@ -# knots-box: Bitcoin Knots and the DATUM Gateway. -# -# These ports are read twice: by the role that deploys the service here, and by -# the socket-proxy / Caddy plays that run on the EDGE host and publish them. -# A role default is invisible to that second play, which is why these live in -# host_vars rather than roles//defaults/ - the edge play reads them as -# hostvars['knots_box_local']., and the role picks them up automatically -# because host_vars outranks role defaults. -# -# They used to live in services_config.yml, a file 30 plays had to remember to -# name in vars_files: and that four role defaults silently depended on. -bitcoin_p2p_port: 8333 -datum_gateway_api_port: 7152 -datum_gateway_stratum_port: 23334 - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - bitcoind - - datum-gateway diff --git a/ansible/host_vars/memos_box_local/main.yml b/ansible/host_vars/memos_box_local/main.yml deleted file mode 100644 index f1fc19b..0000000 --- a/ansible/host_vars/memos_box_local/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - memos diff --git a/ansible/host_vars/mempool_box_local/main.yml b/ansible/host_vars/mempool_box_local/main.yml deleted file mode 100644 index f46a1a5..0000000 --- a/ansible/host_vars/mempool_box_local/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -# mempool-box: the Mempool block explorer. -# -# Read by the mempool role here and by the Caddy play on the edge host, which -# proxies to it. See host_vars/knots_box_local/main.yml for why this lives in -# host_vars rather than in the role's defaults. -mempool_frontend_port: 8080 diff --git a/ansible/host_vars/monitoring/main.yml b/ansible/host_vars/monitoring/main.yml deleted file mode 100644 index 2ef805d..0000000 --- a/ansible/host_vars/monitoring/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - caddy diff --git a/ansible/host_vars/nodito/main.yml b/ansible/host_vars/nodito/main.yml index c8176af..c0002f3 100644 --- a/ansible/host_vars/nodito/main.yml +++ b/ansible/host_vars/nodito/main.yml @@ -14,12 +14,7 @@ systemd_service_name: nodito-cpu-temp-monitor # ZFS Pool Configuration zfs_pool_name: "proxmox-tank-1" -# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the -# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently -# replaced and the repo never caught up. Pool creation is guarded by -# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been -# wrong on any disaster-recovery run. -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ" # First disk for RAID 1 mirror +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" @@ -31,15 +26,3 @@ ups_port: auto ups_user: counterweight ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - nut-server - - nut-monitor diff --git a/ansible/host_vars/nodito/vault.yml b/ansible/host_vars/nodito/vault.yml deleted file mode 100644 index e64085e..0000000 --- a/ansible/host_vars/nodito/vault.yml +++ /dev/null @@ -1,11 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -30333035323663393939343061323234336164396465623665346165393534646366333332376463 -3364373463333664363334373964323838336531353364310a636636373539623464336630666164 -61376532616339376562373238383436306664313564663266303534346461666466383965323538 -3163313239626663310a613033336332653165333537313366636361663036383031376561613761 -31313563373062333033323037653939663762343161656264633436343361663737626366663732 -39366666626338323436383134646263643538333564313566346336323563663534653161396136 -39333565393538366238643563323630346166643461643063393631643665363566623631373762 -36643866646637306231653837363838656163613766636265383139333838396535626335343163 -32613935353330636263616333666230323436663935326133636362343836323535623237646235 -6266316363366335323162663039366137633865396237373632 diff --git a/ansible/host_vars/spacey/main.yml b/ansible/host_vars/spacey/main.yml deleted file mode 100644 index a50d116..0000000 --- a/ansible/host_vars/spacey/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - headscale - - caddy diff --git a/ansible/host_vars/vipy/main.yml b/ansible/host_vars/vipy/main.yml deleted file mode 100644 index 3b077df..0000000 --- a/ansible/host_vars/vipy/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - forgejo - - lnbits - - caddy - - phoenixd diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index 0e2c914..c6eed18 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,5 +1,7 @@ - name: Secure Debian hosts: managed + vars_files: + - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index 309db56..9f37c70 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,5 +1,7 @@ - name: Secure Debian hosts: managed + vars_files: + - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/400_host_monitoring.yml b/ansible/infra/400_host_monitoring.yml deleted file mode 100644 index 3f6b119..0000000 --- a/ansible/infra/400_host_monitoring.yml +++ /dev/null @@ -1,216 +0,0 @@ ---- -# Host-level monitoring for the whole estate, reported to Gatus. -# -# Every check here PUSHES. Gatus never reaches out, which matters because nodito -# and its VMs sit behind NAT, and because four of the five checks are internal -# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool -# health and UPS mains status cannot be observed from outside the machine. -# -# Liveness is a push too, and that is a choice rather than a limitation. A -# heartbeat proves the host is running AND can reach the internet; an ICMP probe -# from one vantage point only proves it answers pings from there. And because -# Gatus alerts when a heartbeat window expires, a check that stops running -# raises the alarm by itself - a dead timer looks exactly like a dead host, -# which is the correct reading. -# -# Each host has ONE bearer token, shared across its own checks: a token can only -# write results for that host's endpoints, so a compromised host can lie about -# itself, which it could do anyway. -# -# The push URL must use Gatus's own key format (config/key/key.go): -# key = sanitize(group) + "_" + sanitize(name) -# where sanitize lowercases and replaces / _ . , space # + & with "-". So -# knots_box_local becomes knots-box-local in the URL but stays readable in the -# name. host_key below is the Jinja equivalent; do not hand-write these. - -# ───────────────────────────────────────────────────────────────────────────── -# Register everything with Gatus. -# -# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the -# host plays below take minutes, so every endpoint exists before its first push -# arrives. Registering afterwards would 404 every first report. -# -# Heartbeat windows are several times the check interval, so one missed run - a -# slow apt run, a reboot - does not raise an alarm, but a check that has -# genuinely stopped does. -# ───────────────────────────────────────────────────────────────────────────── -# ───────────────────────────────────────────────────────────────────────────── -# Alerting thresholds, and why they differ by check type. -# -# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a -# different amount of wall-clock time per check: -# -# push/heartbeat endpoints a failure is produced once per heartbeat window -# pulled endpoints a failure is produced once per interval -# -# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day -# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m -# window on a 5-minute push is precisely "one missed push forgiven"; stacking a -# threshold of 3 on top triples a tolerance that was already chosen. -# -# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints -# have no built-in tolerance, so the threshold is where it belongs for them. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the host checks with Gatus - hosts: observability - become: yes - vars: - monitored: "{{ groups['managed'] | sort }}" - - tasks: - - name: Build the liveness endpoint list - ansible.builtin.set_fact: - liveness_endpoints: "{{ liveness_endpoints | default([]) + [{ - 'name': item, - 'group': 'liveness', - 'token': gatus_push_tokens[item], - 'heartbeat': '11m'}] }}" - loop: "{{ monitored }}" - - - name: Build the disk endpoint list - ansible.builtin.set_fact: - disk_endpoints: "{{ disk_endpoints | default([]) + [{ - 'name': item, - 'group': 'disk', - 'token': gatus_push_tokens[item], - 'heartbeat': '7h'}] }}" - loop: "{{ monitored }}" - - - name: Register liveness endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: liveness - gatus_endpoint_external: "{{ liveness_endpoints }}" - - - name: Register disk endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: disk - gatus_endpoint_external: "{{ disk_endpoints }}" - - - name: Register the hypervisor endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: hypervisor - gatus_endpoint_external: - - name: cpu - group: hypervisor - token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "11m" - - name: zfs - group: hypervisor - token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "7h" - - name: ups - group: hypervisor - token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "11m" - -- name: Deploy host liveness and disk checks - hosts: managed - become: yes - vars: - gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" - host_token: "{{ gatus_push_tokens[inventory_hostname] }}" - - tasks: - - name: Is the host up? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: liveness - healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}" - healthcheck_check: liveness - healthcheck_interval: "5min" - healthcheck_boot_delay: "1min" - healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external" - healthcheck_push_token: "{{ host_token }}" - - - name: Is the disk packed? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: disk-usage - healthcheck_description: "Disk usage for {{ inventory_hostname }}" - healthcheck_check: disk-usage - # Every 6h rather than daily. Disk usage itself moves slowly, but the - # heartbeat can only be as tight as the push frequency - a daily push - # forces a >24h window, and a stuck check then hides for a day and a - # half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the - # hosts so twelve boxes do not all report in the same second. - healthcheck_on_calendar: "*-*-* 00/6:00:00" - healthcheck_randomized_delay: "900" - healthcheck_boot_delay: "5min" - healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external" - healthcheck_push_token: "{{ host_token }}" - -- name: Deploy the hypervisor-only checks - hosts: hypervisor - become: yes - vars: - gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - host_token: "{{ gatus_push_tokens[inventory_hostname] }}" - - tasks: - - name: Is the CPU hot? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: cpu-temp - healthcheck_description: "CPU temperature for {{ inventory_hostname }}" - healthcheck_check: cpu-temp - healthcheck_packages: [curl, lm-sensors] - healthcheck_interval: "5min" - healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external" - healthcheck_push_token: "{{ host_token }}" - - - name: Is ZFS broken? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: zfs-health - healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}" - healthcheck_check: zfs-health - healthcheck_packages: [curl, jq] - healthcheck_zfs_pool: "{{ zfs_pool_name }}" - healthcheck_on_calendar: "*-*-* 00/6:20:00" - healthcheck_boot_delay: "10min" - healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external" - healthcheck_push_token: "{{ host_token }}" - - - name: Is the UPS online? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: ups-status - healthcheck_description: "UPS mains status for {{ ups_name }}" - healthcheck_check: ups-status - healthcheck_ups_name: "{{ ups_name }}" - healthcheck_interval: "5min" - healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external" - healthcheck_push_token: "{{ host_token }}" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml deleted file mode 100644 index ab564a1..0000000 --- a/ansible/infra/401_service_monitoring.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -# Is each systemd-deployed service actually running? -# -# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before -# it alarms, so a reboot or a slow check does not page anyone, but a host that -# stops reporting does. -# -# This closes the gap that let a real bug run unnoticed: a backup script left -# forgejo, lnbits, headscale and memos stopped, and NOTHING caught it. The dumps -# exited 0, the artefacts were correct, the deploy said failed=0, and liveness -# only proves the HOST is up - not that anything on it is serving. -# -# One endpoint PER UNIT, not per host. A host running four services needs four -# endpoints, or a single red light says "something on vipy is down" without -# saying which - and that is the question you actually have at 3am. But only ONE -# timer per host: the check iterates that host's units and pushes a result for -# each, the same way check-backups.sh reports per source. Four units on vipy -# would otherwise mean four scripts, four services and four timers. -# -# Which units each host runs is in host_vars//main.yml as -# monitored_services, because "what runs here" is a property of the machine. -# -# Keys are host-qualified because unit names collide - caddy runs on four -# machines. Gatus computes sanitize(group)_sanitize(name), so group "services" -# and name "vipy/caddy" give services_vipy-caddy. - -# ───────────────────────────────────────────────────────────────────────────── -# Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the -# host play above takes minutes, so every endpoint exists before its first push. -# ───────────────────────────────────────────────────────────────────────────── -# ───────────────────────────────────────────────────────────────────────────── -# Alerting thresholds, and why they differ by check type. -# -# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a -# different amount of wall-clock time per check: -# -# push/heartbeat endpoints a failure is produced once per heartbeat window -# pulled endpoints a failure is produced once per interval -# -# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day -# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m -# window on a 5-minute push is precisely "one missed push forgiven"; stacking a -# threshold of 3 on top triples a tolerance that was already chosen. -# -# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints -# have no built-in tolerance, so the threshold is where it belongs for them. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the service checks with Gatus - hosts: observability - become: yes - - tasks: - # Two plain steps rather than one clever expression: first collect which - # units each host declares, then flatten that into endpoints. - - name: Collect the units each host declares - ansible.builtin.set_fact: - host_units: "{{ host_units | default([]) + [{'host': item, 'units': hostvars[item].monitored_services}] }}" - loop: "{{ groups['managed'] | sort }}" - when: hostvars[item].monitored_services | default([]) | length > 0 - - - name: Build one endpoint per unit - ansible.builtin.set_fact: - service_endpoints: "{{ service_endpoints | default([]) + [{ - 'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1, - 'group': 'services', - 'token': gatus_push_tokens[item.0.host], - 'heartbeat': '11m'}] }}" - loop: "{{ host_units | subelements('units') }}" - - - name: Register the service endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: services - gatus_endpoint_external: "{{ service_endpoints }}" - -- name: Monitor systemd services on every host that has them - hosts: managed - become: yes - vars: - gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" - - tasks: - - name: Is every deployed service running? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: service-health - healthcheck_description: "systemd services on {{ inventory_hostname }}" - healthcheck_check: systemd-units - healthcheck_units: "{{ monitored_services }}" - healthcheck_units_key_prefix: "services_{{ host_key }}" - healthcheck_interval: "5min" - healthcheck_boot_delay: "2min" - # The per-unit results go to keys under this collection; the role's own - # single-result push is unused here, so only the base is set. - healthcheck_push_base: "{{ gatus_api }}" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - when: monitored_services | default([]) | length > 0 diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml deleted file mode 100644 index d924468..0000000 --- a/ansible/infra/402_public_monitoring.yml +++ /dev/null @@ -1,141 +0,0 @@ ---- -# Domain expiry, DNS correctness, and public endpoint reachability. -# -# These are the first checks in the estate that PULL rather than push, and that -# is the right way round for them: all three are about how the outside world -# sees us, so they must be measured from outside. Gatus polls from the -# observability host and needs nothing installed anywhere else - there is no -# script, no timer and no token, because nothing is reporting in. -# -# That also means these have no heartbeat. A heartbeat answers "did the thing -# that was supposed to report in do so"; when Gatus does the checking itself, -# failure is immediate and self-evident. - -- name: Register the public-facing checks with Gatus - hosts: observability - become: yes - - vars: - # Expected A records, derived from inventory rather than written down again. - # The estate's recurring bug is an address recorded in a second place and - # then left behind when the machine moved, so the check asserts against - # ansible_host - if a box is renumbered, inventory is the one edit. - dns_records: - - {sub: "{{ subdomains.gatus }}", host: monitoring} - - {sub: "{{ subdomains.headscale }}", host: spacey} - - {sub: "{{ subdomains.vaultwarden }}", host: vipy} - - {sub: "{{ subdomains.forgejo }}", host: vipy} - - {sub: "{{ subdomains.lnbits }}", host: vipy} - - {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy} - - {sub: "{{ subdomains.personal_blog }}", host: vipy} - - {sub: "{{ subdomains.memos }}", host: vipy} - - {sub: "{{ subdomains.mempool }}", host: vipy} - - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - - # A public resolver on purpose: this must test what the internet sees, not - # what a local cache or the tailnet's MagicDNS happens to answer. - dns_resolver: "1.1.1.1" - - # Expected status per site, checked live before being written down. - # 401 is the CORRECT answer for the two behind basic auth - asserting 200 - # there would go green precisely when the auth broke. - public_sites: - - {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401} - - {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200} - - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200} - - {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200} - - {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200} - - {name: avisame, sub: "{{ subdomains.ntfy_emergency_app }}", path: "/", status: 200} - - {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200} - - {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200} - - {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200} - - {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401} - - # Ports published from the edge host by socket_proxy. - public_tcp: - - {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"} - - {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"} - - {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"} - - tasks: - # ── Domain expiry ──────────────────────────────────────────────────────── - # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the - # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole - # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so - # whatever the apex serves - a real site, or the registrar's parking page - - # is irrelevant. - # - # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway - # because it uses a free whois service that must not be hammered. - # 336h = 14 days of runway, because renewal is a manual act at the registrar. - - name: Build the domain endpoints - ansible.builtin.set_fact: - domain_endpoints: "{{ domain_endpoints | default([]) + [{ - 'name': item, - 'group': 'domain', - 'url': 'https://' ~ item, - 'interval': '24h', - 'conditions': ['[DOMAIN_EXPIRATION] > 336h'], - 'alerts': [{'type': 'signal', 'failure-threshold': 1, - 'success-threshold': 1, 'send-on-resolved': true, - 'minimum-reminder-interval': '168h'}]}] }}" - loop: "{{ monitored_domains }}" - - # ── DNS ────────────────────────────────────────────────────────────────── - # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The - # domain check stays at 24h because it does a WHOIS/RDAP lookup against a - # free service. Alert on the first failure - at a 6h interval, waiting for - # three would be nearly a day. - - name: Build the DNS endpoints - ansible.builtin.set_fact: - dns_endpoints: "{{ dns_endpoints | default([]) + [{ - 'name': item.sub ~ '.' ~ root_domain, - 'group': 'dns', - 'url': dns_resolver, - 'interval': '6h', - 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, - 'conditions': ['[DNS_RCODE] == NOERROR', - '[BODY] == ' ~ hostvars[item.host].ansible_host], - 'alerts': [{'type': 'signal', 'failure-threshold': 1, - 'success-threshold': 1, 'send-on-resolved': true, - 'minimum-reminder-interval': '24h'}]}] }}" - loop: "{{ dns_records }}" - - # ── Public HTTP ────────────────────────────────────────────────────────── - # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has - # no heartbeat window, so unlike the push checks the tolerance has to live - # in the threshold - and one failed poll of a public site is usually a blip. - - name: Build the public HTTP endpoints - ansible.builtin.set_fact: - http_endpoints: "{{ http_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'public', - 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, - 'interval': '5m', - 'conditions': ['[STATUS] == ' ~ item.status, - '[CERTIFICATE_EXPIRATION] > 168h'], - 'alerts': [{'type': 'signal', 'failure-threshold': 3, - 'success-threshold': 2, 'send-on-resolved': true, - 'minimum-reminder-interval': '6h'}]}] }}" - loop: "{{ public_sites }}" - - # ── Public TCP ─────────────────────────────────────────────────────────── - - name: Build the public TCP endpoints - ansible.builtin.set_fact: - tcp_endpoints: "{{ tcp_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'public', - 'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port, - 'interval': '5m', - 'conditions': ['[CONNECTED] == true'], - 'alerts': [{'type': 'signal', 'failure-threshold': 3, - 'success-threshold': 2, 'send-on-resolved': true, - 'minimum-reminder-interval': '6h'}]}] }}" - loop: "{{ public_tcp }}" - - - name: Register the public-facing endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_name: public - gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}" diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml deleted file mode 100644 index c92b767..0000000 --- a/ansible/infra/403_service_probe_registration.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -# The per-service health probes. -# -# These are NOT the same thing as the systemd checks in infra/401. Those answer -# "is the unit running"; these answer "does the service actually respond" - an -# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch -# from the Mempool backend. A process can be perfectly `active` and useless, -# which is precisely the gap these close. -# -# The checks themselves live in each service's own role, deployed by that -# service's playbook. This play only registers where they report, because the -# endpoints must exist in Gatus before the first push arrives. -# -# They used to push to Uptime Kuma. The scripts now POST with a bearer token -# instead of GETting ?status=up, and each host uses its own token. - -# ───────────────────────────────────────────────────────────────────────────── -# Alerting thresholds, and why they differ by check type. -# -# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a -# different amount of wall-clock time per check: -# -# push/heartbeat endpoints a failure is produced once per heartbeat window -# pulled endpoints a failure is produced once per interval -# -# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day -# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m -# window on a 5-minute push is precisely "one missed push forgiven"; stacking a -# threshold of 3 on top triples a tolerance that was already chosen. -# -# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints -# have no built-in tolerance, so the threshold is where it belongs for them. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the per-service probes with Gatus - hosts: observability - become: yes - - vars: - probes: - - {name: bitcoin-knots, host: knots_box_local} - - {name: datum-gateway, host: knots_box_local} - - {name: fulcrum, host: fulcrum_box_local} - - {name: phoenixd, host: vipy} - - {name: forgejo-runner, host: forgejo_runner_local} - - {name: mempool-mariadb, host: mempool_box_local} - - {name: mempool-backend, host: mempool_box_local} - - {name: mempool-frontend, host: mempool_box_local} - - tasks: - - name: Build the probe endpoint list - ansible.builtin.set_fact: - probe_endpoints: "{{ probe_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'probe', - 'token': gatus_push_tokens[item.host], - 'heartbeat': '11m'}] }}" - loop: "{{ probes }}" - - - name: Register the probe endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: probes - gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/infra/409_remove_legacy_monitoring.yml b/ansible/infra/409_remove_legacy_monitoring.yml deleted file mode 100644 index 8f57e46..0000000 --- a/ansible/infra/409_remove_legacy_monitoring.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -# Remove the Uptime-Kuma-era monitoring that 400/401/402 replaced. -# -# Deleting the playbooks that installed these is NOT enough: the units are on -# the hosts, enabled, and keep firing regardless of what the repo says. Two of -# them still push to https://uptime.contrapeso.xyz every 15 minutes. A playbook -# that is deleted without a cleanup leaves its output running forever, with -# nothing in the repo left to explain it. -# -# What replaced what, all verified against the deployed scripts before removal: -# -# disk-usage-monitor -> disk-usage-healthcheck (infra/400) -# The old one checked ONLY "/" at 80%. The replacement walks every real -# filesystem, excluding tmpfs/devtmpfs/squashfs/overlay, at 85%. Strictly -# more coverage, so nothing is lost. -# -# system-healthcheck -> liveness-healthcheck (infra/400) -# The old script computed uptime and pushed. That is exactly a liveness -# heartbeat and nothing more. -# -# nodito-cpu-temp-monitor -> cpu-temp-healthcheck (infra/400) -# zfs-health-monitor -> zfs-health-healthcheck (infra/400) -# The ZFS check logic was ported verbatim - same five conditions - so only -# the reporting transport changed. -# -# NOT removed, because they are not monitoring: -# zfs-monthly-scrub.{timer,service} the actual scrub (infra/nodito/32) -# pull-backups, check-backups the backup machinery (playbooks/backups) -# -# This play is idempotent and kept permanently rather than run once and deleted: -# on a host that never had these it does nothing, and it guarantees a rebuilt or -# restored machine cannot quietly bring them back. - -- name: Remove the legacy Uptime Kuma monitoring units - hosts: managed - become: yes - - vars: - legacy_units: - - disk-usage-monitor - - system-healthcheck - - nodito-cpu-temp-monitor - - zfs-health-monitor - legacy_dirs: - - /opt/disk-monitoring - - /opt/system-healthcheck - - /opt/nodito-monitoring - - /opt/zfs-monitoring - - tasks: - - name: Find which legacy units exist here - ansible.builtin.stat: - path: "/etc/systemd/system/{{ item.0 }}.{{ item.1 }}" - register: legacy_unit_files - loop: "{{ legacy_units | product(['timer', 'service']) | list }}" - - # Stop and disable BEFORE deleting the unit file: systemd cannot disable a - # unit whose file has already gone, which would leave a dangling symlink in - # multi-user.target.wants and a warning on every daemon-reload. - - name: Stop and disable the legacy units - ansible.builtin.systemd: - name: "{{ item.item.0 }}.{{ item.item.1 }}" - state: stopped - enabled: no - loop: "{{ legacy_unit_files.results }}" - loop_control: - label: "{{ item.item.0 }}.{{ item.item.1 }}" - when: item.stat.exists - failed_when: false - - - name: Remove the legacy unit files - ansible.builtin.file: - path: "/etc/systemd/system/{{ item.item.0 }}.{{ item.item.1 }}" - state: absent - loop: "{{ legacy_unit_files.results }}" - loop_control: - label: "{{ item.item.0 }}.{{ item.item.1 }}" - when: item.stat.exists - - - name: Reload systemd - ansible.builtin.systemd: - daemon_reload: yes - - - name: Remove the legacy monitoring scripts and their logs - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: "{{ legacy_dirs }}" - - # An orphan predating all of this: mode 0644, not executable, referenced by - # no unit and no cron entry, pushing to a Kuma monitor. Superseded by - # ups-status-healthcheck. - - name: Remove the orphaned hand-written UPS heartbeat - ansible.builtin.file: - path: /usr/local/bin/ups-heartbeat.sh - state: absent - - - name: Confirm nothing still pushes to Uptime Kuma - ansible.builtin.shell: >- - grep -rl "uptime.contrapeso.xyz" /etc/systemd/system /usr/local/bin /opt 2>/dev/null || true - register: kuma_refs - changed_when: false - - - name: Report any remaining references - ansible.builtin.debug: - msg: >- - {{ 'clean - nothing references Uptime Kuma' - if kuma_refs.stdout | trim | length == 0 - else 'STILL REFERENCING KUMA: ' ~ kuma_refs.stdout_lines | join(', ') }} diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml new file mode 100644 index 0000000..dcd9bdc --- /dev/null +++ b/ansible/infra/410_disk_usage_alerts.yml @@ -0,0 +1,343 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ +- name: Deploy Disk Usage Monitoring + hosts: managed + become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml + - ../infra_secrets.yml + + vars: + disk_usage_threshold_percent: 80 + disk_check_interval_minutes: 15 + monitored_mount_point: "/" + monitoring_script_dir: /opt/disk-monitoring + monitoring_script_path: "{{ monitoring_script_dir }}/disk_usage_monitor.sh" + log_file: "{{ monitoring_script_dir }}/disk_usage_monitor.log" + systemd_service_name: disk-usage-monitor + # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname and mount point + set_fact: + monitor_name: "disk-usage-{{ host_name.stdout }}-{{ monitored_mount_point | replace('/', 'root') }}" + monitor_friendly_name: "Disk Usage: {{ host_name.stdout }} ({{ monitored_mount_point }})" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': True, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when usage exceeds {{ disk_usage_threshold_percent }}%" + "{{ (disk_check_interval_minutes * 60) + 60 }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_disk_usage_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for disk monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create disk usage monitoring script + copy: + dest: "{{ monitoring_script_path }}" + content: | + #!/bin/bash + + # Disk Usage Monitoring Script + # Monitors disk usage and sends alerts to Uptime Kuma + # Mode: "No news is good news" - only sends alerts when disk usage is HIGH + + LOG_FILE="{{ log_file }}" + USAGE_THRESHOLD="{{ disk_usage_threshold_percent }}" + UPTIME_KUMA_URL="{{ uptime_kuma_disk_usage_push_url }}" + MOUNT_POINT="{{ monitored_mount_point }}" + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to get disk usage percentage + get_disk_usage() { + local mount_point="$1" + local usage="" + + # Get disk usage percentage (without % sign) + usage=$(df -h "$mount_point" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') + + if [ -z "$usage" ]; then + log_message "ERROR: Could not read disk usage for $mount_point" + return 1 + fi + + echo "$usage" + } + + # Function to get disk usage details + get_disk_details() { + local mount_point="$1" + df -h "$mount_point" 2>/dev/null | awk 'NR==2 {print "Used: "$3" / Total: "$2" ("$5" full)"}' + } + + # Function to send alert to Uptime Kuma when disk usage exceeds threshold + # With upside-down mode enabled, sending status=up will trigger an alert + send_uptime_kuma_alert() { + local usage="$1" + local details="$2" + local message="DISK FULL WARNING: ${MOUNT_POINT} is ${usage}% full (Threshold: ${USAGE_THRESHOLD}%) - ${details}" + + log_message "ALERT: $message" + + # Send push notification to Uptime Kuma with status=up + # In upside-down mode, status=up is treated as down/alert + response=$(curl -s -w "\n%{http_code}" -G \ + --data-urlencode "status=up" \ + --data-urlencode "msg=$message" \ + "$UPTIME_KUMA_URL" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" + else + log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" + fi + } + + # Main monitoring logic + main() { + log_message "Starting disk usage check for $MOUNT_POINT" + + # Get current disk usage + current_usage=$(get_disk_usage "$MOUNT_POINT") + + if [ $? -ne 0 ] || [ -z "$current_usage" ]; then + log_message "ERROR: Could not read disk usage" + exit 1 + fi + + # Get disk details + disk_details=$(get_disk_details "$MOUNT_POINT") + + log_message "Current disk usage: ${current_usage}% - $disk_details" + + # Check if usage exceeds threshold + if [ "$current_usage" -gt "$USAGE_THRESHOLD" ]; then + log_message "WARNING: Disk usage ${current_usage}% exceeds threshold ${USAGE_THRESHOLD}%" + send_uptime_kuma_alert "$current_usage" "$disk_details" + else + log_message "Disk usage is within normal range - no alert needed (no news is good news)" + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for disk usage monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.service" + content: | + [Unit] + Description=Disk Usage Monitor + After=network.target + + [Service] + Type=oneshot + ExecStart={{ monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for disk usage monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" + content: | + [Unit] + Description=Run Disk Usage Monitor every {{ disk_check_interval_minutes }} minute(s) + Requires={{ systemd_service_name }}.service + + [Timer] + OnBootSec={{ disk_check_interval_minutes }}min + OnUnitActiveSec={{ disk_check_interval_minutes }}min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start disk usage monitoring timer + systemd: + name: "{{ systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test disk usage monitoring script + command: "{{ monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "Disk usage monitoring script failed to execute properly" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml new file mode 100644 index 0000000..05532a7 --- /dev/null +++ b/ansible/infra/420_system_healthcheck.yml @@ -0,0 +1,325 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ +- name: Deploy System Healthcheck Monitoring + hosts: managed + become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml + - ../infra_secrets.yml + + vars: + healthcheck_interval_seconds: 60 # Send healthcheck every 60 seconds (1 minute) + healthcheck_timeout_seconds: 90 # Uptime Kuma should alert if no ping received within 90s + healthcheck_retries: 1 # Number of retries before alerting + monitoring_script_dir: /opt/system-healthcheck + monitoring_script_path: "{{ monitoring_script_dir }}/system_healthcheck.sh" + log_file: "{{ monitoring_script_dir }}/system_healthcheck.log" + systemd_service_name: system-healthcheck + # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "system-healthcheck-{{ host_name.stdout }}" + monitor_friendly_name: "System Healthcheck: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_healthcheck_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_healthcheck_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Regular healthcheck ping every {{ healthcheck_interval_seconds }}s" + "{{ healthcheck_timeout_seconds }}" + "{{ healthcheck_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_healthcheck_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for healthcheck monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create system healthcheck script + copy: + dest: "{{ monitoring_script_path }}" + content: | + #!/bin/bash + + # System Healthcheck Script + # Sends regular heartbeat pings to Uptime Kuma + # This ensures the system is running and able to communicate + + LOG_FILE="{{ log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_healthcheck_push_url }}" + HOSTNAME=$(hostname) + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to send healthcheck ping to Uptime Kuma + send_healthcheck() { + local uptime_seconds=$(awk '{print int($1)}' /proc/uptime) + local uptime_days=$((uptime_seconds / 86400)) + local uptime_hours=$(((uptime_seconds % 86400) / 3600)) + local uptime_minutes=$(((uptime_seconds % 3600) / 60)) + + local message="System healthy - Uptime: ${uptime_days}d ${uptime_hours}h ${uptime_minutes}m" + + log_message "Sending healthcheck ping: $message" + + # Send push notification to Uptime Kuma with status=up + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Healthcheck ping sent successfully (HTTP $http_code)" + else + log_message "ERROR: Failed to send healthcheck ping (HTTP $http_code)" + return 1 + fi + } + + # Main healthcheck logic + main() { + log_message "Starting system healthcheck for $HOSTNAME" + + # Send healthcheck ping + if send_healthcheck; then + log_message "Healthcheck completed successfully" + else + log_message "ERROR: Healthcheck failed" + exit 1 + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for system healthcheck + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.service" + content: | + [Unit] + Description=System Healthcheck Monitor + After=network.target + + [Service] + Type=oneshot + ExecStart={{ monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for system healthcheck + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" + content: | + [Unit] + Description=Run System Healthcheck every minute + Requires={{ systemd_service_name }}.service + + [Timer] + OnBootSec=30sec + OnUnitActiveSec={{ healthcheck_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start system healthcheck timer + systemd: + name: "{{ systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test system healthcheck script + command: "{{ monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "System healthcheck script failed to execute properly" + + - name: Display monitor information + debug: + msg: | + ✓ System healthcheck monitoring deployed successfully! + + Monitor Name: {{ monitor_friendly_name }} + Monitor Group: {{ uptime_kuma_monitor_group }} + Healthcheck Interval: Every {{ healthcheck_interval_seconds }} seconds (1 minute) + Timeout: {{ healthcheck_timeout_seconds }} seconds (90s) + Retries: {{ healthcheck_retries }} + + The system will send a heartbeat ping every minute. + Uptime Kuma will alert if no ping is received within 90 seconds (with 1 retry). + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_healthcheck_monitor.py + state: absent + delegate_to: localhost + become: no + diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml new file mode 100644 index 0000000..5c9e855 --- /dev/null +++ b/ansible/infra/430_cpu_temp_alerts.yml @@ -0,0 +1,329 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ +- name: Deploy CPU Temperature Monitoring + hosts: hypervisor + become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml + - ../infra_secrets.yml + + vars: + temp_threshold_celsius: 80 + temp_check_interval_minutes: 1 + monitoring_script_dir: /opt/nodito-monitoring + monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" + log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" + systemd_service_name: nodito-cpu-temp-monitor + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "cpu-temp-{{ host_name.stdout }}" + monitor_friendly_name: "CPU Temperature: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma CPU temperature monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_cpu_temp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + monitors = api.get_monitors() + notifications = api.get_notifications() + + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name=group_name) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': True, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + api.edit_monitor(existing_monitor['id'], **monitor_data) + else: + api.add_monitor(**monitor_data) + + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_cpu_temp_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when temperature exceeds {{ temp_threshold_celsius }}°C" + "{{ (temp_check_interval_minutes * 60) + 60 }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_cpu_temp_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for temperature monitoring + package: + name: + - lm-sensors + - curl + - jq + - bc + state: present + + - name: Create monitoring script directory + file: + path: "{{ monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create CPU temperature monitoring script + copy: + dest: "{{ monitoring_script_path }}" + content: | + #!/bin/bash + + # CPU Temperature Monitoring Script + # Monitors CPU temperature and sends alerts to Uptime Kuma + + LOG_FILE="{{ log_file }}" + TEMP_THRESHOLD="{{ temp_threshold_celsius }}" + UPTIME_KUMA_URL="{{ uptime_kuma_cpu_temp_push_url }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + get_cpu_temp() { + local temp="" + + if command -v sensors >/dev/null 2>&1; then + temp=$(sensors 2>/dev/null | grep -E "Core 0|Package id 0|Tdie|Tctl" | head -1 | grep -oE '[0-9]+\.[0-9]+°C' | grep -oE '[0-9]+\.[0-9]+') + fi + + if [ -z "$temp" ] && [ -f /sys/class/thermal/thermal_zone0/temp ]; then + temp=$(cat /sys/class/thermal/thermal_zone0/temp) + temp=$(echo "scale=1; $temp/1000" | bc -l 2>/dev/null || echo "$temp") + fi + + if [ -z "$temp" ] && command -v acpi >/dev/null 2>&1; then + temp=$(acpi -t 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1) + fi + + echo "$temp" + } + + send_uptime_kuma_alert() { + local temp="$1" + local message="CPU Temperature Alert: ${temp}°C (Threshold: ${TEMP_THRESHOLD}°C)" + + log_message "ALERT: $message" + + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/°/%C2%B0/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g') + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" + else + log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" + fi + } + + main() { + log_message "Starting CPU temperature check" + + current_temp=$(get_cpu_temp) + + if [ -z "$current_temp" ]; then + log_message "ERROR: Could not read CPU temperature" + exit 1 + fi + + log_message "Current CPU temperature: ${current_temp}°C" + + if (( $(echo "$current_temp > $TEMP_THRESHOLD" | bc -l) )); then + log_message "WARNING: CPU temperature ${current_temp}°C exceeds threshold ${TEMP_THRESHOLD}°C" + send_uptime_kuma_alert "$current_temp" + else + log_message "CPU temperature is within normal range" + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for CPU temperature monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.service" + content: | + [Unit] + Description=CPU Temperature Monitor + After=network.target + + [Service] + Type=oneshot + ExecStart={{ monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for CPU temperature monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" + content: | + [Unit] + Description=Run CPU Temperature Monitor every {{ temp_check_interval_minutes }} minute(s) + Requires={{ systemd_service_name }}.service + + [Timer] + OnBootSec={{ temp_check_interval_minutes }}min + OnUnitActiveSec={{ temp_check_interval_minutes }}min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start CPU temperature monitoring timer + systemd: + name: "{{ systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test CPU temperature monitoring script + command: "{{ monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "CPU temperature monitoring script failed to execute properly" + + - name: Display monitoring configuration + debug: + msg: + - "CPU Temperature Monitoring configured successfully" + - "Temperature threshold: {{ temp_threshold_celsius }}°C" + - "Check interval: {{ temp_check_interval_minutes }} minute(s)" + - "Monitor Name: {{ monitor_friendly_name }}" + - "Monitor Group: {{ uptime_kuma_monitor_group }}" + - "Uptime Kuma Push URL: {{ uptime_kuma_cpu_temp_push_url }}" + - "Monitoring script: {{ monitoring_script_path }}" + - "Systemd Service: {{ systemd_service_name }}.service" + - "Systemd Timer: {{ systemd_service_name }}.timer" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_cpu_temp_monitor.py + state: absent + delegate_to: localhost + become: no + diff --git a/ansible/infra/900_install_rsync.yml b/ansible/infra/900_install_rsync.yml index b690009..6c6b10e 100644 --- a/ansible/infra/900_install_rsync.yml +++ b/ansible/infra/900_install_rsync.yml @@ -1,5 +1,7 @@ - name: Install rsync hosts: managed + vars_files: + - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 4decb5d..5b5e4ce 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -1,10 +1,14 @@ - name: Join machine to headscale mesh network hosts: managed become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml vars: headscale_host_name: "spacey" headscale_subdomain: "{{ subdomains.headscale }}" headscale_domain: "https://{{ headscale_subdomain }}.{{ root_domain }}" + headscale_namespace: "{{ service_settings.headscale.namespace }}" tasks: - name: Set facts for headscale server connection diff --git a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml index 4edab06..86f693f 100644 --- a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml +++ b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml @@ -1,6 +1,8 @@ - name: Bootstrap Nodito SSH Key Access hosts: hypervisor become: true + vars_files: + - ../infra_vars.yml tasks: - name: Install sudo package diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml index 0fab184..378a674 100644 --- a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -1,6 +1,8 @@ - name: Switch Proxmox VE from Enterprise to Community Repositories hosts: hypervisor become: true + vars_files: + - ../infra_vars.yml tasks: - name: Check for deb822 sources format diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 2efbf8c..8cdde6a 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -1,6 +1,9 @@ - name: Setup ZFS RAID 1 Pool for Proxmox Storage hosts: hypervisor become: true + vars_files: + - ../infra_vars.yml + - nodito_vars.yml tasks: - name: Verify Proxmox VE is running @@ -136,18 +139,17 @@ Config file exists: {{ storage_cfg_file.stat.exists }} Storage check result: {{ storage_exists_check.rc }} Pool exists: {{ zfs_pool_exists.rc == 0 }} + Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }} Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }} - # Registration is add-only on purpose. There used to be a "Remove existing - # storage if it exists" task here that ran `pvesm remove` whenever the - # storage WAS present, paired with an add that only ran when it was ABSENT. - # The two conditions are mutually exclusive, so a real run against a - # correctly-configured hypervisor removed the storage entry backing every VM - # and never put it back. It would also have dropped `mountpoint /var/lib/vz`, - # which the live entry has and which `pvesm add` below does not set. - # - # If the storage entry ever needs its options changed, edit - # /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here. + - name: Remove existing storage if it exists + command: pvesm remove {{ zfs_pool_name }} + register: pvesm_remove_result + failed_when: false + when: + - zfs_pool_exists.rc == 0 + - storage_exists_check.rc == 0 + - name: Add ZFS pool storage to Proxmox using pvesm command: > pvesm add zfspool {{ zfs_pool_name }} @@ -169,56 +171,511 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" -# ───────────────────────────────────────────────────────────────────────────── -# The monthly scrub. +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. # -# The ZFS HEALTH CHECK that used to share this play is gone: it is now the -# zfs-health check in infra/400_host_monitoring.yml, which carries the same five -# conditions - pool state, device states, resilver in progress, read/write/ -# checksum errors, and errors from the last scan - but reports to Gatus like -# every other host check instead of owning its own push plumbing. +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. # -# The scrub itself stays here, because it is not monitoring: it is the -# maintenance that gives the health check something true to report. A pool that -# is never scrubbed has no idea whether it is healthy. -# ───────────────────────────────────────────────────────────────────────────── -- name: Schedule the monthly ZFS scrub +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ +- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs hosts: hypervisor become: true vars_files: - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml vars: + zfs_check_interval_seconds: 86400 # 24 hours + zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) + zfs_check_retries: 1 + zfs_monitoring_script_dir: /opt/zfs-monitoring + zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" + zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" + zfs_systemd_health_service_name: zfs-health-monitor zfs_systemd_scrub_service_name: zfs-monthly-scrub + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "zfs-health-{{ host_name.stdout }}" + monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma ZFS health monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_zfs_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma ZFS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_zfs_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" + "{{ zfs_check_timeout_seconds }}" + "{{ zfs_check_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for ZFS monitoring + package: + name: + - curl + - jq + state: present + + - name: Create monitoring script directory + file: + path: "{{ zfs_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create ZFS health monitoring script + copy: + dest: "{{ zfs_monitoring_script_path }}" + content: | + #!/bin/bash + + # ZFS Pool Health Monitoring Script + # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy + # If any issues detected, does NOT send heartbeat (triggers timeout alert) + + LOG_FILE="{{ zfs_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" + POOL_NAME="{{ zfs_pool_name }}" + HOSTNAME=$(hostname) + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to check pool health using JSON output + check_pool_health() { + local pool="$1" + local issues_found=0 + + # Get pool status as JSON + local pool_json + pool_json=$(zpool status -j "$pool" 2>&1) + + if [ $? -ne 0 ]; then + log_message "ERROR: Failed to get pool status for $pool" + log_message " -> $pool_json" + return 1 + fi + + # Check 1: Pool state must be ONLINE + local pool_state + pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') + + if [ "$pool_state" != "ONLINE" ]; then + log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" + issues_found=1 + else + log_message "OK: Pool state is ONLINE" + fi + + # Check 2: Check all vdevs and devices for non-ONLINE states + local bad_states + bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.state? and .state != "ONLINE") | + "\(.name // "unknown"): \(.state)" + ' 2>/dev/null) + + if [ -n "$bad_states" ]; then + log_message "ISSUE: Found devices not in ONLINE state:" + echo "$bad_states" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: All devices are ONLINE" + fi + + # Check 3: Check for resilvering in progress + local scan_function scan_state + scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + + if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then + local resilver_progress + resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') + log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" + issues_found=1 + fi + + # Check 4: Check for read/write/checksum errors on all devices + # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison + local devices_with_errors + devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | + "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" + ' 2>/dev/null) + + if [ -n "$devices_with_errors" ]; then + log_message "ISSUE: Found devices with I/O errors:" + echo "$devices_with_errors" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: No read/write/checksum errors detected" + fi + + # Check 5: Check for scan errors (from last scrub/resilver) + local scan_errors + scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') + + if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then + log_message "ISSUE: Last scan reported $scan_errors errors" + issues_found=1 + else + log_message "OK: No scan errors" + fi + + return $issues_found + } + + # Function to get last scrub info for status message + get_scrub_info() { + local pool="$1" + local pool_json + pool_json=$(zpool status -j "$pool" 2>/dev/null) + + local scan_func scan_state scan_start + scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') + + if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then + echo "scrub in progress (started $scan_start)" + elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then + echo "last scrub: $scan_start" + else + echo "no scrub history" + fi + } + + # Function to send heartbeat to Uptime Kuma + send_heartbeat() { + local message="$1" + + log_message "Sending heartbeat to Uptime Kuma: $message" + + # URL encode the message + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent successfully (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + # Main health check logic + main() { + log_message "==========================================" + log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" + + # Run all health checks + if check_pool_health "$POOL_NAME"; then + # All checks passed - send heartbeat + local scrub_info + scrub_info=$(get_scrub_info "$POOL_NAME") + + local message="Pool $POOL_NAME healthy ($scrub_info)" + send_heartbeat "$message" + + log_message "Health check completed: ALL OK" + exit 0 + else + # Issues found - do NOT send heartbeat (will trigger timeout alert) + log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" + log_message "Uptime Kuma will alert after timeout due to missing heartbeat" + exit 1 + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" + content: | + [Unit] + Description=ZFS Pool Health Monitor + After=zfs.target network.target + + [Service] + Type=oneshot + ExecStart={{ zfs_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for daily ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Pool Health Monitor daily + Requires={{ zfs_systemd_health_service_name }}.service + + [Timer] + OnBootSec=5min + OnUnitActiveSec={{ zfs_check_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + - name: Create systemd service for ZFS monthly scrub - template: - src: templates/zfs-monthly-scrub.service.j2 + copy: dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" + content: | + [Unit] + Description=ZFS Monthly Scrub for {{ zfs_pool_name }} + After=zfs.target + + [Service] + Type=oneshot + ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target owner: root group: root mode: '0644' - name: Create systemd timer for monthly ZFS scrub - template: - src: templates/zfs-monthly-scrub.timer.j2 + copy: dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Scrub on last day of every month at 4:00 AM + Requires={{ zfs_systemd_scrub_service_name }}.service + + [Timer] + OnCalendar=*-*~01 04:00:00 + Persistent=true + + [Install] + WantedBy=timers.target owner: root group: root mode: '0644' - - name: Enable and start the monthly scrub timer + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start ZFS health monitoring timer + systemd: + name: "{{ zfs_systemd_health_service_name }}.timer" + enabled: yes + state: started + + - name: Enable and start ZFS monthly scrub timer systemd: name: "{{ zfs_systemd_scrub_service_name }}.timer" enabled: yes state: started - daemon_reload: yes - - name: Report the scrub schedule + - name: Test ZFS health monitoring script + command: "{{ zfs_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "ZFS health monitoring script failed - check pool health" + + - name: Display monitoring configuration debug: - msg: >- - Monthly scrub of {{ zfs_pool_name }}: - last day of each month at 04:00. - Health is reported separately by the zfs-health check - (infra/400_host_monitoring.yml). + msg: | + ✓ ZFS Pool Health Monitoring deployed successfully! + + Monitor Name: {{ monitor_friendly_name }} + Monitor Group: {{ uptime_kuma_monitor_group }} + Pool Name: {{ zfs_pool_name }} + + Health Check: + - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) + - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) + - Script: {{ zfs_monitoring_script_path }} + - Log: {{ zfs_log_file }} + - Service: {{ zfs_systemd_health_service_name }}.service + - Timer: {{ zfs_systemd_health_service_name }}.timer + + Monthly Scrub: + - Schedule: Last day of month at 4:00 AM + - Service: {{ zfs_systemd_scrub_service_name }}.service + - Timer: {{ zfs_systemd_scrub_service_name }}.timer + + Conditions monitored: + - Pool state (must be ONLINE) + - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) + - Resilver status (alerts if resilvering) + - Read/Write/Checksum errors + - Scrub errors + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_zfs_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index faf93a5..3b687ee 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -1,6 +1,9 @@ - name: Create Proxmox template from Debian cloud image (no VM clone) hosts: hypervisor become: true + vars_files: + - ../../infra_vars.yml + - nodito_vars.yml vars: # Defaults (override via vars_files or --extra-vars as needed) diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index 12fe720..f39fb56 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -1,33 +1,12 @@ - name: Setup NUT (Network UPS Tools) for CyberPower UPS hosts: hypervisor become: true + vars_files: + - ../../infra_vars.yml + - nodito_vars.yml + - nodito_secrets.yml tasks: - # ------------------------------------------------------------------ - # Safety catch - # - # /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by - # hand in January 2026 and carry a working password. host_vars/nodito/vault.yml - # (formerly infra/nodito/nodito_secrets.yml) still holds the literal string - # CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that - # working pair with a placeholder and restart NUT - leaving the hypervisor's - # UPS unmonitored and unable to trigger a clean shutdown on mains loss. - # - # Until the real password is put in the vault, stop here. - # ansible-vault edit host_vars/nodito/vault.yml - # ------------------------------------------------------------------ - - name: Refuse to run with a placeholder UPS password - assert: - that: - - ups_password is defined - - ups_password | length > 0 - - ups_password != "CHANGE_ME_TO_SECURE_PASSWORD" - fail_msg: >- - ups_password is unset or still the placeholder. Applying this play would - overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on - nodito and restart NUT. Put the real password in the vault first: - ansible-vault edit host_vars/nodito/vault.yml - # ------------------------------------------------------------------ # Installation # ------------------------------------------------------------------ @@ -96,45 +75,90 @@ # Configuration files # ------------------------------------------------------------------ - name: Configure NUT mode (standalone) - template: + copy: dest: /etc/nut/nut.conf - src: templates/nut.conf.j2 + content: | + # Managed by Ansible + MODE=standalone owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure UPS device - template: + copy: dest: /etc/nut/ups.conf - src: templates/ups.conf.j2 + content: | + # Managed by Ansible + [{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd to listen on localhost - template: + copy: dest: /etc/nut/upsd.conf - src: templates/upsd.conf.j2 + content: | + # Managed by Ansible + LISTEN 127.0.0.1 3493 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd users - template: + copy: dest: /etc/nut/upsd.users - src: templates/upsd.users.j2 + content: | + # Managed by Ansible + [{{ ups_user }}] + password = {{ ups_password }} + upsmon master owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsmon - template: + copy: dest: /etc/nut/upsmon.conf - src: templates/upsmon.conf.j2 + content: | + # Managed by Ansible + MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + + MINSUPPLIES 1 + SHUTDOWNCMD "/sbin/shutdown -h +0" + POLLFREQ 5 + POLLFREQALERT 5 + HOSTSYNC 15 + DEADTIME 15 + POWERDOWNFLAG /etc/killpower + + # Notifications + NOTIFYMSG ONLINE "UPS %s on line power" + NOTIFYMSG ONBATT "UPS %s on battery" + NOTIFYMSG LOWBATT "UPS %s battery is low" + NOTIFYMSG FSD "UPS %s: forced shutdown in progress" + NOTIFYMSG COMMOK "Communications with UPS %s established" + NOTIFYMSG COMMBAD "Communications with UPS %s lost" + NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" + NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + + # Log all events to syslog + NOTIFYFLAG ONLINE SYSLOG + NOTIFYFLAG ONBATT SYSLOG + NOTIFYFLAG LOWBATT SYSLOG + NOTIFYFLAG FSD SYSLOG + NOTIFYFLAG COMMOK SYSLOG + NOTIFYFLAG COMMBAD SYSLOG + NOTIFYFLAG SHUTDOWN SYSLOG + NOTIFYFLAG REPLBATT SYSLOG owner: root group: nut mode: "0640" @@ -225,11 +249,334 @@ - nut-server - nut-monitor -# The UPS heartbeat play that used to live here is gone. What it deployed - -# /opt/ups-monitoring plus a ups-heartbeat timer - is now the ups-status check -# in infra/400_host_monitoring.yml, which reports to Gatus like every other -# host check instead of carrying its own push plumbing. + +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. # -# This playbook is now purely NUT setup: the driver, upsd, upsmon and the -# shutdown behaviour. Monitoring whether the UPS is on mains is a separate -# concern and belongs with the other host checks. +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ +- name: Setup UPS Heartbeat Monitoring with Uptime Kuma + hosts: hypervisor + become: true + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml + - nodito_secrets.yml + + vars: + ups_heartbeat_interval_seconds: 60 + ups_heartbeat_timeout_seconds: 120 + ups_heartbeat_retries: 1 + ups_monitoring_script_dir: /opt/ups-monitoring + ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" + ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" + ups_systemd_service_name: ups-heartbeat + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "ups-{{ host_name.stdout }}" + monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma UPS monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_ups_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + monitors = api.get_monitors() + notifications = api.get_notifications() + + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name=group_name) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + api.edit_monitor(existing_monitor['id'], **monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + api.add_monitor(**monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma UPS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_ups_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" + "{{ ups_heartbeat_timeout_seconds }}" + "{{ ups_heartbeat_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL as fact + set_fact: + uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Install required packages for UPS monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ ups_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create UPS heartbeat monitoring script + copy: + dest: "{{ ups_monitoring_script_path }}" + content: | + #!/bin/bash + + # UPS Heartbeat Monitoring Script + # Sends heartbeat to Uptime Kuma only when UPS is on mains power + # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) + + LOG_FILE="{{ ups_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" + UPS_NAME="{{ ups_name }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + send_heartbeat() { + local message="$1" + + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent: $message (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + main() { + local status charge runtime load + + status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) + + if [ -z "$status" ]; then + log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" + exit 1 + fi + + charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) + runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) + load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) + + if [[ "$status" == *"OL"* ]]; then + local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" + send_heartbeat "$message" + exit 0 + else + log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" + content: | + [Unit] + Description=UPS Heartbeat Monitor + After=network.target nut-monitor.service + + [Service] + Type=oneshot + ExecStart={{ ups_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" + content: | + [Unit] + Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds + Requires={{ ups_systemd_service_name }}.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start UPS heartbeat timer + systemd: + name: "{{ ups_systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test UPS heartbeat script + command: "{{ ups_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "UPS heartbeat script failed - check UPS status and communication" + + - name: Display monitoring configuration + debug: + msg: + - "UPS Monitoring configured successfully" + - "" + - "NUT Configuration:" + - " UPS Name: {{ ups_name }}" + - " UPS Description: {{ ups_desc }}" + - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" + - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" + - "" + - "Uptime Kuma Monitoring:" + - " Monitor Name: {{ monitor_friendly_name }}" + - " Monitor Group: {{ uptime_kuma_monitor_group }}" + - " Push URL: {{ uptime_kuma_ups_push_url }}" + - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" + - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" + - "" + - "Scripts and Services:" + - " Script: {{ ups_monitoring_script_path }}" + - " Log: {{ ups_log_file }}" + - " Service: {{ ups_systemd_service_name }}.service" + - " Timer: {{ ups_systemd_service_name }}.timer" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_ups_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/nodito/nodito_secrets.yml b/ansible/infra/nodito/nodito_secrets.yml new file mode 100644 index 0000000..c34b1e8 --- /dev/null +++ b/ansible/infra/nodito/nodito_secrets.yml @@ -0,0 +1,12 @@ +$ANSIBLE_VAULT;1.1;AES256 +39386134336137333139343861353965353665626161303634323563386563356364373265616163 +3237623438636161323366313363356335613566303338630a663732316663333136653737663935 +34373466316538663962636134303530323837316563336136653636643430363566376363323666 +3531613636393331310a306633613030343138656237663061653438336562323135633732356563 +33396136656564353465636361613161326465303166373131343562363338303966666362643532 +33383561366539393930633766363533313363653733393263376634316362643863376635366638 +36396333633231616662323465323932656565396138313264613832346261616231333265636439 +61666334613662373839613833613663333436373365376534643662656335316536303739616437 +65333635613330353536353162646234323266316338643435653864666364313734386665303830 +34383961633734646134373866623330663038366130306265656466653562643764346162313333 +353664363335303433393330306332666336 diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml new file mode 100644 index 0000000..c0002f3 --- /dev/null +++ b/ansible/infra/nodito/nodito_vars.yml @@ -0,0 +1,28 @@ +# Nodito CPU Temperature Monitoring Configuration + +# Temperature Monitoring Configuration +temp_threshold_celsius: 80 +temp_check_interval_minutes: 1 + +# Script Configuration +monitoring_script_dir: /opt/nodito-monitoring +monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" +log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" + +# System Configuration +systemd_service_name: nodito-cpu-temp-monitor + +# ZFS Pool Configuration +zfs_pool_name: "proxmox-tank-1" +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror +zfs_pool_mountpoint: "/var/lib/vz" + +# UPS Configuration (CyberPower CP900EPFCLCD via USB) +ups_name: cyberpower +ups_desc: "CyberPower CP900EPFCLCD" +ups_driver: usbhid-ups +ups_port: auto +ups_user: counterweight +ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power +ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power diff --git a/ansible/infra/nodito/templates/nut.conf.j2 b/ansible/infra/nodito/templates/nut.conf.j2 deleted file mode 100644 index 1f8a72f..0000000 --- a/ansible/infra/nodito/templates/nut.conf.j2 +++ /dev/null @@ -1,2 +0,0 @@ -# Managed by Ansible -MODE=standalone diff --git a/ansible/infra/nodito/templates/ups.conf.j2 b/ansible/infra/nodito/templates/ups.conf.j2 deleted file mode 100644 index 0fa0a60..0000000 --- a/ansible/infra/nodito/templates/ups.conf.j2 +++ /dev/null @@ -1,9 +0,0 @@ -# Managed by Ansible -maxretry = 3 - -[{{ ups_name }}] - driver = {{ ups_driver }} - port = {{ ups_port }} - desc = "{{ ups_desc }}" - offdelay = {{ ups_offdelay }} - ondelay = {{ ups_ondelay }} diff --git a/ansible/infra/nodito/templates/upsd.conf.j2 b/ansible/infra/nodito/templates/upsd.conf.j2 deleted file mode 100644 index baddf21..0000000 --- a/ansible/infra/nodito/templates/upsd.conf.j2 +++ /dev/null @@ -1,2 +0,0 @@ -# Managed by Ansible -LISTEN 127.0.0.1 3493 diff --git a/ansible/infra/nodito/templates/upsd.users.j2 b/ansible/infra/nodito/templates/upsd.users.j2 deleted file mode 100644 index 8d43784..0000000 --- a/ansible/infra/nodito/templates/upsd.users.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# Managed by Ansible -[{{ ups_user }}] - password = {{ ups_password }} - upsmon master diff --git a/ansible/infra/nodito/templates/upsmon.conf.j2 b/ansible/infra/nodito/templates/upsmon.conf.j2 deleted file mode 100644 index dc301e1..0000000 --- a/ansible/infra/nodito/templates/upsmon.conf.j2 +++ /dev/null @@ -1,34 +0,0 @@ -# Managed by Ansible -MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master - -MINSUPPLIES 1 -SHUTDOWNCMD "/sbin/shutdown -h +0" -POLLFREQ 5 -POLLFREQALERT 5 -HOSTSYNC 15 -DEADTIME 15 -POWERDOWNFLAG "/etc/killpower" -OFFDURATION 30 -RBWARNTIME 43200 -NOCOMMWARNTIME 300 -FINALDELAY 5 - -# Notifications -NOTIFYMSG ONLINE "UPS %s on line power" -NOTIFYMSG ONBATT "UPS %s on battery" -NOTIFYMSG LOWBATT "UPS %s battery is low" -NOTIFYMSG FSD "UPS %s: forced shutdown in progress" -NOTIFYMSG COMMOK "Communications with UPS %s established" -NOTIFYMSG COMMBAD "Communications with UPS %s lost" -NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" -NOTIFYMSG REPLBATT "UPS %s battery needs replacing" - -# Log all events to syslog -NOTIFYFLAG ONLINE SYSLOG -NOTIFYFLAG ONBATT SYSLOG -NOTIFYFLAG LOWBATT SYSLOG -NOTIFYFLAG FSD SYSLOG -NOTIFYFLAG COMMOK SYSLOG -NOTIFYFLAG COMMBAD SYSLOG -NOTIFYFLAG SHUTDOWN SYSLOG -NOTIFYFLAG REPLBATT SYSLOG diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 deleted file mode 100644 index 04b3514..0000000 --- a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=ZFS Monthly Scrub for {{ zfs_pool_name }} -After=zfs.target - -[Service] -Type=oneshot -ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} -User=root -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 deleted file mode 100644 index 36ad3d7..0000000 --- a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Run ZFS Scrub on last day of every month at 4:00 AM -Requires={{ zfs_systemd_scrub_service_name }}.service - -[Timer] -OnCalendar=*-*~01 04:00:00 -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml new file mode 100644 index 0000000..4803d86 --- /dev/null +++ b/ansible/infra_secrets.yml @@ -0,0 +1,60 @@ +$ANSIBLE_VAULT;1.1;AES256 +34616531626537376463326332306434383163393761363536633133363161373631376437653234 +3831656536313531343433376264643261396634633965370a303931336236323766313065636535 +31343338616364386132623232373266663665353563666535383232666262663062323864303932 +3439373734316639650a633732333064633335383833663261326464336538636666323063646331 +66653464636534663935616461636132613162303530663237376661343338396133323431623561 +30303563303465633231306339326561313533333536376433393130376634303932643339656336 +30616130343733626162366561393939393766623138616537393032376665396139333561623830 +33623130336235633034666561383165616439316334323664623661343438663733373833306335 +30396165313731636435636263353133326135346236323030653734353831626238666663656364 +62386164363361373464383730333733626333333736306535336635613634646535383237326332 +37636532663933663739303634333235343137333363316430643263326231613635643633636437 +30336462633233396430633939616661643261666136656363363461343230653738613166346434 +63376363666462346539343063343562623962303130616535306439653134633630353963373337 +64313239353162363862643432666438383330636638333464323731643163643236313535313030 +65656231383738356537326661663163376634613031396436646633376139313237343534653065 +64323532346133313265346334353530393633396339316330366536646565353836396662303866 +65373030393463306664623235626437613965303837313365373632643935656630303936373837 +64663539623764656636613263346638376665626262663430333231633735653563643864343835 +62303637383332653366383066326536316336306539623230353066343739316430356437313365 +33323031613130643832636133636263303565623330306135333762363036633737343933623437 +33633436363062643436376632336235666330656265316333323361316430656631383734343238 +65616366666536363866343361666563643336306532666332643230656264303565353032393932 +36613738326239376161383064646634623639356439663965323237303361343838373737636235 +39656465333063633136643463356564636537326339653033633063366265636138656466646264 +32643961343863656464353439346531633036613138353333353631626665396239326465303632 +39336664333038636435366333623336353963323535316436646136313739373537376537353735 +66333265633232346536313933333836643439656666636236626266323333383935383565616139 +38626133356364613562393664643563346539366364636332613162663337356232333735393431 +30343635393462356431646336636663333735343164616261343836363366616162613563613066 +66333034613234626431356536323832393466363135653231336238356334353638613838626430 +63636630313533363563376161663637353362383431663130646666646433353864373736613662 +62343666633862643839666532656132653939663437373164393536653739376265303130353636 +62616538326330333136626238306133363634326365366637363961333635356133613564373131 +64626139656633663065353862343135393233333231386166346532656534623361623061336434 +37616565626134333863376532393861376132323434656433613731613834633963386635353336 +39653263346637303836663763653264396139383363616461393333333362616332663863363834 +61613832326466646133626337663364333566326633313664613236346135653332373035313034 +62356530313065393737393634313166313039616635363365633031336434396463326462316236 +62353838636533303664663132353838636632353733316639643964666139383539333761663531 +37366530373165653063303032383461326535613336343635626538373635356266396666346133 +61383434323565306438356634646338666232363238393932353365363461376130323363363236 +34643862653937353461383265663933646264373365623465313666643662633334366437646434 +34343332613832366235663563353435323738356339383338333361383561336366376238393237 +38656265663164376666616366623062393530366261623361383365376235643035353838313235 +32396430376139313430316439333539343266393965353030636638656661316137313437663832 +64366661323865383262323036613230316233666566386133363633303931323464663238323861 +62333132346165353830653062353763306232366563323430383933653163373562326536393362 +66353931376262383931613931303034356537363137366235643430323465623162363031373034 +64653563666533363062333831336463376530306639616631303266386136666233653638653732 +31313537393838333730303235363661326333646531653935633935373362303732653565656438 +36626330623063373832333739643666396135653838643164376264393863646433353036643731 +34623636623366353466333237373265323936643134613230326565303663306431373634356334 +37613732393262626135306632623336613364323632316636623037656538313236666562633438 +37306361333130376538366232393637646263656239303431623963626265386634313735373162 +64393733303132383464386330626135336237303563326238633437346164333738646431333730 +62616533363638643261306565373835303832313539656564386132393863373039653938616261 +36343031356361336563316230343433633033353130373933313361326164633765316433616162 +62356338393634653330623933313433343630386264396337373532376335316362333164363963 +363663666365336564663232363462613162 diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example new file mode 100644 index 0000000..c95234d --- /dev/null +++ b/ansible/infra_secrets.yml.example @@ -0,0 +1,38 @@ +# Uptime Kuma login credentials +# Used by the disk monitoring playbook to create monitors automatically + + +# ntfy credentials +# Used for notification channel setup in Uptime Kuma + +ntfy_username: "your_ntfy_username" +ntfy_password: "your_ntfy_password" + +# headscale-ui credentials +# Used for HTTP basic authentication via Caddy +# Provide either: +# - headscale_ui_password: plain text password (will be hashed automatically) +# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate) + +headscale_ui_username: "admin" +headscale_ui_password: "your_secure_password_here" +# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password + +bitcoin_rpc_user: "bitcoinrpc" +bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" + +# Mempool MariaDB credentials +# Used by: services/mempool/deploy_mempool_playbook.yml +mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" + +# Forgejo Runner registration token +# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml +# See: services/forgejo-runner/SETUP.md for how to obtain this token +forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE" + +# DATUM Gateway secrets +# Used by: services/datum-gateway/deploy_datum_gateway_playbook.yml +datum_mining_address: "YOUR_BITCOIN_ADDRESS_FOR_BLOCK_REWARDS" +datum_gateway_admin_password: "CHANGE_ME_TO_SECURE_PASSWORD" +datum_dashboard_username: "admin" +datum_dashboard_password_hash: "$2a$14$..." # Generate with: caddy hash-password diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml new file mode 100644 index 0000000..36d35f8 --- /dev/null +++ b/ansible/infra_vars.yml @@ -0,0 +1,9 @@ +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: contrapeso.xyz + +# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the +# playbooks are kept deliberately — the check logic is meant to be rewired to +# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. +uptime_kuma_enabled: false diff --git a/ansible/inventory.ini b/ansible/inventory.ini index c39439d..b9b2906 100644 --- a/ansible/inventory.ini +++ b/ansible/inventory.ini @@ -1,7 +1,7 @@ [vps] vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -monitoring ansible_host=64.226.70.190 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua [nodito_host] nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua @@ -15,10 +15,11 @@ memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counter forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +nonkeiwaisi_local ansible_host=nonkeiwaisi-box lan_ip=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua # Local connection to laptop: this assumes you're running ansible commands from your personal laptop [lapy] -localhost ansible_connection=local ansible_user=counterweight +localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C [arbret] prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua @@ -26,12 +27,8 @@ prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 [edge] vipy -# The group is `observability`, NOT `monitoring` — there is a HOST named -# `monitoring` on line 5, and a group with the same name makes `hosts: monitoring` -# ambiguous. Ansible resolved it to the host and warned: -# [WARNING]: Found both group and host with same name: monitoring -[observability] -monitoring +[monitoring] +watchtower [vpn_control] spacey @@ -67,7 +64,7 @@ nodito_vms # Hosts that run Caddy and therefore have /etc/caddy/sites-enabled. [caddy:children] edge -observability +monitoring vpn_control [backup_store] diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 0be3807..0686ab6 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -7,10 +7,6 @@ ansible.builtin.include_role: name: backup_store vars: - # check-backups.sh reports one result per source plus one for the store - # itself, so it needs the collection URL and appends each key. - backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" backup_store_sources: - name: arbret source: "arbret@prd-arbret:/opt/arbret/backups/" @@ -30,78 +26,3 @@ - name: forgejo source: "backup-pull@prd-vipy:/opt/backups/forgejo/" retention_days: 14 - -# ───────────────────────────────────────────────────────────────────────────── -# Register the backup checks with Gatus. -# -# Two groups on purpose, because they answer different questions and fail for -# different reasons: -# -# backup-dump did the SOURCE produce an artefact? Pushed by each dump right -# after it runs, so a broken dump is visible within minutes. -# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is -# retention pruning? Pushed by check-backups.sh at 05:30. -# -# The store alone could catch almost everything, because the artefact filename -# carries the source's dump timestamp - a source whose timer died still pulls -# "ok" forever, but the timestamp gives it away. What the source side adds is -# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it -# cannot tell you whether the dump broke or the pull did. -# -# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` -# deliberately excludes, so nothing of ours runs there. It is store-checked only. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the backup checks with Gatus - hosts: observability - become: yes - vars: - # Sources we deploy the dump for, and the host each one runs on. - dump_sources: - - {name: headscale, host: spacey} - - {name: memos, host: memos_box_local} - - {name: vaultwarden, host: vipy} - - {name: lnbits, host: vipy} - - {name: forgejo, host: vipy} - store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] - - tasks: - # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter - # than a day plus slack. The store-side check catches the same fault within - # 6h by reading the artefact's dump timestamp out of the filename, so this is - # the slow backstop rather than the primary signal. - - name: Build the dump endpoint list - ansible.builtin.set_fact: - dump_endpoints: "{{ dump_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'backup-dump', - 'token': gatus_push_tokens[item.host], - 'heartbeat': '26h'}] }}" - loop: "{{ dump_sources }}" - - - name: Build the store endpoint list - ansible.builtin.set_fact: - store_endpoints: "{{ store_endpoints | default([]) + [{ - 'name': item, - 'group': 'backup-store', - 'token': gatus_push_tokens['small_backups_local'], - 'heartbeat': '7h'}] }}" - loop: "{{ store_sources }}" - - - name: Register the backup endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - # Push endpoints: the heartbeat window is the tolerance, so alert on - # the first failure rather than waiting for three 7h windows to pass. - gatus_endpoint_default_alerts: - - type: signal - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 12h - gatus_endpoint_name: backups - gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ - 'name': 'pull job', - 'group': 'backup-store', - 'token': gatus_push_tokens['small_backups_local'], - 'heartbeat': '7h'}] }}" diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml index b195190..7de6006 100644 --- a/ansible/roles/backup_source/defaults/main.yml +++ b/ansible/roles/backup_source/defaults/main.yml @@ -29,16 +29,3 @@ backup_source_retention_days: 7 # Schedule. The box pulls at 04:00, so dumps must land before that. backup_source_on_calendar: "*-*-* 02:00:00" - -# ── Reporting ──────────────────────────────────────────────────────────────── -# Where to report that this dump ran and produced a plausible artefact. -# Gatus external endpoint: -# POST {url}?success=true|false&error=... -# Authorization: Bearer {token} -# Empty is valid and is not an error: the unit's exit code is still the answer, -# and the STORE will independently notice a stale dump within ~26h because the -# artefact filename carries this dump's timestamp. Reporting here only buys -# earlier detection and tells you it was the DUMP that broke rather than the -# pull. -backup_source_push_url: "" -backup_source_push_token: "" diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index a964c5d..4283b84 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -16,27 +16,12 @@ # Declared here rather than assumed. Stage 1 installed it by hand; this is what # makes a rebuilt host get it too. -# Cache refresh is best-effort on purpose. An unrelated third-party repo with a -# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource -# key) makes `apt-get update` return warnings, which the apt module treats as a -# hard failure — and that must not stop backups being configured. Installing the -# package is NOT best-effort: if age is genuinely unavailable, the next task fails. -- name: Refresh the apt cache (best effort) - ansible.builtin.apt: - update_cache: yes - cache_valid_time: 3600 - failed_when: false - changed_when: false - - name: Ensure age is installed ansible.builtin.apt: - name: - - age - # curl is needed only when backup_source_push_url is set, but installing it - # unconditionally keeps the task idempotent and it is present on every - # Debian host here anyway. - - curl + name: age state: present + update_cache: yes + cache_valid_time: 3600 # The pull account: unprivileged, no sudo, exists only so small-backups-box can # read the dump directory. Trust points one way — the box can read backups, and @@ -88,18 +73,14 @@ mode: '0750' validate: "bash -n %s" -# The .service carries the push token in an Environment= line, so it is 0600. -# The .timer holds nothing secret and stays world-readable. - name: "Install the {{ backup_source_name }}-backup systemd units" ansible.builtin.template: - src: "backup.{{ item.unit }}.j2" - dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}" + src: "backup.{{ item }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" owner: root group: root - mode: "{{ item.mode }}" - loop: - - {unit: service, mode: "0600"} - - {unit: timer, mode: "0644"} + mode: '0644' + loop: [service, timer] notify: Reload systemd for backup units - name: "Enable the {{ backup_source_name }}-backup timer" diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 index 7a1df21..91ad063 100644 --- a/ansible/roles/backup_source/templates/backup.service.j2 +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -9,8 +9,6 @@ After={{ backup_source_stop_service if '.' in backup_source_stop_service else ba [Service] Type=oneshot ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh -Environment=BACKUP_PUSH_URL={{ backup_source_push_url }} -Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }} StandardOutput=journal StandardError=journal SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index 018c313..18fabd6 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -41,76 +41,18 @@ chmod 700 "$BACKUP_DIR" # here or they accumulate forever. rm -f "${BACKUP_DIR}/${NAME}_"*.partial -# --- Reporting ------------------------------------------------------------- -# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed -# backup even though the script "finished". Both are reported as failures. -PUSH_URL="${BACKUP_PUSH_URL:-}" -PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}" - -report() { - local success="$1" message="$2" - [ -n "$PUSH_URL" ] || return 0 - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true -} - -# Reports on ANY exit path, so a dump that dies halfway still reports rather -# than going quiet. The size of the FINISHED artefact decides success, not -# merely reaching the end of the script. -# -# This is called FROM the single EXIT trap below - it must never register an -# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather -# than adding to it, so a second trap here silently discards the one that -# restarts the service, and a backup run leaves the service stopped. That is -# precisely the failure the restart trap exists to prevent. -report_outcome() { - local rc="$1" - if [ "$rc" -ne 0 ]; then - report "false" "${NAME} dump exited ${rc}" - elif [ ! -s "$ARTIFACT" ]; then - report "false" "${NAME} produced no artefact at ${ARTIFACT}" - else - report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)" - fi -} - -# --- One EXIT handler, doing both jobs ------------------------------------- -# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous -# handler rather than adding to it. Registering a second one here would -# silently discard the service restart and leave the service stopped after -# every backup - which is the exact bug the restart exists to prevent, and it -# is invisible until someone notices the service is down. -on_exit() { - local rc=$? {% if backup_source_stop_service or backup_source_stop_command %} - log "Restarting ${SERVICE}..." - eval "$START_CMD" || true -{% endif %} - report_outcome "$rc" -} -trap on_exit EXIT - -{% if backup_source_stop_service or backup_source_stop_command %} -# --- Stop the service; the trap above guarantees it comes back ------------- +# --- Stop the service, and guarantee it comes back --- # The trap is the point: without it a failed dump leaves the service down until # the next timer fires. Every hand-written script this replaced had that bug. -# It is armed BEFORE the stop, so even a failure during the stop restarts. log "Stopping ${SERVICE}..." eval "$STOP_CMD" +trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT {% endif %} # --- Dump straight into age; plaintext never touches the disk --- log "Writing ${ARTIFACT}..." {{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" -{% if backup_source_pull_user %} -# Match the final ownership immediately, so even a partial left by a later -# failure is not an unreadable obstacle to the pull. -chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial" -chmod 640 "${ARTIFACT}.partial" -{% endif %} mv "${ARTIFACT}.partial" "$ARTIFACT" {% if backup_source_pull_user %} # Readable by the pull account and nobody else. The contents are age-encrypted diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml index e6d17a3..c0c128b 100644 --- a/ansible/roles/backup_store/defaults/main.yml +++ b/ansible/roles/backup_store/defaults/main.yml @@ -9,21 +9,3 @@ backup_store_on_calendar: "*-*-* 04:00:00" # source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" # retention_days: 90 backup_store_sources: [] - -# ── Reporting ──────────────────────────────────────────────────────────────── -# check-backups.sh reports one result PER SOURCE plus one for the store itself, -# so the base URL is the endpoints collection and the script appends each key. -# Empty is valid: the script still prints its report and exits 0/1. -backup_store_check_push_base: "" -backup_store_check_push_token: "" - -# Every six hours, offset past the 04:00 pull so the first run of the day sees a -# finished pull. The BACKUPS are daily, but this check is not - it reads the -# source's dump timestamp out of the artefact filename, so running it more often -# catches "the source stopped dumping" within hours rather than a day, and lets -# the Gatus heartbeat be 7h instead of 30h. -backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00" - -# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the -# pull is at 04:00, so 26h tolerates exactly one missed night before alarming. -backup_store_check_max_age_hours: 26 diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml index 55500c6..d138830 100644 --- a/ansible/roles/backup_store/tasks/main.yml +++ b/ansible/roles/backup_store/tasks/main.yml @@ -33,42 +33,6 @@ validate: "bash -n %s" become: yes -# An assertion that last night actually worked. Generated from the same source -# list as the puller, so it can never drift out of sync with what is supposed to -# be arriving. Runs on a timer AND is useful by hand. -- name: Install the backup check script - ansible.builtin.template: - src: check-backups.sh.j2 - dest: /usr/local/bin/check-backups.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - become: yes - -# The .service carries the push token, so it is 0600; the .timer is not secret. -- name: Install the check-backups systemd units - ansible.builtin.template: - src: "check-backups.{{ item.unit }}.j2" - dest: "/etc/systemd/system/check-backups.{{ item.unit }}" - owner: root - group: root - mode: "{{ item.mode }}" - loop: - - {unit: service, mode: "0600"} - - {unit: timer, mode: "0644"} - become: yes - -# restarted, not started: `started` is a no-op on an already-active timer, so a -# changed schedule would never be picked up. -- name: Enable the check-backups timer - ansible.builtin.systemd: - name: check-backups.timer - enabled: yes - state: restarted - daemon_reload: yes - become: yes - - name: Install the pull-backups systemd units ansible.builtin.template: src: "pull-backups.{{ item }}.j2" diff --git a/ansible/roles/backup_store/templates/check-backups.service.j2 b/ansible/roles/backup_store/templates/check-backups.service.j2 deleted file mode 100644 index 3e50b6a..0000000 --- a/ansible/roles/backup_store/templates/check-backups.service.j2 +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Verify the nightly backup pull actually worked -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -User={{ ansible_user_id }} -ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }} -Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }} -Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 deleted file mode 100644 index 37c9916..0000000 --- a/ansible/roles/backup_store/templates/check-backups.sh.j2 +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash -# Assert the nightly backups actually worked. -# -# Run as {{ ansible_user_id }} on this host. Needs no sudo. -# -# What it CANNOT do: verify contents. The age identity lives only on lapy, so -# this host cannot decrypt anything it holds — by design. These are freshness, -# completeness and integrity checks. To verify content, decrypt on lapy: -# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups//" \ -# | age -d -i ~/.age/counterweight_age | tar -tzf - | head -# -# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL. -set -uo pipefail - -STORE="{{ backup_store_dir }}" -MAX_AGE_H="${1:-26}" # an artefact older than this is stale -NOW=$(date +%s) -fails=0; warns=0 - -# Colour only when attached to a terminal: this gets piped into files and, later, -# probably into a notification. -if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' -else R=''; Y=''; G=''; N=''; fi - -# Per-source verdicts, so each source can be reported independently. A single -# aggregate red light tells you backups are broken; it does not tell you which -# one, which is the thing you need at 3am. -declare -A SRC_FAIL SRC_MSG -CURRENT="" - -red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); - [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; } -yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } -ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; - [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; } - -# --- Reporting ------------------------------------------------------------- -# Each source gets its own Gatus external endpoint, plus one for the store -# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means -# report nowhere, which is valid: the exit code is still the whole answer. -PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}" -PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}" - -report() { - local key="$1" success="$2" message="$3" - [ -n "$PUSH_BASE" ] || return 0 - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true -} - -hours_since() { echo $(( (NOW - $1) / 3600 )); } - -# Pull the dump timestamp out of _YYYYmmdd_HHMMSS.. This is when -# the SOURCE produced it, which is the thing that actually matters: a source -# whose timer died still pulls "ok" forever, because yesterday's artefact is -# still sitting there. Checking only the pull would miss exactly that. -dump_epoch() { - local base ts - base=$(basename "$1") - ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1 - [ -n "$ts" ] || return 1 - date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null -} - -check_source() { - local name="$1" keep="$2" dir="$STORE/$1" - printf '\n%s\n' "== $name" - CURRENT="$name" - SRC_FAIL[$name]=0 - SRC_MSG[$name]="" - - [ -d "$dir" ] || { red "$name: no directory $dir"; return; } - - local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l) - [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; } - - local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l) - [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these" - - local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1) - local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p) - - # 1. Did the SOURCE dump recently? - local de; de=$(dump_epoch "$newest") - if [ -z "${de:-}" ]; then - yell "$name: cannot parse a dump timestamp from $(basename "$newest")" - else - local dh; dh=$(hours_since "$de") - if [ "$dh" -lt 0 ]; then - # A future-dated artefact would otherwise stay "fresh" forever and the - # staleness check would never fire again — the exact silent failure this - # script exists to catch. - red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?" - elif [ "$dh" -gt "$MAX_AGE_H" ]; then - red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run" - else - ok "$name: dumped ${dh}h ago" - fi - fi - - # 2. Did the PULL bring it over recently? - local ph; ph=$(hours_since "$(stat -c %Y "$newest")") - if [ "$ph" -gt "$MAX_AGE_H" ]; then - red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)" - else - ok "$name: pulled ${ph}h ago" - fi - - # 3. Is it plausibly a real backup? - local sz; sz=$(stat -c %s "$newest") - if [ "$sz" -eq 0 ]; then - red "$name: newest artefact is ZERO bytes" - elif [ -n "$prev" ]; then - local psz; psz=$(stat -c %s "$prev") - if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then - # Not automatically wrong: headscale legitimately shrank 297K -> 20K when - # a clean stop checkpointed its write-ahead log into the database. - yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect" - else - ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)" - fi - else - ok "$name: $(numfmt --to=iec "$sz") (first artefact)" - fi - - # 4. Is retention pruning? Allow generous slack for multiple dumps per day. - if [ "$n" -gt $(( keep * 3 + 10 )) ]; then - yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working" - fi -} - -echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" -echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." - -# --- the pull job itself --- -# Reserved key, reported as backup-store_pull-job. The store's own machinery is -# a different alarm from any one source being stale, and it is a LEADING -# indicator where the per-source checks are lagging ones: those only fire once -# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer, -# a failed pull job or a filling disk are all visible here immediately, and they -# name the cause instead of showing six stale sources with no explanation. -CURRENT="__store" -SRC_FAIL[__store]=0 -SRC_MSG[__store]="" -printf '\n%s\n' "== pull-backups.service" -result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) -status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) -when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null) -[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})" -if [ -n "$when" ]; then - wh=$(hours_since "$(date -d "$when" +%s)") - [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?" -fi -systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ - && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \ - || red "pull-backups.timer is NOT enabled" - -# --- each source --- -{% for src in backup_store_sources %} -check_source "{{ src.name }}" {{ src.retention_days }} -CURRENT="" -# The store key is what Gatus computes from group+name: sanitize("backup-store") -# + "_" + sanitize("{{ src.name }}"). -report "backup-store_{{ src.name }}" \ - "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \ - "${SRC_MSG[{{ src.name }}]:-no result}" -{% endfor %} - -# --- capacity --- -CURRENT="__store" -printf '\n%s\n' "== disk" -use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') -avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') -if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" -elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" -else ok "store is ${use}% full, ${avail} free"; fi - -CURRENT="" -report "backup-store_pull-job" \ - "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \ - "${SRC_MSG[__store]:-no result}" - -printf '\n%s\n' "-----" -if [ "$fails" -gt 0 ]; then - echo "RESULT: $fails failure(s), $warns warning(s)" - echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager" - exit 1 -fi -if [ "$warns" -gt 0 ]; then - echo "RESULT: all checks passed, $warns warning(s)" -else - echo "RESULT: all checks passed" -fi -exit 0 diff --git a/ansible/roles/backup_store/templates/check-backups.timer.j2 b/ansible/roles/backup_store/templates/check-backups.timer.j2 deleted file mode 100644 index ba5c5a8..0000000 --- a/ansible/roles/backup_store/templates/check-backups.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Run the backup verification after the nightly pull -Requires=check-backups.service - -[Timer] -OnCalendar={{ backup_store_check_on_calendar }} -# Run a missed occurrence on the next boot rather than skipping the day. -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 index 1db7d9c..267900c 100644 --- a/ansible/roles/backup_store/templates/pull-backups.sh.j2 +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -22,12 +22,7 @@ pull_{{ src.name | replace('-', '_') }}() { local dir="${STORE}/{{ src.name }}" mkdir -p "$dir" log "Pulling {{ src.name }} from {{ src.source }}..." - # --exclude '*.partial': a dump that died mid-write leaves one behind, owned - # root:root 0600 because the chown only happens after a successful mv. Without - # this exclude the pull account cannot read it and rsync fails for the WHOLE - # source — so one failed dump would silently block every subsequent pull of - # that service. An incomplete artefact is never worth transferring anyway. - if rsync -az --timeout=120 --exclude '*.partial' \ + if rsync -az --timeout=120 \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ "{{ src.source }}" "$dir/"; then log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" diff --git a/ansible/roles/bitcoin_knots/README.md b/ansible/roles/bitcoin_knots/README.md deleted file mode 100644 index 8e35900..0000000 --- a/ansible/roles/bitcoin_knots/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# `bitcoin_knots` - -Builds Bitcoin Knots from source with PGP + SHA256 verification of the release -tarball, runs it as a full node on `knots-box`, and keeps a health check on a -systemd timer. The second play in the calling playbook publishes the P2P port -from the edge host via `socket_proxy`. - -Converted from `deploy_bitcoin_knots_playbook.yml` (892 lines) under Plan 6. The -playbook is now 40 lines. - -## The build is guarded; the chain is never touched - -`build.yml` is 32 tasks, every one carrying -`when: not bitcoind_binary_exists.stat.exists`. On a host that already has the -binary the whole download / verify / 30-60 minute compile skips — **including the -two `state: absent` deletions**, which target `/opt/bitcoin-knots/source` and the -extracted build directory. - -The chain lives elsewhere and nothing here touches it: - -| | | -|---|---| -| `bitcoin_knots_dir` | `/opt/bitcoin-knots` — build tree, safe to delete | -| `bitcoin_data_dir` | `/var/lib/bitcoin` — config, logs, wallets | -| `bitcoin_large_data_dir` | `/mnt/knots_data` — **~875 GB of blockchain** | - -The signature-verification tasks are the security control of this role. They are -copied verbatim; do not "simplify" them. - -## ⚠ This node is half of the mining setup - -`bitcoin.conf` carries a DATUM Gateway section that was hand-added on the node -and was **missing from the playbook's template**: - -```ini -blockmaxsize=3985000 -blockmaxweight=3985000 -blocknotify=killall -USR1 datum_gateway -maxmempool=1000 -blockreconstructionextratxn=1000000 -``` - -`blocknotify` is how `datum_gateway` learns a new block landed. Running the old -playbook would have deleted all of it, and solo mining would have carried on -grinding against a stale template — a silent failure that costs money rather -than raising an error. The template now carries it behind -`bitcoin_datum_gateway_enabled`. - -**bitcoin-knots and datum-gateway are one system, not two services.** Changing -either config means thinking about both. - -## The restart handler, and why exactness matters now - -The hand-written `Restart bitcoind` handler carried -`when: uptime_kuma_enabled | default(false)`, so it had been inert since the -decommissioning: `bitcoin.conf` and the systemd unit both notify it and neither -could restart anything. A config change applied to disk, reported success, and -never took effect. - -It is ungated here — which raises the bar for the template. **Any** residual -difference between the template and the live file, down to a trailing newline, -means the task reports `changed` and restarts a Bitcoin node on every run. It -took four rounds of `--check --diff` to reach `changed=0`: the DATUM section, an -explanatory comment that was rendering into the deployed file (now a `{# #}` -Jinja comment), a `# Pruning (optional)` comment the live file had, and one -trailing blank line. - -## `dbcache` - -Computed as 90% of RAM unless `bitcoin_dbcache_mb_override` is set. The live node -was hand-tuned to **200 MB**; the calculation produces 3528. As with fulcrum, -`set_fact` outranks role defaults, so the *calculation* honours the override — a -value pinned only in `defaults/` is silently ignored. - -## Monitoring: one variable, no product knowledge - -The check tests bitcoind's RPC and records the answer in its exit code, which -systemd keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. Set -`healthcheck_push_url` to report anywhere that accepts an HTTP ping. - -The timer had last fired **2026-08-09** while still reporting `active` and -`enabled` — the same `OnBootSec` + `OnUnitActiveSec` dead chain as fulcrum, where -nothing re-arms it if the service does not run in a given boot. The role runs the -check once after enabling, which both smoke-tests it and supplies the reference -the timer schedules from. diff --git a/ansible/roles/bitcoin_knots/defaults/main.yml b/ansible/roles/bitcoin_knots/defaults/main.yml deleted file mode 100644 index 1045e77..0000000 --- a/ansible/roles/bitcoin_knots/defaults/main.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Bitcoin Knots Configuration Variables - -# Version - REQUIRED: Specify exact version/tag to build -# The only version string. There used to be a second, v-prefixed copy -# (bitcoin_knots_version) that nothing read - two hand-maintained copies of one -# fact, with nothing keeping them in step. -bitcoin_knots_version_short: "29.2.knots20251110" - -# Directories -bitcoin_knots_dir: /opt/bitcoin-knots -bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" -bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets -bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) -bitcoin_conf_dir: /etc/bitcoin - -# Network -bitcoin_rpc_port: 8332 -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -bitcoin_p2p_port: 8333 -bitcoin_rpc_bind: "0.0.0.0" - -# Build options -bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores -bitcoin_build_prefix: /usr/local - -# Configuration options -bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) -bitcoin_max_connections: 125 -# dbcache will be calculated as 90% of host RAM automatically in playbook - -# ZMQ Configuration -bitcoin_zmq_enabled: true -bitcoin_zmq_bind: "tcp://0.0.0.0" -bitcoin_zmq_port_rawblock: 28332 -bitcoin_zmq_port_rawtx: 28333 -bitcoin_zmq_port_hashblock: 28334 -bitcoin_zmq_port_hashtx: 28335 - -# Service user -bitcoin_user: bitcoin -bitcoin_group: bitcoin - -# --- Health check ---------------------------------------------------------- -# Checks bitcoind RPC and records the answer in its exit code, which systemd -# keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. -# -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means -# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping -# works; nothing here is specific to a monitoring product. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" - -# --- Logging ---------------------------------------------------------------- -# The live node logs to a file. Set to "" to use printtoconsole=1 (journald). -bitcoin_logfile: "{{ bitcoin_data_dir }}/debug.log" - -# --- dbcache ---------------------------------------------------------------- -# Computed as 90% of RAM unless this is set. The live node was hand-tuned to -# 200 MB; the calculation would have produced 3528. As with fulcrum, note that -# set_fact outranks role defaults, so the CALCULATION has to honour this - a -# value pinned only in defaults/ is silently ignored. -bitcoin_dbcache_mb_override: 200 - -# --- DATUM Gateway ---------------------------------------------------------- -# This node feeds block templates to datum_gateway on knots-box. These settings -# were hand-added to bitcoin.conf and were missing from the template, so a -# playbook run would have removed them and broken the mining setup. -bitcoin_datum_gateway_enabled: true -bitcoin_blockmaxsize: 3985000 -bitcoin_blockmaxweight: 3985000 -bitcoin_blocknotify: "killall -USR1 datum_gateway" -bitcoin_maxmempool: 1000 -bitcoin_blockreconstructionextratxn: 1000000 diff --git a/ansible/roles/bitcoin_knots/handlers/main.yml b/ansible/roles/bitcoin_knots/handlers/main.yml deleted file mode 100644 index 49ac219..0000000 --- a/ansible/roles/bitcoin_knots/handlers/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Ungated on purpose. The hand-written handler carried -# when: uptime_kuma_enabled | default(false) -# so it has been inert since the decommissioning. Two tasks notify it — -# bitcoin.conf and the systemd unit — and neither could actually restart -# bitcoind. A configuration change to a Bitcoin node therefore applied to disk, -# reported success, and silently never took effect. -# -# Restarting bitcoind re-opens the chainstate; it does not reindex. -- name: Restart bitcoind - systemd: - name: bitcoind - state: restarted - daemon_reload: yes diff --git a/ansible/roles/bitcoin_knots/tasks/build.yml b/ansible/roles/bitcoin_knots/tasks/build.yml deleted file mode 100644 index 5a1b900..0000000 --- a/ansible/roles/bitcoin_knots/tasks/build.yml +++ /dev/null @@ -1,222 +0,0 @@ ---- -# Every task here is guarded by `when: not bitcoind_binary_exists.stat.exists`, -# so on a host that already has the binary the whole download / verify / build -# sequence skips — including the two `state: absent` deletions, which target -# /opt/bitcoin-knots/{source,bitcoin-} and never the chain data in -# /mnt/knots_data. -- name: Check if bitcoind binary already exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_binary_exists - changed_when: false - -- name: Install gnupg for signature verification - apt: - name: gnupg - state: present - when: not bitcoind_binary_exists.stat.exists - -- name: Import Luke Dashjr's Bitcoin Knots signing key - command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 - register: key_import - changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" - when: not bitcoind_binary_exists.stat.exists - failed_when: key_import.rc != 0 - -- name: Display imported key fingerprint - command: gpg --fingerprint 90C8019E36C2E964 - register: key_fingerprint - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Download SHA256SUMS file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - -- name: Download SHA256SUMS.asc signature file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - -- name: Verify PGP signature on SHA256SUMS file - command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS - register: sha256sums_verification - changed_when: false - failed_when: false # Don't fail here - check for 'Good signature' in next task - when: not bitcoind_binary_exists.stat.exists - - -- name: Display SHA256SUMS verification result - debug: - msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" - when: not bitcoind_binary_exists.stat.exists - -- name: Fail if SHA256SUMS signature verification failed - fail: - msg: "SHA256SUMS signature verification failed. Aborting build." - when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) - -- name: Remove any existing tarball to force fresh download - file: - path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - state: absent - when: not bitcoind_binary_exists.stat.exists - -- name: Download Bitcoin Knots source tarball - get_url: - url: "{{ bitcoin_source_tarball_url }}" - dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - mode: '0644' - validate_certs: yes - force: yes - when: not bitcoind_binary_exists.stat.exists - -- name: Calculate SHA256 checksum of downloaded tarball - command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - register: tarball_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Extract expected checksum from SHA256SUMS file - shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' - register: expected_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - failed_when: expected_checksum.stdout == "" - -- name: Display checksum comparison - debug: - msg: - - "Expected: {{ expected_checksum.stdout | trim }}" - - "Actual: {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists - -- name: Verify tarball checksum matches SHA256SUMS - fail: - msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] - -- name: Remove existing source directory if it exists (to force fresh extraction) - file: - path: "{{ bitcoin_knots_source_dir }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - -- name: Remove extracted directory if it exists (from previous runs) - file: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - -- name: Extract verified source tarball - unarchive: - src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - dest: "{{ bitcoin_knots_dir }}" - remote_src: yes - when: not bitcoind_binary_exists.stat.exists - -- name: Check if extracted directory exists - stat: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - register: extracted_dir_stat - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Rename extracted directory to expected name - command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" - when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists - -- name: Check if CMakeLists.txt exists - stat: - path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" - register: cmake_exists - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Create CMake build directory - file: - path: "{{ bitcoin_knots_source_dir }}/build" - state: directory - mode: '0755' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - -- name: Configure Bitcoin Knots build with CMake - command: > - cmake - -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} - -DBUILD_BITCOIN_WALLET=OFF - -DCMAKE_BUILD_TYPE=Release - -DWITH_ZMQ=ON - .. - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: configure_result - changed_when: true - -- name: Verify CMake enabled ZMQ - shell: | - set -e - cd "{{ bitcoin_knots_source_dir }}/build" - cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: zmq_check - changed_when: false - -- name: Fail if CMakeLists.txt not found - fail: - msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." - when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) - -- name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) - command: cmake --build . -j{{ bitcoin_build_jobs }} - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - async: 3600 - poll: 0 - register: build_result - changed_when: true - -- name: Check build status - async_status: - jid: "{{ build_result.ansible_job_id }}" - register: build_job_result - until: build_job_result.finished - retries: 120 - delay: 60 - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined - -- name: Fail if build failed - fail: - msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) - -- name: Install Bitcoin Knots binaries - command: cmake --install . - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - changed_when: true - -- name: Verify bitcoind binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_installed - changed_when: false - -- name: Verify bitcoin-cli binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" - register: bitcoin_cli_installed - changed_when: false - -- name: Fail if binaries not found - fail: - msg: "Bitcoin Knots binaries not found after installation" - when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists diff --git a/ansible/roles/bitcoin_knots/tasks/configure.yml b/ansible/roles/bitcoin_knots/tasks/configure.yml deleted file mode 100644 index 52fd3fd..0000000 --- a/ansible/roles/bitcoin_knots/tasks/configure.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Ownership copied verbatim from the playbook this replaces; verified -# mechanically against `git show HEAD:` rather than retyped from memory. -- name: Create bitcoin.conf configuration file - ansible.builtin.template: - src: bitcoin.conf.j2 - dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0640' - notify: Restart bitcoind - -- name: Create systemd service file for bitcoind - ansible.builtin.template: - src: bitcoind.service.j2 - dest: /etc/systemd/system/bitcoind.service - owner: root - group: root - mode: '0644' - notify: Restart bitcoind diff --git a/ansible/roles/bitcoin_knots/tasks/healthcheck.yml b/ansible/roles/bitcoin_knots/tasks/healthcheck.yml deleted file mode 100644 index c6c8db3..0000000 --- a/ansible/roles/bitcoin_knots/tasks/healthcheck.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -# Everything here answers "is bitcoind healthy" and records the answer. The -# Uptime Kuma specifics that used to follow — an embedded Python script creating -# monitors over the API, a /tmp credentials file, push-URL extraction and a -# systemd Environment= rewrite — are gone. Where it reports is now one variable, -# healthcheck_push_url. See the role README. -- name: Install curl for health check script - apt: - name: curl - state: present - -- name: Create Bitcoin Knots health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create systemd service for Bitcoin Knots health check - ansible.builtin.template: - src: healthcheck.service.j2 - dest: /etc/systemd/system/bitcoin-knots-healthcheck.service - owner: root - group: root - mode: "0600" - -- name: Create systemd timer for Bitcoin Knots health check - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer - owner: root - group: root - mode: '0644' - -- name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - -- name: Enable and restart the Bitcoin Knots health check timer - systemd: - name: bitcoin-knots-healthcheck.timer - enabled: yes - state: restarted - daemon_reload: yes - -# Runs the check once, which is both a smoke test and the thing that actually -# arms the timer. This timer is OnBootSec + OnUnitActiveSec with no OnCalendar: -# OnBootSec elapses once, and OnUnitActiveSec needs the SERVICE to have run this -# boot to have anything to schedule from. Restarting the timer does not supply -# that reference; running the service does. The live timer had last fired on -# 2026-08-09 while still reporting `active` and `enabled`. -- name: Run the Bitcoin Knots health check once to arm the timer - command: systemctl start bitcoin-knots-healthcheck.service - changed_when: false diff --git a/ansible/roles/bitcoin_knots/tasks/install.yml b/ansible/roles/bitcoin_knots/tasks/install.yml deleted file mode 100644 index 3003b6e..0000000 --- a/ansible/roles/bitcoin_knots/tasks/install.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -- name: Calculate dbcache as a share of system RAM - set_fact: - bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" - when: bitcoin_dbcache_mb_override | string | length == 0 - -- name: Use the explicit dbcache override - set_fact: - bitcoin_dbcache_mb: "{{ bitcoin_dbcache_mb_override }}" - when: bitcoin_dbcache_mb_override | string | length > 0 - changed_when: false - -- name: Display calculated dbcache value - debug: - msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" - - -- name: Install build dependencies - apt: - name: - - build-essential - - libtool - - autotools-dev - - automake - - pkg-config - - bsdmainutils - - python3 - - python3-pip - - libevent-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-test-dev - - libboost-thread-dev - - libboost-chrono-dev - - libboost-program-options-dev - - libboost-dev - - libssl-dev - - libdb-dev - - libminiupnpc-dev - - libzmq3-dev - - libnatpmp-dev - - libsqlite3-dev - - git - - curl - - wget - - cmake - state: present - update_cache: yes - -- name: Create bitcoin group - group: - name: "{{ bitcoin_group }}" - system: yes - state: present - -- name: Create bitcoin user - user: - name: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ bitcoin_data_dir }}" - create_home: yes - state: present - -- name: Create bitcoin-knots directory - file: - path: "{{ bitcoin_knots_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Create bitcoin-knots source directory - file: - path: "{{ bitcoin_knots_source_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Create bitcoin data directory (for config, logs, wallets) - file: - path: "{{ bitcoin_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - -- name: Create bitcoin large data directory (for blockchain) - file: - path: "{{ bitcoin_large_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - -- name: Create bitcoin config directory - file: - path: "{{ bitcoin_conf_dir }}" - state: directory - owner: root - group: root - mode: '0755' diff --git a/ansible/roles/bitcoin_knots/tasks/main.yml b/ansible/roles/bitcoin_knots/tasks/main.yml deleted file mode 100644 index ab869a7..0000000 --- a/ansible/roles/bitcoin_knots/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: build.yml -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/bitcoin_knots/tasks/service.yml b/ansible/roles/bitcoin_knots/tasks/service.yml deleted file mode 100644 index 6f6149e..0000000 --- a/ansible/roles/bitcoin_knots/tasks/service.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start bitcoind service - systemd: - name: bitcoind - enabled: yes - state: started - -- name: Wait for bitcoind RPC to be available - uri: - url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" - method: POST - body_format: json - body: - jsonrpc: "1.0" - id: "healthcheck" - method: "getblockchaininfo" - params: [] - user: "{{ bitcoin_rpc_user }}" - password: "{{ bitcoin_rpc_password }}" - status_code: 200 - timeout: 10 - register: rpc_check - until: rpc_check.status == 200 - retries: 30 - delay: 5 - ignore_errors: yes - -- name: Display RPC connection status - debug: - msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" - -# ═════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 deleted file mode 100644 index 5277a3c..0000000 --- a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 +++ /dev/null @@ -1,67 +0,0 @@ -# Bitcoin Knots Configuration -# Generated by Ansible - -# Data directory (blockchain storage) -datadir={{ bitcoin_large_data_dir }} - -# RPC Configuration -server=1 -rpcuser={{ bitcoin_rpc_user }} -rpcpassword={{ bitcoin_rpc_password }} -rpcbind={{ bitcoin_rpc_bind }} -rpcport={{ bitcoin_rpc_port }} -rpcallowip=0.0.0.0/0 - -# Network Configuration -listen=1 -port={{ bitcoin_p2p_port }} -maxconnections={{ bitcoin_max_connections }} - -# Performance -dbcache={{ bitcoin_dbcache_mb }} - -# Transaction Index (optional) -{% if bitcoin_enable_txindex %} -txindex=1 -{% endif %} - -{# The live node carries this comment and the template never produced it, so a - run would have silently deleted it. Harmless in itself, but matching it keeps - this task at `ok` - which means any future `changed` here is a real signal - rather than known noise. #} -# Pruning (optional) - -# Logging -logtimestamps=1 -{% if bitcoin_logfile %} -logfile={{ bitcoin_logfile }} -{% else %} -printtoconsole=1 -{% endif %} - -# ZMQ Configuration -{% if bitcoin_zmq_enabled | default(false) %} -zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} -zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} -zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} -zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} -{% endif %} - -# Security -disablewallet=1 -{% if bitcoin_datum_gateway_enabled %} - -{# These were hand-added on the node and were NOT in this template, so running - the playbook would have stripped them. blocknotify is how datum_gateway - learns a new block landed; without it solo mining keeps grinding on a stale - template - a silent failure that costs money rather than raising an error. - Kept as a Jinja comment so the explanation stays in the repo and out of the - deployed config. #} -# Specific for DATUM gateway -blockmaxsize={{ bitcoin_blockmaxsize }} -blockmaxweight={{ bitcoin_blockmaxweight }} -blocknotify={{ bitcoin_blocknotify }} -maxmempool={{ bitcoin_maxmempool }} -blockreconstructionextratxn={{ bitcoin_blockreconstructionextratxn }} - -{% endif %} diff --git a/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 b/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 deleted file mode 100644 index ac1140f..0000000 --- a/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Bitcoin Knots daemon -After=network.target - -[Service] -Type=simple -User={{ bitcoin_user }} -Group={{ bitcoin_group }} -ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf -Restart=always -RestartSec=10 -TimeoutStopSec=600 -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 deleted file mode 100644 index 9df056e..0000000 --- a/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Bitcoin Knots Health Check -After=network.target bitcoind.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 deleted file mode 100644 index 538b8dd..0000000 --- a/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Bitcoin Knots health check — managed by Ansible (roles/bitcoin_knots) -# -# The exit code is the answer and systemd keeps it: -# systemctl is-failed bitcoin-knots-healthcheck.service -# Reporting anywhere else is optional and generic. -# -# - -RPC_HOST="{{ bitcoin_rpc_bind }}" -RPC_PORT={{ bitcoin_rpc_port }} -RPC_USER="{{ bitcoin_rpc_user }}" -RPC_PASSWORD="{{ bitcoin_rpc_password }}" -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -# Check if bitcoind RPC is responding -check_bitcoind() { - local response - response=$(curl -s --max-time 30 \ - --user "${RPC_USER}:${RPC_PASSWORD}" \ - --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ - --header 'Content-Type: application/json' \ - "http://${RPC_HOST}:${RPC_PORT}" 2>&1) - - if [ $? -eq 0 ]; then - # Check if response contains a non-null error - # Successful responses have "error": null, failures have "error": {...} - if echo "$response" | grep -q '"error":null\|"error": null'; then - return 0 - else - return 1 - fi - else - return 1 - fi -} - -report() { - local status=$1 - local msg=$2 - - # No push URL is normal, not an error: the exit code below is still a - # complete answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - if ! curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${encoded_msg}"; then - return 1 - fi -} - -# Main health check -if check_bitcoind; then - report "up" "OK" - exit 0 -else - report "down" "bitcoind RPC not responding" - exit 1 -fi diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 deleted file mode 100644 index d5857ab..0000000 --- a/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Bitcoin Knots Health Check Timer -Requires=bitcoind.service - -[Timer] -OnBootSec=1min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/caddy_site/defaults/main.yml b/ansible/roles/caddy_site/defaults/main.yml index 0a59906..cafa3dc 100644 --- a/ansible/roles/caddy_site/defaults/main.yml +++ b/ansible/roles/caddy_site/defaults/main.yml @@ -15,8 +15,7 @@ caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"} # expression for the username silently passes through as literal text. caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}] -# Placement. This is now the only definition of caddy_sites_dir - services_config.yml -# used to carry an identical copy, which was removed as redundant. +# Placement. caddy_sites_dir comes from services_config.yml; this is the fallback. caddy_sites_dir: /etc/caddy/sites-enabled # Rendered site files can carry credentials (basic_auth hashes), so --diff is diff --git a/ansible/roles/datum_gateway/README.md b/ansible/roles/datum_gateway/README.md deleted file mode 100644 index cdf35e5..0000000 --- a/ansible/roles/datum_gateway/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# `datum_gateway` - -Builds and runs [DATUM Gateway](https://github.com/OCEAN-xyz/datum_gateway), the -solo/pooled mining gateway, on `knots-box`. The calling playbook adds two more -plays on the edge host: the dashboard via `caddy_site`, and the public Stratum -port via `socket_proxy`. - -Converted from `deploy_datum_gateway_playbook.yml` (802 lines) under Plan 6. The -playbook is now 68 lines and keeps all three plays. - -## ⚠ This is half of a system - -The Bitcoin Knots node on the same host feeds this gateway through -`blocknotify=killall -USR1 datum_gateway` in `bitcoin.conf` — see -`roles/bitcoin_knots/README.md`, where that line was found to be missing from the -template entirely. **Changing either config means thinking about both.** - -Interrupting Stratum costs mining shares. Check before any run that restarts it: - -```bash -ss -tn state established '( sport = :23334 )' -``` - -## Two pieces of drift where the node was right - -The repo and the node had diverged on values that matter, and the deployment -would have applied the repo's: - -| | node (correct) | repo said | -|---|---|---| -| `datum_mining_address` | `bc1qvrj3g84…` | `bc1qdse9dsg…` | -| `pool_pass_workers` / `_full_users` | `false` | `true` | - -The address is the one that would have hurt: **it is where block rewards are -paid**, and unlike fulcrum and bitcoin-knots the `Restart datum-gateway` handler -here was *never* gated, so the change would have applied immediately rather than -sitting inert. Both corrected in the vault and defaults, with notes. - -Verify semantics rather than text when touching `config.json` — render it and -compare parsed JSON, because the live file is single-line and the template is -pretty-printed, so a textual diff is all noise: - -```python -json.load(open('live.json')) == json.load(open('rendered.json')) -``` - -## `config.json` holds real secrets — diff is suppressed - -The file carries `bitcoind.rpcpassword` and `api.admin_password`. `--diff` -prints rendered content, so the task sets `diff: false` by default; pass -`-e datum_reveal_config=true` to opt in. - -Note `pool_pass_workers` / `pool_pass_full_users` are **booleans**, not -passwords, despite the names — they control DATUM's pool-password passthrough. -`mining.pool_address` is a Bitcoin address and public by nature. - -## Expect `changed` on the compile every run - -`Configure cmake build` and `Compile datum_gateway` are bare `command:` tasks -with no `changed_when`, so they always report changed and always re-run. The -build is reproducible — `Install datum_gateway binary` sees identical content and -does not replace it, so the installed binary keeps its original timestamp — but -the compile itself is wasted work on every run. That is the idempotent floor, not -drift. - -## Monitoring: one variable, no product knowledge - -The check tests the gateway API and records the answer in its exit code, which -systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. Set -`healthcheck_push_url` to report anywhere accepting an HTTP ping. - -Unlike the other services here, only the health-check *timer* handler was gated -by `uptime_kuma_enabled`; the main deployment restart worked throughout. diff --git a/ansible/roles/datum_gateway/defaults/main.yml b/ansible/roles/datum_gateway/defaults/main.yml deleted file mode 100644 index 0c30aa7..0000000 --- a/ansible/roles/datum_gateway/defaults/main.yml +++ /dev/null @@ -1,61 +0,0 @@ -# DATUM Gateway Configuration Variables -# https://github.com/OCEAN-xyz/datum_gateway - -# Version - pin to a specific tag -datum_gateway_version: "v0.4.1beta" - -# Directories -datum_gateway_dir: /opt/datum-gateway -datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" -datum_gateway_config_dir: /etc/datum-gateway -datum_gateway_log_dir: /var/log/datum-gateway - -# Binary -datum_gateway_bin_path: /usr/local/bin/datum_gateway - -# Ports -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -datum_gateway_stratum_port: 23334 -datum_gateway_api_port: 7152 - -# Stratum settings -datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) - -# Service user -datum_gateway_user: datum -datum_gateway_group: datum - -# Build options -datum_gateway_build_jobs: 4 - -# Bitcoin node connection -# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. -# datum_bitcoin_rpc_url should include http:// and port. -datum_bitcoin_rpc_url: "http://127.0.0.1:8332" -# Note: bitcoin_rpc_user and bitcoin_rpc_password come from group_vars/all/vault.yml - -# Mining config -datum_coinbase_tag_primary: "DATUM" -datum_coinbase_tag_secondary: "BY ORDER OF BIP110" - -# Both false on the node; the vars file said true. Corrected 2026-09-13 to -# match reality, on the same basis as datum_mining_address: the running node -# is authoritative. These control DATUM's pool-password passthrough. -datum_pool_pass_workers: false -datum_pool_pass_full_users: false -datum_pooled_mining_only: true - - -# --- Health check ----------------------------------------------------------- -# Checks the DATUM Gateway API and records the answer in its exit code, which -# systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. -# -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means -# check, exit honestly, report nowhere. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" diff --git a/ansible/roles/datum_gateway/handlers/main.yml b/ansible/roles/datum_gateway/handlers/main.yml deleted file mode 100644 index b7cfe25..0000000 --- a/ansible/roles/datum_gateway/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Restart datum-gateway - systemd: - name: datum-gateway - state: restarted - daemon_reload: yes - -# Ungated. This one carried `when: uptime_kuma_enabled | default(false)` while -# the main Restart datum-gateway handler above did not — so on this service the -# deployment restart worked and only the health-check timer restart was dead. -- name: Restart datum-gateway health check timer - systemd: - name: datum-gateway-healthcheck.timer - state: restarted - daemon_reload: yes diff --git a/ansible/roles/datum_gateway/tasks/configure.yml b/ansible/roles/datum_gateway/tasks/configure.yml deleted file mode 100644 index 463dec0..0000000 --- a/ansible/roles/datum_gateway/tasks/configure.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Ownership copied verbatim from the playbook this replaces and verified -# mechanically against `git show HEAD:`. -- name: Write DATUM Gateway config.json - ansible.builtin.template: - src: config.json.j2 - dest: "{{ datum_gateway_config_dir }}/config.json" - owner: "{{ datum_gateway_user }}" - group: "{{ datum_gateway_group }}" - mode: '0640' - # config.json carries the bitcoind RPC password, the API admin password and - # the pool passwords. `--diff` prints rendered content, so running with --diff - # put all of them on the terminal and into any log capturing it. Suppressed by - # default; pass -e datum_reveal_config=true when you genuinely need the diff. - diff: "{{ datum_reveal_config | default(false) | bool }}" - notify: Restart datum-gateway - -- name: Create datum-gateway systemd service - ansible.builtin.template: - src: datum-gateway.service.j2 - dest: /etc/systemd/system/datum-gateway.service - owner: root - group: root - mode: '0644' - notify: Restart datum-gateway diff --git a/ansible/roles/datum_gateway/tasks/healthcheck.yml b/ansible/roles/datum_gateway/tasks/healthcheck.yml deleted file mode 100644 index b5ccb90..0000000 --- a/ansible/roles/datum_gateway/tasks/healthcheck.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -# Everything here answers "is DATUM Gateway healthy" and records the answer. The -# Uptime Kuma specifics that used to follow — an embedded Python script creating -# monitors over the API, a /tmp credentials file, a push-URL file read back and -# parsed, and a systemd Environment= rewrite — are gone. Where it reports is now -# one variable, healthcheck_push_url. -- name: Create DATUM Gateway health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: /usr/local/bin/datum-gateway-healthcheck-push.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create datum-gateway health check systemd service - ansible.builtin.template: - src: healthcheck.service.j2 - dest: /etc/systemd/system/datum-gateway-healthcheck.service - owner: root - group: root - mode: "0600" - notify: Restart datum-gateway health check timer - -- name: Create datum-gateway health check systemd timer - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: /etc/systemd/system/datum-gateway-healthcheck.timer - owner: root - group: root - mode: '0644' - notify: Restart datum-gateway health check timer - -- name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - -# Ungated: enabling a timer is deployment, not monitoring. -- name: Enable and restart the datum-gateway health check timer - systemd: - name: datum-gateway-healthcheck.timer - enabled: yes - state: restarted - daemon_reload: yes - -# Arms the timer and smoke-tests the check. See roles/bitcoin_knots/README.md for -# why restarting the timer alone is not enough with OnBootSec + OnUnitActiveSec. -- name: Run the DATUM Gateway health check once to arm the timer - command: systemctl start datum-gateway-healthcheck.service - changed_when: false diff --git a/ansible/roles/datum_gateway/tasks/install.yml b/ansible/roles/datum_gateway/tasks/install.yml deleted file mode 100644 index 61dcadb..0000000 --- a/ansible/roles/datum_gateway/tasks/install.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Install DATUM Gateway build dependencies - apt: - name: - - cmake - - build-essential - - git - - libjansson-dev - - libmicrohttpd-dev - - libsodium-dev - - libcurl4-openssl-dev - # Runtime-only (netcat for health check) - - netcat-openbsd - state: present - update_cache: yes - -# =========================================== -# System User and Directories -# =========================================== -- name: Create datum system user - user: - name: "{{ datum_gateway_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ datum_gateway_dir }}" - create_home: no - comment: "DATUM Gateway" - -- name: Create DATUM Gateway directories - file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ datum_gateway_group }}" - mode: "{{ item.mode }}" - loop: - - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - -# =========================================== -# Build from Source -# =========================================== -- name: Clone DATUM Gateway repository at {{ datum_gateway_version }} - git: - repo: https://github.com/OCEAN-xyz/datum_gateway.git - dest: "{{ datum_gateway_source_dir }}" - version: "{{ datum_gateway_version }}" - force: yes - register: git_clone - -- name: Configure cmake build - command: cmake . -DCMAKE_BUILD_TYPE=Release - args: - chdir: "{{ datum_gateway_source_dir }}" - -- name: Compile datum_gateway - command: make -j{{ datum_gateway_build_jobs }} - args: - chdir: "{{ datum_gateway_source_dir }}" - -- name: Install datum_gateway binary - copy: - src: "{{ datum_gateway_source_dir }}/datum_gateway" - dest: "{{ datum_gateway_bin_path }}" - remote_src: yes - owner: root - group: root - mode: "0755" - notify: Restart datum-gateway - -# =========================================== -# Configuration diff --git a/ansible/roles/datum_gateway/tasks/main.yml b/ansible/roles/datum_gateway/tasks/main.yml deleted file mode 100644 index 57d9a45..0000000 --- a/ansible/roles/datum_gateway/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/datum_gateway/tasks/service.yml b/ansible/roles/datum_gateway/tasks/service.yml deleted file mode 100644 index c8581fd..0000000 --- a/ansible/roles/datum_gateway/tasks/service.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start datum-gateway - systemd: - name: datum-gateway - enabled: yes - state: started - -# =========================================== -# Health Check Script + Systemd Timer -# =========================================== -# ═════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/datum_gateway/templates/config.json.j2 b/ansible/roles/datum_gateway/templates/config.json.j2 deleted file mode 100644 index e17a386..0000000 --- a/ansible/roles/datum_gateway/templates/config.json.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{ - "bitcoind": { - "rpcuser": "{{ bitcoin_rpc_user }}", - "rpcpassword": "{{ bitcoin_rpc_password }}", - "rpcurl": "{{ datum_bitcoin_rpc_url }}", - "notify_fallback": true - }, - "stratum": { - "listen_port": {{ datum_gateway_stratum_port }}, - "vardiff_min": {{ datum_vardiff_min }} - }, - "mining": { - "pool_address": "{{ datum_mining_address }}", - "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", - "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" - }, - "api": { - "admin_password": "{{ datum_gateway_admin_password }}", - "listen_port": {{ datum_gateway_api_port }}, - "modify_conf": false - }, - "logger": { - "log_to_console": true, - "log_to_file": true, - "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", - "log_rotate_daily": true, - "log_level_console": 2, - "log_level_file": 1 - }, - "datum": { - "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, - "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, - "pooled_mining_only": {{ datum_pooled_mining_only | lower }} - } -} diff --git a/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 b/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 deleted file mode 100644 index 18f7a3c..0000000 --- a/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=DATUM Gateway - Bitcoin Mining Gateway -Documentation=https://github.com/OCEAN-xyz/datum_gateway -After=network.target bitcoind.service -Wants=bitcoind.service - -[Service] -User={{ datum_gateway_user }} -Group={{ datum_gateway_group }} -Type=simple -ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json -Restart=on-failure -RestartSec=10 -StandardOutput=journal -StandardError=journal - -# Prevent config from being read by other users -ReadWritePaths={{ datum_gateway_log_dir }} -ReadOnlyPaths={{ datum_gateway_config_dir }} - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.service.j2 b/ansible/roles/datum_gateway/templates/healthcheck.service.j2 deleted file mode 100644 index e21672e..0000000 --- a/ansible/roles/datum_gateway/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=DATUM Gateway Health Check -After=network.target datum-gateway.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 b/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 deleted file mode 100644 index 43ec6b2..0000000 --- a/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# DATUM Gateway health check — managed by Ansible (roles/datum_gateway) -# -# The exit code is the answer and systemd keeps it: -# systemctl is-failed datum-gateway-healthcheck.service -# Reporting anywhere else is optional and generic. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -STRATUM_PORT={{ datum_gateway_stratum_port }} - -check_datum() { - # Service must be active and stratum port must be listening - systemctl is-active --quiet datum-gateway && \ - nc -z 127.0.0.1 "${STRATUM_PORT}" -} - -report() { - local status=$1 - local msg=$2 - # No push URL is normal, not an error: the exit code below is still a - # complete answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true -} - -if check_datum; then - report "up" "OK" - exit 0 -else - report "down" "DATUM Gateway not responding" - exit 1 -fi diff --git a/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 b/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 deleted file mode 100644 index c14d1d2..0000000 --- a/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=DATUM Gateway Health Check Timer - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/forgejo_runner/README.md b/ansible/roles/forgejo_runner/README.md deleted file mode 100644 index e7f761b..0000000 --- a/ansible/roles/forgejo_runner/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# `forgejo_runner` - -Installs and runs a Forgejo Actions runner, registers it with the Forgejo -instance, and keeps a health check on a systemd timer. - -Converted from `deploy_forgejo_runner_playbook.yml` (409 lines) under Plan 6. -The playbook is now 16 lines. - -## Phases - -`tasks/main.yml` imports five files in order: - -| | | -|---|---| -| `prerequisites.yml` | Docker must be present | -| `install.yml` | binary, system user, working directory | -| `configure.yml` | config file, registration with the instance | -| `service.yml` | systemd unit, start, assert it came up | -| `healthcheck.yml` | check script, unit, timer | - -`import_tasks`, not `include_tasks` — static imports are visible to -`--list-tasks`, which is how the conversion was verified against the playbook it -replaced. - -## Monitoring: one variable, no product knowledge - -This role contains **nothing specific to any monitoring system**. What used to -be here — an ~80-line embedded Python script creating monitors over the Uptime -Kuma API, a `/tmp` credentials file, token extraction, a systemd `Environment=` -rewrite, and 8 `when: uptime_kuma_enabled` guards — is gone. - -What remains answers the actual question, *is this service healthy*, and records -it two ways: - -- **the exit code**, which systemd keeps: `systemctl is-failed - forgejo-runner-healthcheck.service` is a complete answer with no monitoring - system involved at all; -- **a log file** at `{{ healthcheck_log_file }}`. - -To report health somewhere, set one variable: - -```yaml -healthcheck_push_url: "https://example/api/push/TOKEN" -``` - -Any endpoint accepting an HTTP ping works. Empty (the default) means check, log, -exit honestly, report nowhere — which is also the right setting for a *pull*-based -monitor like Prometheus' textfile collector, since that reads unit state instead. - -The push URL is a credential (anyone holding it can forge an "up"), so callers -pass it from the vault rather than committing it. - -## One behaviour change, deliberate - -`Assert runner is running` used to be guarded by `uptime_kuma_enabled`, so it -never ran. It is not a monitoring task — it is the deployment checking its own -work — and the deprecation banner swept it up by mistake. It is ungated here, -which means a runner that fails to start now fails the play instead of -deploying "successfully" in silence. diff --git a/ansible/roles/forgejo_runner/defaults/main.yml b/ansible/roles/forgejo_runner/defaults/main.yml deleted file mode 100644 index aba6b61..0000000 --- a/ansible/roles/forgejo_runner/defaults/main.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -# Binary -forgejo_runner_version: "6.3.1" -forgejo_runner_arch: "linux-amd64" -forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" -forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" - -# Runtime -forgejo_runner_user: "runner" -forgejo_runner_dir: "/opt/forgejo-runner" -forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" -forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" - -# The Forgejo instance this runner registers with. -forgejo_instance_url: "https://forgejo.contrapeso.xyz" -# forgejo_runner_registration_token comes from the vault. - -# --- Health check ----------------------------------------------------------- -# The check answers "is this service healthy" and records the answer two ways: -# a log file, and its own exit code. The exit code is the durable artefact — -# systemd stores it, so `systemctl is-failed forgejo-runner-healthcheck.service` -# answers the question with no monitoring system involved at all. -healthcheck_interval_seconds: 60 -healthcheck_script_dir: /opt/forgejo-runner-healthcheck -healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" -healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" -healthcheck_service_name: forgejo-runner-healthcheck - -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. -# -# Empty means "check, log, exit honestly, report nowhere". Set it to any URL -# that accepts an HTTP ping and the check will report there. Nothing in this -# role is specific to a particular monitoring product: the Uptime Kuma API -# calls, monitor creation and token handling that used to live here are gone. -# -# A pull-based monitor (Prometheus node_exporter textfile, say) needs this left -# empty — it reads the systemd unit state instead. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" diff --git a/ansible/roles/forgejo_runner/tasks/configure.yml b/ansible/roles/forgejo_runner/tasks/configure.yml deleted file mode 100644 index ab96d52..0000000 --- a/ansible/roles/forgejo_runner/tasks/configure.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Check if config already exists - stat: - path: "{{ forgejo_runner_config_path }}" - register: config_stat - -- name: Generate default config - shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not config_stat.stat.exists - -- name: Set config file ownership - file: - path: "{{ forgejo_runner_config_path }}" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not config_stat.stat.exists - -# ── 6. Register runner ───────────────────────────────────────────── -- name: Check if runner is already registered - stat: - path: "{{ forgejo_runner_dir }}/.runner" - register: runner_stat - -- name: Register runner with Forgejo instance - command: > - {{ forgejo_runner_bin_path }} register --no-interactive - --instance {{ forgejo_instance_url }} - --token {{ forgejo_runner_registration_token }} - --name forgejo-runner-box - --labels "{{ forgejo_runner_labels }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not runner_stat.stat.exists - -- name: Set runner registration file ownership - file: - path: "{{ forgejo_runner_dir }}/.runner" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not runner_stat.stat.exists - diff --git a/ansible/roles/forgejo_runner/tasks/healthcheck.yml b/ansible/roles/forgejo_runner/tasks/healthcheck.yml deleted file mode 100644 index 42fbaf7..0000000 --- a/ansible/roles/forgejo_runner/tasks/healthcheck.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -# Everything here answers "is the service healthy" and records the answer. -# The Uptime Kuma specifics that used to surround it — an embedded Python script -# that created monitors over the API, a /tmp credentials file, token extraction, -# and a systemd Environment= rewrite — are gone. What reports where is now one -# variable, healthcheck_push_url. See the role README. -- name: Create healthcheck script directory - ansible.builtin.file: - path: "{{ healthcheck_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Create forgejo-runner healthcheck script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: "{{ healthcheck_script_path }}" - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create healthcheck systemd service - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" - owner: root - group: root - mode: "0600" - -- name: Create healthcheck systemd timer - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" - owner: root - group: root - mode: '0644' - -- name: Reload systemd for healthcheck units - systemd: - daemon_reload: yes - -- name: Enable and start healthcheck timer - systemd: - name: "{{ healthcheck_service_name }}.timer" - enabled: yes - state: started - -- name: Test healthcheck script - command: "{{ healthcheck_script_path }}" - register: healthcheck_test - changed_when: false - -- name: Verify healthcheck script works - assert: - that: - - healthcheck_test.rc == 0 - fail_msg: "Healthcheck script failed to execute properly" - -- name: Display deployment summary - debug: - msg: | - Forgejo Runner deployed successfully! - - Runner Name: forgejo-runner-box - Instance: {{ forgejo_instance_url }} - Working Directory: {{ forgejo_runner_dir }} - Service: forgejo-runner.service ({{ runner_active.stdout }}) - - Healthcheck Monitor: {{ healthcheck_service_name }} - Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s - Reporting to: {{ healthcheck_push_url | default('', true) | regex_replace('/api/push/.*', '/api/push/***') | default('(nowhere - set healthcheck_push_url)', true) }} diff --git a/ansible/roles/forgejo_runner/tasks/install.yml b/ansible/roles/forgejo_runner/tasks/install.yml deleted file mode 100644 index b95ab4c..0000000 --- a/ansible/roles/forgejo_runner/tasks/install.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Download forgejo-runner binary - get_url: - url: "{{ forgejo_runner_url }}" - dest: "{{ forgejo_runner_bin_path }}" - mode: '0755' - -# ── 3. Create runner system user ─────────────────────────────────── -- name: Create runner system user - user: - name: "{{ forgejo_runner_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ forgejo_runner_dir }}" - create_home: no - groups: docker - append: yes - comment: 'Forgejo Runner' - -# ── 4. Create working directory ──────────────────────────────────── -- name: Create forgejo-runner working directory - file: - path: "{{ forgejo_runner_dir }}" - state: directory - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - mode: '0750' diff --git a/ansible/roles/forgejo_runner/tasks/main.yml b/ansible/roles/forgejo_runner/tasks/main.yml deleted file mode 100644 index aa71001..0000000 --- a/ansible/roles/forgejo_runner/tasks/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# import_tasks, not include_tasks: these are unconditional phases, and static -# imports are visible to `--list-tasks`. That matters because the task list is -# how this refactor was verified against the playbook it replaced. -- ansible.builtin.import_tasks: prerequisites.yml -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/forgejo_runner/tasks/prerequisites.yml b/ansible/roles/forgejo_runner/tasks/prerequisites.yml deleted file mode 100644 index d441709..0000000 --- a/ansible/roles/forgejo_runner/tasks/prerequisites.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Check if Docker is installed - command: docker --version - register: docker_check - changed_when: false - failed_when: docker_check.rc != 0 - -- name: Fail if Docker is not available - assert: - that: - - docker_check.rc == 0 - fail_msg: "Docker is required for forgejo-runner but is not installed" diff --git a/ansible/roles/forgejo_runner/tasks/service.yml b/ansible/roles/forgejo_runner/tasks/service.yml deleted file mode 100644 index f971e52..0000000 --- a/ansible/roles/forgejo_runner/tasks/service.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Create forgejo-runner systemd service - ansible.builtin.template: - src: forgejo-runner.service.j2 - dest: /etc/systemd/system/forgejo-runner.service - owner: root - group: root - mode: '0644' - -- name: Reload systemd - systemd: - daemon_reload: yes - -- name: Enable and start forgejo-runner service - systemd: - name: forgejo-runner - enabled: yes - state: started - -- name: Verify forgejo-runner is active - command: systemctl is-active forgejo-runner - register: runner_active - changed_when: false - -# Ungated on purpose. This was previously guarded by `uptime_kuma_enabled`, but -# it is not a monitoring task — it is the deployment asserting its own success. -# The deprecation banner swept it up along with the Kuma plumbing, which meant a -# broken runner deployed "successfully" and silently. -- name: Assert runner is running - assert: - that: - - runner_active.stdout == "active" - fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" diff --git a/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 b/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 deleted file mode 100644 index d3db25d..0000000 --- a/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Forgejo Runner -Documentation=https://forgejo.org/docs/latest/admin/actions/ -After=docker.service -Requires=docker.service - -[Service] -Type=simple -User={{ forgejo_runner_user }} -Group={{ forgejo_runner_user }} -WorkingDirectory={{ forgejo_runner_dir }} -ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 deleted file mode 100644 index 173f42b..0000000 --- a/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Forgejo Runner Healthcheck -After=network.target - -[Service] -Type=oneshot -ExecStart={{ healthcheck_script_path }} -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -User=root -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 deleted file mode 100644 index 9aa5ac1..0000000 --- a/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Forgejo Runner healthcheck — managed by Ansible (roles/forgejo_runner) -# -# Answers "is forgejo-runner healthy" and records it two ways: this log, and the -# exit code. The exit code is the durable artefact — systemd keeps it, so -# systemctl is-failed {{ healthcheck_service_name }}.service -# answers the question with no monitoring system involved. -# -# Reporting is optional and generic: if a push URL is configured it also pings -# it. Nothing here knows or cares which monitoring product is on the other end. - -LOG_FILE="{{ healthcheck_log_file }}" -# Read from the environment rather than templated in, so the unit file is the -# only place the token lives and the script is not secret. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" -} - -# Gatus external endpoint: a POST with a bearer token and success=true|false. -# -# This used to report ONLY success - it exited before pushing when the runner -# was down - so a failure was invisible until the heartbeat window expired. -# Reporting the failure is the whole point of having a check. -report() { - local ok="$1" msg="$2" - [ -n "$PUSH_URL" ] || return 0 - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${ok}&error=${msg// /%20}" || true -} - -main() { - if ! systemctl is-active --quiet forgejo-runner; then - log_message "ERROR: forgejo-runner is not active" - report false "forgejo-runner is not active" - exit 1 - fi - - log_message "forgejo-runner is active" - report true "active" - exit 0 -} - -main diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 deleted file mode 100644 index 4cba51a..0000000 --- a/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Run Forgejo Runner Healthcheck every minute -Requires={{ healthcheck_service_name }}.service - -[Timer] -OnBootSec=30sec -OnUnitActiveSec={{ healthcheck_interval_seconds }}sec -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/fulcrum/README.md b/ansible/roles/fulcrum/README.md deleted file mode 100644 index ddefabb..0000000 --- a/ansible/roles/fulcrum/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# `fulcrum` - -Deploys [Fulcrum](https://github.com/cculianu/Fulcrum), an Electrum server -indexing the Bitcoin Knots node, on `fulcrum-box`. The second play in the -calling playbook publishes its SSL port from the edge host via `socket_proxy`. - -Converted from `deploy_fulcrum_playbook.yml` (685 lines) under Plan 6. The -playbook is now 33 lines. - -## The index is the expensive thing - -`{{ fulcrum_db_dir }}` is ~192 GB and takes days to rebuild. Nothing in this role -touches it beyond `state: directory` with the ownership it already has -(`fulcrum:fulcrum 0755`). Restarting Fulcrum re-opens the database; it does not -reindex. - -## Three things this conversion fixed, all pre-existing - -**`bitcoind` pointed at the wrong machine.** The vars file carried -`bitcoin_rpc_host: "192.168.1.140"` commented "IP of knots_box_local", but `.140` -is **fulcrum-box itself** — knots-box is `.135`. The DHCP leases had reshuffled. -The live config had already been hand-corrected to `knots-box`; running the -playbook would have reverted it and broken indexing. Now addressed by Tailscale -name, like everything else in this repo. - -**The restart handler was inert.** It carried -`when: uptime_kuma_enabled | default(false)`, so the three tasks that notify it -(SSL certificate, `fulcrum.conf`, systemd unit) could not restart anything. A -configuration change would write to disk, report success, and silently never take -effect. Ungated. - -**`db_mem` was about to quadruple.** The role computes a share of RAM; on this -5931 MB host 75% is 4448 MB, leaving ~1.4 GB for the OS and Fulcrum's non-cache -memory. The live value had been hand-tuned to 2048. `fulcrum_db_mem_mb_override` -pins it. Note `set_fact` outranks role defaults, so the *calculation* has to -honour the override — pinning it in `defaults/` alone is silently ignored. - -## The health check timer, and how to read it - -The timer is `OnBootSec` + `OnUnitActiveSec` with no `OnCalendar`. That -combination has a failure mode worth knowing: `OnBootSec` is monotonic and -elapses once; `OnUnitActiveSec` schedules relative to the **service** last being -active. If the service does not run in a given boot, there is no reference to -schedule from and the timer sits `active` and `enabled` doing nothing. That is -exactly what had happened here — last trigger **2026-02-17**, seven months of no -health check, with every surface-level indicator green. - -Restarting the timer does not supply that reference; running the service does. -So the role runs the check once after enabling the timer, which is both the fix -and a smoke test. - -**Diagnosing this is easy to get wrong**: `NextElapseUSecRealtime` is always -empty for a monotonic timer, so it looks broken even when it is fine. Read -`NextElapseUSecMonotonic`, or just use `systemctl list-timers`. - -The timer also no longer carries `Requires=fulcrum.service`. On a timer that -means "stop watching when the watched thing stops", which is backwards for a -health check. - -## Monitoring: one variable, no product knowledge - -The check tests the Electrum TCP port and records the answer in its exit code, -which systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. To report -elsewhere set `healthcheck_push_url` to any endpoint accepting an HTTP ping. The -Uptime Kuma API calls, monitor creation and token handling are gone. diff --git a/ansible/roles/fulcrum/defaults/main.yml b/ansible/roles/fulcrum/defaults/main.yml deleted file mode 100644 index 3e97fa0..0000000 --- a/ansible/roles/fulcrum/defaults/main.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Fulcrum Configuration Variables - -# Version - Pinned to specific release -fulcrum_version: "2.1.0" # Fulcrum version to install - -# Directories -fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) -fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) -fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk -fulcrum_binary_path: /usr/local/bin/Fulcrum - -# Network - Bitcoin RPC connection -# Bitcoin Knots is on a different host (knots_box_local) -# Using RPC user/password authentication (credentials from group_vars/all/vault.yml) -# Addressed by Tailscale name, never a LAN IP. This was -# bitcoin_rpc_host: "192.168.1.140" # IP of knots_box_local -# but .140 is fulcrum-box ITSELF - knots-box is .135. The DHCP leases had -# reshuffled (the same drift that transposed the inventory), so running this -# playbook would have pointed Fulcrum at itself and broken indexing. The live -# config had already been hand-corrected to knots-box; this makes the repo -# agree with it. -bitcoin_rpc_host: "knots-box" -bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml - -# Network - Fulcrum server -fulcrum_tcp_port: 50001 -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/fulcrum_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -fulcrum_ssl_port: 50002 -# Binding address for Fulcrum TCP/SSL server: -# - "127.0.0.1" = localhost only (use when Caddy is on the same box) -# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) -# - Specific IP = bind to specific network interface -fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) -fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port -# If Caddy is on a different box, set this to the IP address that Caddy will use to connect - -# SSL/TLS Configuration -fulcrum_ssl_enabled: true -fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" -fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" -fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert - - -# Performance -# db_mem will be calculated as 75% of available RAM automatically in playbook -# db_mem is computed as this share of RAM unless fulcrum_db_mem_mb is set -# explicitly. On a 5931 MB host 75% is 4448 MB, which leaves ~1.4 GB for the -# OS and for Fulcrum's non-cache memory; the live config had been hand-tuned -# down to 2048 and that setting is respected below. -fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache - -# Configuration options -fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs -fulcrum_peering: false # Disable peering with other Fulcrum servers -fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications - -# Service user -fulcrum_user: fulcrum -fulcrum_group: fulcrum - - -# --- Health check ----------------------------------------------------------- -# Checks the Electrum TCP port and records the answer in its exit code, which -# systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. -# -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means -# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping -# works; nothing here is specific to a monitoring product. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" - -# Explicit db_mem in MB. When set it wins over fulcrum_db_mem_percent; empty -# means compute from RAM. Set here because the live host had been hand-tuned to -# 2048 and a silent jump to 4448 is not something a refactor should do. -fulcrum_db_mem_mb_override: 2048 diff --git a/ansible/roles/fulcrum/handlers/main.yml b/ansible/roles/fulcrum/handlers/main.yml deleted file mode 100644 index c0d9cc4..0000000 --- a/ansible/roles/fulcrum/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# Ungated on purpose. The hand-written handler carried -# when: uptime_kuma_enabled | default(false) -# so it has been inert since the decommissioning: three tasks notify it (the SSL -# certificate, fulcrum.conf and the systemd unit), and none of them could -# actually restart Fulcrum. A configuration change therefore applied to disk and -# silently never took effect — the worst kind of quiet failure, because the -# playbook reports success and the running service keeps its old settings. -# -# Restarting Fulcrum re-opens its database; it does not reindex. -- name: Restart fulcrum - systemd: - name: fulcrum - state: restarted - daemon_reload: yes diff --git a/ansible/roles/fulcrum/tasks/healthcheck.yml b/ansible/roles/fulcrum/tasks/healthcheck.yml deleted file mode 100644 index 983bffb..0000000 --- a/ansible/roles/fulcrum/tasks/healthcheck.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -# Everything here answers "is Fulcrum healthy" and records the answer. The -# Uptime Kuma specifics that used to follow — an embedded Python script creating -# monitors over the API, a /tmp credentials file, push-URL extraction and a -# systemd Environment= rewrite — are gone. Where it reports is now one variable, -# healthcheck_push_url. See the role README. -- name: Create Fulcrum health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: /usr/local/bin/fulcrum-healthcheck-push.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create systemd service for Fulcrum health check - ansible.builtin.template: - src: healthcheck.service.j2 - dest: /etc/systemd/system/fulcrum-healthcheck.service - owner: root - group: root - mode: "0600" - -- name: Create systemd timer for Fulcrum health check - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: /etc/systemd/system/fulcrum-healthcheck.timer - owner: root - group: root - mode: '0644' - -- name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - -# state: restarted, not started. The hand-written timer had got itself stuck -# `active` with no next elapse and had not fired since 2026-02-17; `started` on -# an already-active timer is a no-op and would have left it stuck. Restarting -# re-arms it. See the note in healthcheck.timer.j2. -- name: Enable and restart the Fulcrum health check timer - systemd: - name: fulcrum-healthcheck.timer - enabled: yes - state: restarted - daemon_reload: yes - -# Run the check once, which is both a smoke test and the thing that actually -# arms the timer. -# -# This timer is OnBootSec + OnUnitActiveSec with no OnCalendar. OnBootSec is -# monotonic and had long since elapsed; OnUnitActiveSec schedules relative to the -# SERVICE last being active, and the service had not run since 2026-02-17 — so -# there was no reference to schedule from and the timer sat `active` and -# `enabled` with NextElapseUSecMonotonic=infinity. Restarting the timer alone -# does not supply that reference; running the service does. -# -# (Diagnosing this is easy to get wrong: NextElapseUSecRealtime is always empty -# for a monotonic timer, so it looks broken even when it is fine. Read -# NextElapseUSecMonotonic, or just use `systemctl list-timers`.) -- name: Run the Fulcrum health check once to arm the timer - command: systemctl start fulcrum-healthcheck.service - changed_when: false diff --git a/ansible/roles/fulcrum/tasks/install.yml b/ansible/roles/fulcrum/tasks/install.yml deleted file mode 100644 index 1216a96..0000000 --- a/ansible/roles/fulcrum/tasks/install.yml +++ /dev/null @@ -1,146 +0,0 @@ ---- -- name: Calculate db_mem as a share of system RAM - set_fact: - fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" - when: fulcrum_db_mem_mb_override | string | length == 0 - -- name: Use the explicit db_mem override - set_fact: - fulcrum_db_mem_mb: "{{ fulcrum_db_mem_mb_override }}" - when: fulcrum_db_mem_mb_override | string | length > 0 - changed_when: false - -- name: Display calculated db_mem value - debug: - msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" - -- name: Display Fulcrum version to install - debug: - msg: "Installing Fulcrum version {{ fulcrum_version }}" - -- name: Install required packages - apt: - name: - - curl - - wget - - openssl - state: present - update_cache: yes - -- name: Create fulcrum group - group: - name: "{{ fulcrum_group }}" - system: yes - state: present - -- name: Create fulcrum user - user: - name: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - system: yes - shell: /usr/sbin/nologin - home: /home/{{ fulcrum_user }} - create_home: yes - state: present - -- name: Create Fulcrum database directory (heavy data on special mount) - file: - path: "{{ fulcrum_db_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - -- name: Create Fulcrum config directory - file: - path: "{{ fulcrum_config_dir }}" - state: directory - owner: root - group: "{{ fulcrum_group }}" - mode: '0755' - -- name: Create Fulcrum lib directory (for banner and other data files) - file: - path: "{{ fulcrum_lib_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - -# =========================================== -# SSL Certificate Generation -# =========================================== -- name: Check if SSL certificate already exists - stat: - path: "{{ fulcrum_ssl_cert_path }}" - register: fulcrum_ssl_cert_exists - when: fulcrum_ssl_enabled | default(false) - -- name: Generate self-signed SSL certificate for Fulcrum - command: > - openssl req -x509 -newkey rsa:4096 - -keyout {{ fulcrum_ssl_key_path }} - -out {{ fulcrum_ssl_cert_path }} - -sha256 -days {{ fulcrum_ssl_cert_days }} - -nodes - -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" - args: - creates: "{{ fulcrum_ssl_cert_path }}" - when: fulcrum_ssl_enabled | default(false) - notify: Restart fulcrum - -- name: Set SSL certificate permissions - file: - path: "{{ fulcrum_ssl_cert_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) - -- name: Set SSL key permissions - file: - path: "{{ fulcrum_ssl_key_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0600' - when: fulcrum_ssl_enabled | default(false) - -- name: Check if Fulcrum binary already exists - stat: - path: "{{ fulcrum_binary_path }}" - register: fulcrum_binary_exists - changed_when: false - -- name: Download Fulcrum binary tarball - get_url: - url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - mode: '0644' - when: not fulcrum_binary_exists.stat.exists - -- name: Extract Fulcrum binary - unarchive: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp" - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - -- name: Install Fulcrum binary - copy: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" - dest: "{{ fulcrum_binary_path }}" - owner: root - group: root - mode: '0755' - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - -- name: Verify Fulcrum binary installation - command: "{{ fulcrum_binary_path }} --version" - register: fulcrum_version_check - changed_when: false - -- name: Display Fulcrum version - debug: - msg: "{{ fulcrum_version_check.stdout_lines }}" - diff --git a/ansible/roles/fulcrum/tasks/main.yml b/ansible/roles/fulcrum/tasks/main.yml deleted file mode 100644 index bc7ff05..0000000 --- a/ansible/roles/fulcrum/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/fulcrum/tasks/service.yml b/ansible/roles/fulcrum/tasks/service.yml deleted file mode 100644 index 716b44d..0000000 --- a/ansible/roles/fulcrum/tasks/service.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -- name: Create Fulcrum banner file - ansible.builtin.template: - src: banner.txt.j2 - dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - -- name: Create Fulcrum configuration file - ansible.builtin.template: - src: fulcrum.conf.j2 - dest: "{{ fulcrum_config_dir }}/fulcrum.conf" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0640' - notify: Restart fulcrum - -- name: Create systemd service file for Fulcrum - ansible.builtin.template: - src: fulcrum.service.j2 - dest: /etc/systemd/system/fulcrum.service - owner: root - group: root - mode: '0644' - notify: Restart fulcrum - -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start Fulcrum service - systemd: - name: fulcrum - enabled: yes - state: started - -- name: Wait for Fulcrum to start - wait_for: - port: "{{ fulcrum_tcp_port }}" - host: "{{ fulcrum_tcp_bind }}" - delay: 5 - timeout: 30 - ignore_errors: yes - -- name: Check Fulcrum service status - systemd: - name: fulcrum - register: fulcrum_service_status - changed_when: false - -- name: Display Fulcrum service status - debug: - msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" diff --git a/ansible/roles/fulcrum/templates/banner.txt.j2 b/ansible/roles/fulcrum/templates/banner.txt.j2 deleted file mode 100644 index 1ce4ced..0000000 --- a/ansible/roles/fulcrum/templates/banner.txt.j2 +++ /dev/null @@ -1,3 +0,0 @@ -counterinfra - -PER ASPERA AD ASTRA diff --git a/ansible/roles/fulcrum/templates/fulcrum.conf.j2 b/ansible/roles/fulcrum/templates/fulcrum.conf.j2 deleted file mode 100644 index 69f072c..0000000 --- a/ansible/roles/fulcrum/templates/fulcrum.conf.j2 +++ /dev/null @@ -1,29 +0,0 @@ -# Fulcrum Configuration -# Generated by Ansible - -# Bitcoin Core/Knots RPC settings -bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} -rpcuser = {{ bitcoin_rpc_user }} -rpcpassword = {{ bitcoin_rpc_password }} - -# Fulcrum server general settings -datadir = {{ fulcrum_db_dir }} -tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} -peering = {{ 'true' if fulcrum_peering else 'false' }} -zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} - -# SSL/TLS Configuration -{% if fulcrum_ssl_enabled | default(false) %} -ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} -cert = {{ fulcrum_ssl_cert_path }} -key = {{ fulcrum_ssl_key_path }} -{% endif %} - -# Anonymize client IP addresses and TxIDs in logs -anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} - -# Max RocksDB Memory in MiB -db_mem = {{ fulcrum_db_mem_mb }}.0 - -# Banner -banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt diff --git a/ansible/roles/fulcrum/templates/fulcrum.service.j2 b/ansible/roles/fulcrum/templates/fulcrum.service.j2 deleted file mode 100644 index 02f4aa5..0000000 --- a/ansible/roles/fulcrum/templates/fulcrum.service.j2 +++ /dev/null @@ -1,24 +0,0 @@ -# MiniBolt: systemd unit for Fulcrum -# /etc/systemd/system/fulcrum.service - -[Unit] -Description=Fulcrum -After=network.target - -StartLimitBurst=2 -StartLimitIntervalSec=20 - -[Service] -ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf - -User={{ fulcrum_user }} -Group={{ fulcrum_group }} - -# Process management -#################### -Type=simple -KillSignal=SIGINT -TimeoutStopSec=300 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.service.j2 b/ansible/roles/fulcrum/templates/healthcheck.service.j2 deleted file mode 100644 index b519808..0000000 --- a/ansible/roles/fulcrum/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Fulcrum Health Check -After=network.target fulcrum.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.sh.j2 b/ansible/roles/fulcrum/templates/healthcheck.sh.j2 deleted file mode 100644 index 4449fed..0000000 --- a/ansible/roles/fulcrum/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Fulcrum health check — managed by Ansible (roles/fulcrum) -# -# Checks that Fulcrum's Electrum TCP port is accepting connections, and records -# the answer in the exit code, which systemd keeps: -# systemctl is-failed fulcrum-healthcheck.service -# That is a complete answer with no monitoring system involved. Reporting -# elsewhere is optional and generic — set healthcheck_push_url. - -FULCRUM_HOST="{{ fulcrum_tcp_bind }}" -FULCRUM_PORT={{ fulcrum_tcp_port }} -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -check_fulcrum() { - timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null -} - -report() { - local status=$1 msg=$2 - # No push URL is normal, not an error: the exit code below is still a - # complete answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true -} - -if check_fulcrum; then - report "up" "OK" - exit 0 -else - echo "Fulcrum TCP port ${FULCRUM_PORT} not responding" - report "down" "Fulcrum TCP port not responding" - exit 1 -fi diff --git a/ansible/roles/fulcrum/templates/healthcheck.timer.j2 b/ansible/roles/fulcrum/templates/healthcheck.timer.j2 deleted file mode 100644 index df16ff3..0000000 --- a/ansible/roles/fulcrum/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Fulcrum Health Check Timer -# NOTE: this deliberately does NOT carry `Requires=fulcrum.service`, which the -# hand-written unit had. Requires on a timer means the timer is stopped when the -# required unit stops — i.e. "if the thing I am watching goes down, stop -# watching it", which is backwards for a health check and leaves nothing to -# re-arm the timer when the service returns. The live timer had been `active` -# and `enabled` with NextElapseUSecMonotonic=infinity and a last trigger of -# 2026-02-17: seven months with no health check and no outward sign of it. - -[Timer] -OnBootSec=1min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/gatus/README.md b/ansible/roles/gatus/README.md deleted file mode 100644 index 129057c..0000000 --- a/ansible/roles/gatus/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# gatus - -Deploys [Gatus](https://github.com/TwiN/gatus) — health checks, a status page, -and alerting — **as the upstream container image**, on the -`observability` group. - -## Why the container - -Upstream publishes **no binary release assets**. The image is the only artefact -they ship and therefore the only one they test, so it is what `docker run` in -their README gets you, and it is what this role deploys. - -Building from source is entirely possible — their Dockerfile is a bare -`CGO_ENABLED=0 go build`, the Vue dashboard is compiled in via `//go:embed -static` in `web/static.go`, and the sqlite driver is pure-Go -`modernc.org/sqlite` so nothing needs linking. This role did that at first. The -reasons it doesn't now: - -- It produces a binary upstream never ran. -- It means compiling the AWS SDK, gRPC and the Google API libraries on the - smallest box in the estate. On this VPS that load was heavy enough that - unrelated Ansible tasks timed out while it ran. - -The cost of the container is a daemon on the machine whose job is to notice -when everything else breaks. That is a real trade, made deliberately. - -## Pinned by digest, not by tag - -```yaml -gatus_image_digest: "sha256:c5f210d0…" -gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" -``` - -A tag is mutable — `v5.36.0` can be repushed — so pinning the tag alone is a -weaker promise than it looks. The digest is a content address: if it resolves, -it is byte-for-byte the image reviewed here, and `docker compose pull` either -fetches exactly that or fails. `gatus_version` is kept alongside it purely so a -human can read which release it is; **the two must be updated together.** - -## What `FROM scratch` means for running it - -The image has no `/etc/passwd`, so there is no user to drop to by name and the -default is root. The compose file runs it by numeric id (`10001:10001`) and the -data directory on the host is owned to match. Everything else is locked down to -approximate what the systemd unit used to do natively: - -| systemd | compose | -|---|---| -| `ProtectSystem=strict` | `read_only: true` | -| `NoNewPrivileges=true` | `security_opt: [no-new-privileges:true]` | -| `CapabilityBoundingSet=` | `cap_drop: [ALL]` | -| `AmbientCapabilities=CAP_NET_RAW` | `cap_add: [NET_RAW]` (for `icmp://`) | - -## Configuration is a directory, not a file - -`GATUS_CONFIG_PATH` points at `/opt/gatus/config`, and Gatus merges every `*.yaml` -underneath it — maps deep-merge, lists append. This role owns exactly one file: - -``` -/opt/gatus/config/00-base.yaml web, storage, ui, alerting, security (this role) -/opt/gatus/config/endpoints/*.yaml one file per service (gatus_endpoint) -``` - -**A primitive defined in two files is ambiguous and upstream refuses it.** So -anything that is not a list belongs in `00-base.yaml` and nowhere else. Endpoints -are lists, so each service's file appends cleanly — the same shape as -`caddy_site`, where each service contributes its own vhost. - -## Pull and push - -Gatus polls. For anything with a reachable HTTP or TCP surface that is the -better check, because it tests the path a user actually takes. The monitoring -host joins the headscale mesh via `infra/920`, so internal boxes are reachable -by MagicDNS name and can be polled directly rather than having to report in. - -For state with no pollable surface — ZFS pool health, UPS mains status, disk -usage, backup freshness — Gatus has **external endpoints**, a push API: - -``` -POST /api/v1/endpoints/{group}_{name}/external?success=true&error=&duration= -Authorization: Bearer -``` - -with `heartbeat.interval` to alert when nothing reports in. That is the same -shape as the generic `healthcheck_push_url` already wired into every service -role, so those scripts need a URL, a POST, and an auth header — not a rewrite. - -## Variables - -See `defaults/main.yml`. The ones that matter: - -| Variable | Default | Note | -|---|---|---| -| `gatus_version` / `gatus_image_digest` | `v5.36.0` / `sha256:c5f210d0…` | must move together | -| `gatus_bind_address` | `127.0.0.1` | **never bind publicly** — the push API shares this listener | -| `gatus_storage_type` | `sqlite` | `memory` loses all history on restart | -| `gatus_alerting` | `{}` | pass-through; any provider Gatus supports | -| `gatus_allow_icmp` | `true` | adds back `NET_RAW` for `icmp://` checks | - -`gatus_alerting` empty is valid and is the current state: every condition is -still evaluated and recorded, there is just nowhere to shout yet. - -## Verifying - -```bash -docker ps --filter name=gatus -docker logs gatus --tail 50 -curl -s localhost:8080/health -ls /opt/gatus/config/endpoints/ -``` diff --git a/ansible/roles/gatus/defaults/main.yml b/ansible/roles/gatus/defaults/main.yml deleted file mode 100644 index 6f6ec09..0000000 --- a/ansible/roles/gatus/defaults/main.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- -# Gatus, deployed the way upstream distributes it: the container image. -# -# Upstream publishes NO binary release assets - the image is the only artefact -# they ship, and therefore the only artefact they test. Building from source is -# possible (`go build` alone is enough; the Vue dashboard is compiled in via -# `//go:embed static`, and CGO_ENABLED=0 works because the sqlite driver is -# pure-Go modernc.org/sqlite) but it produces a binary upstream never ran, and -# it means a full compile of the AWS SDK and gRPC on the smallest box in the -# estate. - -# ── Image ──────────────────────────────────────────────────────────────────── -gatus_version: "v5.36.0" -# Pinned by DIGEST, not by tag. A tag is mutable - `v5.36.0` can be repushed - -# so pinning the tag alone is a weaker promise than it looks. The digest is the -# content address: if it resolves, it is byte-for-byte the image reviewed here. -# Both must be updated together; the tag is kept only so humans can read it. -gatus_image_digest: "sha256:c5f210d095fa78e6efaa20ffeb14803f2ba4f10615e16a6d12087697149617f0" -gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" - -# ── Paths (host side) ──────────────────────────────────────────────────────── -gatus_dir: /opt/gatus -gatus_config_dir: "{{ gatus_dir }}/config" -gatus_data_dir: "{{ gatus_dir }}/data" - -# Gatus merges every *.yaml under GATUS_CONFIG_PATH and its subdirectories: -# maps deep-merge, lists append. That is why this role ships a config DIRECTORY -# rather than one file - each service contributes its own endpoint file, the -# same way each service contributes a vhost through `caddy_site`. -# -# Primitives must be defined exactly once across all files or the merge is -# ambiguous, so everything that is not a list lives in the base file and -# nowhere else. -gatus_base_config_file: "00-base.yaml" -gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" - -# ── Identity ───────────────────────────────────────────────────────────────── -# The image is FROM scratch, so it has no /etc/passwd and no user to drop to by -# name. Run it by numeric uid/gid instead, and own the data volume to match. -gatus_uid: 10001 -gatus_gid: 10001 - -# ── Web ────────────────────────────────────────────────────────────────────── -gatus_port: 8080 -# HOST-side address the container's port is published on. Gatus itself always -# binds 0.0.0.0 inside the container - see the note in config.yaml.j2. Never -# publish this on 0.0.0.0: the external-endpoint push API shares the dashboard's -# listener, and Caddy is what should be in front of both. -gatus_bind_address: "127.0.0.1" -# Pass-through, the same shape as gatus_alerting: whatever is set here is -# rendered verbatim under `ui:`, so every option upstream supports is reachable -# without adding a variable per key. See config/ui/ui.go for the full set - -# title, description, header, dashboard-heading/subheading, logo, link, favicon, -# buttons, custom-css, dark-mode, default-sort-by, default-filter-by. -gatus_ui: - title: "Status" - header: "Status" - # Group by default. The dashboard's own groupByGroup toggle starts OFF and - # remembers per browser in localStorage, so without this the ten groups render - # as one flat list for anyone who has not clicked it. - default-sort-by: group - -# ── Storage ────────────────────────────────────────────────────────────────── -# sqlite, not memory: history has to survive a restart, or the dashboard lies -# about uptime after every deploy. Path is INSIDE the container. -gatus_storage_type: sqlite -gatus_storage_path: "/data/gatus.db" -gatus_storage_caching: true - -# Per-endpoint row caps. Gatus bounds the database by COUNT, not by time, and -# trims inline on insert (storage/store/sql/sql.go, InsertEndpointResult) - so -# there is no retention job to write and no way for this to fill a disk. -# -# History depth is therefore a function of check frequency, not of days: -# 900 results is ~3 days of a 5-minute liveness check, and ~2.5 years of a daily -# disk check. Upstream's default is 100, which would have been 8 hours of -# liveness - not enough to still see a weekend incident on Monday. -# -# Note the uptime table is separate and its 30-day retention is hard-coded -# upstream (uptimeRetention), so uptime percentages top out at 30 days whatever -# this is set to. -gatus_storage_max_results: 900 -gatus_storage_max_events: 50 - -# ── Alerting ───────────────────────────────────────────────────────────────── -# Pass-through: rendered verbatim under `alerting:`, so any provider Gatus -# supports works without touching this role. Empty means "check and record, -# alert nowhere" - valid, and the default until a provider is chosen. -gatus_alerting: {} -gatus_default_alerts: [] - -# ── Security ───────────────────────────────────────────────────────────────── -# gatus_basic_auth: {username: admin, password-bcrypt-base64: "..."} -gatus_basic_auth: {} - -gatus_maintenance: {} - -# Keep serving the previous config if a contributed endpoint file is malformed, -# instead of panicking. See the note in config.yaml.j2. -gatus_skip_invalid_config_update: true -gatus_log_level: INFO - -# ── Self-check ─────────────────────────────────────────────────────────────── -# Gatus panics on a config with no endpoints, so the role always ships one. -# Turning this off is only safe once another file in endpoints/ provides one. -gatus_self_check: true - -# ── ICMP ───────────────────────────────────────────────────────────────────── -# Gatus supports icmp:// endpoints. Raw ICMP needs CAP_NET_RAW, which the -# container would get free only if it ran as root; it does not. Set false if -# you never use icmp:// checks and want the capability dropped entirely. -gatus_allow_icmp: true - -# ── Shared network ─────────────────────────────────────────────────────────── -# Gatus runs in a container, so the HOST's loopback is not reachable from it. -# Anything Gatus must talk to locally - the Signal API that sends its alerts - -# has to be on a shared docker network and addressed by service name. -gatus_network: monitoring diff --git a/ansible/roles/gatus/handlers/main.yml b/ansible/roles/gatus/handlers/main.yml deleted file mode 100644 index 25ad865..0000000 --- a/ansible/roles/gatus/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Restart gatus - ansible.builtin.command: - cmd: docker compose up -d --force-recreate - chdir: "{{ gatus_dir }}" diff --git a/ansible/roles/gatus/tasks/configure.yml b/ansible/roles/gatus/tasks/configure.yml deleted file mode 100644 index ef5d7e7..0000000 --- a/ansible/roles/gatus/tasks/configure.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- name: Assert Docker is available - ansible.builtin.command: docker --version - register: gatus_docker_check - changed_when: false - failed_when: gatus_docker_check.rc != 0 - -# Created explicitly rather than by either compose file, so neither the gatus -# stack nor the signal-api stack has to be deployed before the other. -- name: Ensure the shared monitoring network exists - ansible.builtin.command: "docker network create {{ gatus_network }}" - register: gatus_net - changed_when: "'already exists' not in gatus_net.stderr" - failed_when: - - gatus_net.rc != 0 - - "'already exists' not in gatus_net.stderr" - -- name: Create the gatus directories - ansible.builtin.file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode }}" - loop: - - {path: "{{ gatus_dir }}", owner: root, group: root, mode: "0755"} - # Config is root-owned and world-unreadable: it holds external-endpoint - # tokens. The container mounts it read-only and reads it as gatus_uid, so - # that id needs group access - hence the group ownership below. - - {path: "{{ gatus_config_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} - - {path: "{{ gatus_endpoints_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} - # Data is the one path the container writes to, so it must be owned by the - # numeric id the container runs as. `read_only: true` makes everything else - # in the container immutable. - - {path: "{{ gatus_data_dir }}", owner: "{{ gatus_uid }}", group: "{{ gatus_gid }}", mode: "0750"} - -- name: Write the base gatus configuration - ansible.builtin.template: - src: config.yaml.j2 - dest: "{{ gatus_config_dir }}/{{ gatus_base_config_file }}" - owner: root - group: "{{ gatus_gid }}" - mode: "0640" - notify: Restart gatus - -# Without this the container crash-loops on an empty config. See the template. -- name: Write the gatus self-check endpoint - ansible.builtin.template: - src: endpoint-self.yaml.j2 - dest: "{{ gatus_endpoints_dir }}/00-self.yaml" - owner: root - group: "{{ gatus_gid }}" - mode: "0640" - when: gatus_self_check | bool - notify: Restart gatus - -- name: Write the docker compose file - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ gatus_dir }}/docker-compose.yml" - owner: root - group: root - mode: "0644" - notify: Restart gatus diff --git a/ansible/roles/gatus/tasks/main.yml b/ansible/roles/gatus/tasks/main.yml deleted file mode 100644 index e1b67bf..0000000 --- a/ansible/roles/gatus/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# import, never include: dynamic includes are opaque to --list-tasks, which is -# the primary verification tool in this repo. -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml diff --git a/ansible/roles/gatus/tasks/service.yml b/ansible/roles/gatus/tasks/service.yml deleted file mode 100644 index bfcca3b..0000000 --- a/ansible/roles/gatus/tasks/service.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Pulling by digest means this either fetches exactly the reviewed image or -# fails. There is no "latest wins" path. -- name: Pull the pinned gatus image - ansible.builtin.command: - cmd: docker compose pull - chdir: "{{ gatus_dir }}" - register: gatus_pull - changed_when: "'Downloaded newer image' in gatus_pull.stderr or 'Pull complete' in gatus_pull.stderr" - -- name: Start gatus - ansible.builtin.command: - cmd: docker compose up -d --remove-orphans - chdir: "{{ gatus_dir }}" - register: gatus_up - changed_when: "'Started' in gatus_up.stderr or 'Created' in gatus_up.stderr or 'Recreated' in gatus_up.stderr" - -- name: Flush handlers so a config change is live before it is verified - ansible.builtin.meta: flush_handlers - -- name: Wait for gatus to answer - ansible.builtin.uri: - url: "http://{{ gatus_bind_address }}:{{ gatus_port }}/health" - status_code: [200, 404] - register: gatus_health - until: gatus_health.status in [200, 404] - retries: 12 - delay: 5 - -- name: Assert gatus is running - ansible.builtin.assert: - that: - - gatus_health.status in [200, 404] - fail_msg: "gatus did not come up on {{ gatus_bind_address }}:{{ gatus_port }} - check `docker logs gatus`" - success_msg: "gatus is answering on {{ gatus_bind_address }}:{{ gatus_port }}" diff --git a/ansible/roles/gatus/templates/config.yaml.j2 b/ansible/roles/gatus/templates/config.yaml.j2 deleted file mode 100644 index faf61fb..0000000 --- a/ansible/roles/gatus/templates/config.yaml.j2 +++ /dev/null @@ -1,64 +0,0 @@ -# {{ gatus_base_config_file }} — managed by Ansible (roles/gatus) -# -# BASE CONFIGURATION ONLY. -# -# Gatus merges every *.yaml under GATUS_CONFIG_PATH: maps are deep-merged and -# lists are appended, but a primitive defined in two files is ambiguous and -# upstream refuses it. So `web`, `storage`, `ui`, `alerting` and `security` are -# set here and MUST NOT appear in any other file in this directory. -# -# Endpoints are lists, so they append cleanly. Each service drops its own file -# into endpoints/ via the gatus_endpoint role - the same shape as caddy_site. - -web: - # 0.0.0.0 is the CONTAINER's interface, not the host's. This must not be - # 127.0.0.1: that is the container's own loopback, which docker-proxy cannot - # reach, and gatus comes up healthy while the published port refuses every - # connection. - # - # The isolation comes from the port mapping in docker-compose.yml, which - # publishes to {{ gatus_bind_address }} on the host. Caddy fronts that, and it - # matters because the external-endpoint push API shares this listener with the - # dashboard. - address: 0.0.0.0 - port: {{ gatus_port }} - -# Gatus reloads when its config changes. If the NEW config fails to parse it -# calls panic() - unless this is set, in which case it logs the error and keeps -# running on the old config. Endpoint files are contributed by other playbooks, -# so one malformed file would otherwise take the monitor down, which is the -# worst possible time to lose it. -skip-invalid-config-update: {{ gatus_skip_invalid_config_update | bool | lower }} - -storage: - type: {{ gatus_storage_type }} -{% if gatus_storage_type != 'memory' %} - path: {{ gatus_storage_path }} -{% endif %} - caching: {{ gatus_storage_caching | bool | lower }} - maximum-number-of-results: {{ gatus_storage_max_results }} - maximum-number-of-events: {{ gatus_storage_max_events }} - -ui: -{{ gatus_ui | to_nice_yaml(indent=2) | indent(2, true) }} -{% if gatus_alerting %} - -alerting: -{{ gatus_alerting | to_nice_yaml(indent=2) | indent(2, true) }} -{% else %} - -# No alerting provider is configured yet. Gatus still evaluates every condition -# and records every result; it simply has nowhere to shout. Setting -# `gatus_alerting` is the single change needed to wire one up. -{% endif %} -{% if gatus_basic_auth %} - -security: - basic: -{{ gatus_basic_auth | to_nice_yaml(indent=2) | indent(4, true) }} -{% endif %} -{% if gatus_maintenance %} - -maintenance: -{{ gatus_maintenance | to_nice_yaml(indent=2) | indent(2, true) }} -{% endif %} diff --git a/ansible/roles/gatus/templates/docker-compose.yml.j2 b/ansible/roles/gatus/templates/docker-compose.yml.j2 deleted file mode 100644 index 512c9c5..0000000 --- a/ansible/roles/gatus/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,53 +0,0 @@ -# Managed by Ansible (roles/gatus) -services: - gatus: - image: {{ gatus_image }} - container_name: gatus - restart: unless-stopped - - # The image is FROM scratch: no /etc/passwd, so there is no user to drop to - # by name and the default is root. Run it by numeric id instead. - user: "{{ gatus_uid }}:{{ gatus_gid }}" - - ports: - # Loopback on purpose. Caddy fronts this, and the external-endpoint push - # API is served from the same listener as the dashboard - publishing - # 0.0.0.0 would put both straight on the public internet. - - "{{ gatus_bind_address }}:{{ gatus_port }}:{{ gatus_port }}" - - environment: - GATUS_CONFIG_PATH: /config - GATUS_LOG_LEVEL: "{{ gatus_log_level }}" - - volumes: - - {{ gatus_config_dir }}:/config:ro - - {{ gatus_data_dir }}:/data - - # Hardening. The systemd unit this replaced got most of it from - # ProtectSystem/NoNewPrivileges/etc; these are the container equivalents. - read_only: true - security_opt: - - no-new-privileges:true - cap_drop: - - ALL -{% if gatus_allow_icmp %} - cap_add: - # icmp:// endpoints need raw sockets. Dropped above with ALL, added back - # explicitly so the grant is visible rather than inherited from root. - - NET_RAW -{% endif %} - - networks: - # Shared with signal-api, so alerts can be delivered by service name. - # 127.0.0.1 inside this container is the container, not the host. - - {{ gatus_network }} - - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - -networks: - {{ gatus_network }}: - external: true diff --git a/ansible/roles/gatus/templates/endpoint-self.yaml.j2 b/ansible/roles/gatus/templates/endpoint-self.yaml.j2 deleted file mode 100644 index 197ae56..0000000 --- a/ansible/roles/gatus/templates/endpoint-self.yaml.j2 +++ /dev/null @@ -1,25 +0,0 @@ -# 00-self.yaml — managed by Ansible (roles/gatus) -# -# Gatus refuses to start with no endpoints at all: -# panic: error parsing config: configuration should contain at least one -# endpoint or suite -# -# So the role ships one. Checking its own listener is not as circular as it -# looks: it proves the config directory parsed, the container is serving, and -# the storage backend accepted a write. If this row is missing from the -# dashboard, the dashboard is not telling you the truth about anything else. -# -# It is also what keeps `gatus` deployable before any service has contributed -# an endpoint file of its own. - -endpoints: - - name: gatus - group: infrastructure - url: "http://localhost:{{ gatus_port }}/health" - interval: 60s - conditions: - - "[STATUS] == 200" -{% if gatus_default_alerts %} - alerts: -{{ gatus_default_alerts | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} diff --git a/ansible/roles/gatus_endpoint/defaults/main.yml b/ansible/roles/gatus_endpoint/defaults/main.yml deleted file mode 100644 index 90a2c19..0000000 --- a/ansible/roles/gatus_endpoint/defaults/main.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -# One invocation writes ONE file into Gatus's endpoints directory. Gatus merges -# every *.yaml under GATUS_CONFIG_PATH and appends lists, so each caller owns -# its own file and they compose without coordinating - the same shape as -# caddy_site, where each service contributes its own vhost. - -# Filename stem: .yaml -gatus_endpoint_name: "" - -# PULLED endpoints - Gatus makes the request and evaluates conditions. -# - {name, group, url, interval, conditions: [...], alerts: [...]} -# -# A DNS check adds `dns: {query-type, query-name}` - and note that for those, -# `url` is the RESOLVER to ask, not the name being looked up. -# A domain-expiry check is just `url: ` with a [DOMAIN_EXPIRATION] -# condition; it uses WHOIS/RDAP and needs no scheme. -gatus_endpoint_pulled: [] - -# EXTERNAL endpoints - the host pushes its own result. Gatus never reaches out, -# which is what makes this work for machines behind NAT and for state that has -# no pollable surface at all (disk usage, ZFS health, UPS mains). -# -# - {name, group, token, heartbeat, alerts: [...]} -# -# `heartbeat` is the important one: if nothing reports within that window Gatus -# alerts. That is what makes a push check detect its own failure - a dead timer -# looks exactly like a dead host, which is the correct reading. -gatus_endpoint_external: [] - -# Where the files live. Matches roles/gatus. -gatus_config_dir: /opt/gatus/config -gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" -gatus_gid: 10001 - -# Alerts attached to every endpoint in this file that does not specify its own. -# -# Gatus's provider-level `default-alert` only supplies DEFAULTS - an endpoint -# still has to opt in with `alerts: - type: signal` or it alerts on nothing at -# all. With ~90 endpoints that cannot be written by hand, so it is applied here. -# -# failure-threshold is set by the CALLER, because the right value depends on the -# check's cadence and there is no single correct default. See the note in -# infra/400_host_monitoring.yml. -gatus_endpoint_default_alerts: [] - -# UI options applied to every PULLED endpoint that does not set its own. -# -# Only pulled endpoints can carry this - an external (push) endpoint has no `ui` -# field at all, because it has no conditions to display. -# -# Note that every hide-* option already defaults to false upstream, so there is -# nothing to un-hide. The one setting that genuinely shows MORE is -# resolve-successful-conditions: by default a failing check displays the real -# value - "[STATUS] (502) == 200" - while a passing one drops it and shows only -# "[STATUS] == 200". Turning it on resolves both, so a healthy DNS check shows -# the IP it actually resolved rather than just the assertion. -gatus_endpoint_default_ui: - resolve-successful-conditions: true diff --git a/ansible/roles/gatus_endpoint/tasks/main.yml b/ansible/roles/gatus_endpoint/tasks/main.yml deleted file mode 100644 index 40f3448..0000000 --- a/ansible/roles/gatus_endpoint/tasks/main.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Assert gatus_endpoint parameters are sane - ansible.builtin.assert: - that: - - gatus_endpoint_name | length > 0 - - gatus_endpoint_pulled | length > 0 or gatus_endpoint_external | length > 0 - fail_msg: >- - gatus_endpoint needs a name and at least one of gatus_endpoint_pulled or - gatus_endpoint_external. An empty file would contribute nothing, and a - config with no endpoints at all makes Gatus refuse to start. - -- name: "Write Gatus endpoints '{{ gatus_endpoint_name }}'" - ansible.builtin.template: - src: endpoints.yaml.j2 - dest: "{{ gatus_endpoints_dir }}/{{ gatus_endpoint_name }}.yaml" - owner: root - group: "{{ gatus_gid }}" - mode: "0640" - # No handler. Gatus polls its own config every 30s - # (main.listenToConfigurationFileChanges) and reloads itself, so writing the - # file IS the deploy. A handler here would also fail whenever this role is - # used without roles/gatus loaded. diff --git a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 deleted file mode 100644 index 680691e..0000000 --- a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 +++ /dev/null @@ -1,60 +0,0 @@ -# {{ gatus_endpoint_name }}.yaml — managed by Ansible (roles/gatus_endpoint) -# -# Contributed by a playbook, not hand-edited. Gatus appends the lists in every -# *.yaml under its config directory, so this file adds to whatever else is -# registered without knowing about it. -{% if gatus_endpoint_external %} - -external-endpoints: -{% for e in gatus_endpoint_external %} - - name: {{ e.name }} - group: {{ e.group }} - token: "{{ e.token }}" -{% if e.heartbeat is defined %} - heartbeat: - interval: {{ e.heartbeat }} -{% endif %} -{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} -{% if _alerts %} - alerts: -{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} -{# external endpoints have no `ui` field upstream - they carry no conditions #} -{% endfor %} -{% endif %} -{% if gatus_endpoint_pulled %} - -endpoints: -{% for e in gatus_endpoint_pulled %} - - name: {{ e.name }} - group: {{ e.group }} - url: "{{ e.url }}" - interval: {{ e.interval | default('60s') }} -{% if e.dns is defined %} - # A DNS endpoint: `url` is the RESOLVER to ask, not the thing being asked - # about. The name being queried lives in query-name, and [BODY] holds the - # resolved record for the conditions below. - dns: - query-type: {{ e.dns['query-type'] }} - query-name: {{ e.dns['query-name'] }} -{% endif %} -{% if e.client is defined %} - client: -{{ e.client | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} - conditions: -{% for c in e.conditions %} - - "{{ c }}" -{% endfor %} -{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} -{% if _alerts %} - alerts: -{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} -{% set _ui = e.ui | default(gatus_endpoint_default_ui) %} -{% if _ui %} - ui: -{{ _ui | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} -{% endfor %} -{% endif %} diff --git a/ansible/roles/healthcheck/defaults/main.yml b/ansible/roles/healthcheck/defaults/main.yml deleted file mode 100644 index 51534c2..0000000 --- a/ansible/roles/healthcheck/defaults/main.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -# One health check: a script, a systemd service, a timer, and an optional push. -# -# The exit code is the answer and systemd keeps it: -# systemctl is-failed -healthcheck.service -# Reporting anywhere else is optional and generic. Point healthcheck_push_url at -# Gatus, or at whatever replaces it, or at nothing. - -healthcheck_name: "" # e.g. disk-usage -> disk-usage-healthcheck -healthcheck_description: "" - -# The check itself. Pick ONE: -# healthcheck_check: a template under templates/checks/ (without .sh.j2) -# healthcheck_command: a shell one-liner that exits 0 for healthy -healthcheck_check: "" -healthcheck_command: "" - -# systemd timer. OnUnitActiveSec unless healthcheck_on_calendar is set. -healthcheck_interval: "5min" -healthcheck_on_calendar: "" -healthcheck_boot_delay: "2min" - -# ── Reporting ──────────────────────────────────────────────────────────────── -# Gatus external endpoints: -# POST {url}?success=true|false&error=... -# Authorization: Bearer {token} -# Empty url = check and log only, which is a valid state and not an error. -healthcheck_push_url: "" -healthcheck_push_token: "" - -# Some checks report MORE THAN ONE result - a host with four systemd services -# needs four endpoints, or a single red light cannot tell you which one died. -# Those set healthcheck_push_base to the endpoints COLLECTION and the check body -# appends each key itself, the same way check-backups.sh reports per source. -healthcheck_push_base: "" - -# Units for the systemd-units check. Each becomes its own Gatus endpoint. -healthcheck_units: [] -# Prefix for the per-unit endpoint keys, e.g. "services_vipy" -> services_vipy-caddy. -healthcheck_units_key_prefix: "" - -healthcheck_script_dir: /usr/local/bin -healthcheck_log_dir: /var/log/healthchecks - -# Per-check knobs, consumed by the templates under checks/ -healthcheck_disk_threshold: 85 # percent -healthcheck_cpu_temp_threshold: 80 # celsius -healthcheck_zfs_pool: "" -# ZFS degrades badly once a pool passes roughly 80% - allocation gets slow and -# fragmentation becomes hard to undo, and unlike a normal filesystem you cannot -# simply delete your way back to good performance. So this alarms well before -# the pool is actually out of space. -healthcheck_zfs_capacity_threshold: 80 -healthcheck_ups_name: "" diff --git a/ansible/roles/healthcheck/tasks/main.yml b/ansible/roles/healthcheck/tasks/main.yml deleted file mode 100644 index f37a0d5..0000000 --- a/ansible/roles/healthcheck/tasks/main.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -- name: "Assert healthcheck '{{ healthcheck_name }}' is fully specified" - ansible.builtin.assert: - that: - - healthcheck_name | length > 0 - - healthcheck_description | length > 0 - - (healthcheck_check | length > 0) != (healthcheck_command | length > 0) - - not (healthcheck_push_url | length > 0) or (healthcheck_push_token | length > 0) - fail_msg: >- - healthcheck needs a name, a description, exactly one of healthcheck_check - or healthcheck_command, and a token whenever a push URL is set. A push URL - with no token would report to Gatus and be rejected 401 on every run. - -# Deduplicated across the whole play run. This role is included once PER CHECK, -# and a host with several checks was otherwise running apt several times to -# install a curl that was already there - 29 apt transactions estate-wide, and -# the slowest thing in the deploy by a wide margin. The fact below remembers -# what has already been ensured on this host. -- name: Install healthcheck dependencies - ansible.builtin.package: - name: "{{ healthcheck_wanted_packages }}" - state: present - vars: - healthcheck_wanted_packages: >- - {{ (healthcheck_packages | default(['curl'])) - | difference(healthcheck_installed_packages | default([])) }} - when: healthcheck_wanted_packages | length > 0 - -- name: Remember which dependencies this host already has - ansible.builtin.set_fact: - healthcheck_installed_packages: >- - {{ (healthcheck_installed_packages | default([])) - | union(healthcheck_packages | default(['curl'])) }} - -- name: Create the healthcheck log directory - ansible.builtin.file: - path: "{{ healthcheck_log_dir }}" - state: directory - owner: root - group: root - mode: "0750" - -- name: "Install the {{ healthcheck_name }} check script" - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" - owner: root - group: root - mode: "0755" - -# The token is in this unit file, so it must not be world-readable. -- name: "Install the {{ healthcheck_name }} systemd service" - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.service" - owner: root - group: root - mode: "0600" - -- name: "Install the {{ healthcheck_name }} systemd timer" - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.timer" - owner: root - group: root - mode: "0644" - -# `restarted`, not `started`: started is a no-op on an already-active timer, so -# a changed interval or a stuck timer would never be picked up. -- name: "Enable and start the {{ healthcheck_name }} timer" - ansible.builtin.systemd: - name: "{{ healthcheck_name }}-healthcheck.timer" - enabled: yes - state: restarted - daemon_reload: yes - -- name: "Run the {{ healthcheck_name }} check once now" - ansible.builtin.command: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" - environment: - HEALTHCHECK_PUSH_URL: "{{ healthcheck_push_url }}" - HEALTHCHECK_PUSH_TOKEN: "{{ healthcheck_push_token }}" - register: healthcheck_first_run - changed_when: false - failed_when: false - -- name: "Report the first {{ healthcheck_name }} result" - ansible.builtin.debug: - msg: >- - {{ healthcheck_name }}: {{ 'HEALTHY' if healthcheck_first_run.rc == 0 - else 'UNHEALTHY (rc=' ~ healthcheck_first_run.rc ~ ')' }} diff --git a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 deleted file mode 100644 index cf57509..0000000 --- a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ - # Hottest core across every thermal zone and hwmon sensor lm-sensors knows - # about. Reading the hottest rather than an average is deliberate: one core - # throttling is a real problem that an average hides. - local threshold={{ healthcheck_cpu_temp_threshold }} - local hottest=0 label="" - - while read -r t; do - [ -z "$t" ] && continue - t=${t%.*} - if [ "$t" -gt "$hottest" ]; then hottest=$t; fi - done < <(sensors -u 2>/dev/null | awk '/_input:/ && /temp/ {print $2}') - - # Fall back to the kernel thermal zones if lm-sensors reports nothing. - if [ "$hottest" -eq 0 ]; then - for z in /sys/class/thermal/thermal_zone*/temp; do - [ -r "$z" ] || continue - local milli; milli=$(cat "$z" 2>/dev/null) || continue - local c=$((milli / 1000)) - if [ "$c" -gt "$hottest" ]; then hottest=$c; label=$(cat "${z%/temp}/type" 2>/dev/null); fi - done - fi - - if [ "$hottest" -eq 0 ]; then - MESSAGE="no temperature sensors readable" - return 1 - fi - MESSAGE="${hottest}C${label:+ (${label})}" - [ "$hottest" -lt "$threshold" ] diff --git a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 deleted file mode 100644 index b0e76ce..0000000 --- a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 +++ /dev/null @@ -1,19 +0,0 @@ - # Every real filesystem must be under the threshold. tmpfs, devtmpfs, - # squashfs and overlay are excluded: they are either RAM, read-only, or - # container layers, and none of them fills up in a way an operator can act on. - local threshold={{ healthcheck_disk_threshold }} - local worst=0 worst_mount="" over="" - - while read -r pct mount; do - pct=${pct%\%} - [ -z "$pct" ] && continue - if [ "$pct" -gt "$worst" ]; then worst=$pct; worst_mount=$mount; fi - if [ "$pct" -ge "$threshold" ]; then over="${over}${over:+, }${mount} ${pct}%"; fi - done < <(df -P -x tmpfs -x devtmpfs -x squashfs -x overlay --output=pcent,target 2>/dev/null | tail -n +2) - - if [ -n "$over" ]; then - MESSAGE="over ${threshold}%: ${over}" - return 1 - fi - MESSAGE="max ${worst}% on ${worst_mount:-/}" - return 0 diff --git a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 deleted file mode 100644 index 141e478..0000000 --- a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ - # Liveness has no test to run: the fact that this script executed at all is - # the signal. What proves the host is alive is the PUSH arriving at Gatus, - # and what detects the host being dead is the heartbeat window expiring with - # no push. So this always succeeds - the reporting is the check. - MESSAGE="up since $(uptime -p 2>/dev/null || echo unknown)" - return 0 diff --git a/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 b/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 deleted file mode 100644 index 90c6193..0000000 --- a/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 +++ /dev/null @@ -1,33 +0,0 @@ - # One result PER UNIT, not one for the host. A host running four services - # needs four endpoints: a single red light would tell you "something on vipy - # is down" without saying which, and that is the question you actually have. - # - # Keys are host-qualified because unit names collide - caddy runs on four - # machines. Gatus builds the key as sanitize(group)_sanitize(name), so - # group "services" + name "vipy/caddy" becomes services_vipy-caddy. - local prefix="{{ healthcheck_units_key_prefix }}" - local down="" - - for unit in {{ healthcheck_units | join(' ') }}; do - local state substate - state=$(systemctl is-active "$unit" 2>/dev/null || true) - substate=$(systemctl show -p SubState --value "$unit" 2>/dev/null || true) - - if [ "$state" = "active" ]; then - report_key "${prefix}-${unit}" "true" "${unit} active (${substate:-running})" - else - # `failed` and `inactive` are different stories: one crashed, one was - # stopped. Both are down, but the message should say which. - local detail="${unit} is ${state:-unknown}" - [ "$state" = "failed" ] && detail="${detail} (${substate:-failed})" - report_key "${prefix}-${unit}" "false" "$detail" - down="${down}${down:+, }${unit}=${state:-unknown}" - fi - done - - if [ -n "$down" ]; then - MESSAGE="down: ${down}" - return 1 - fi - MESSAGE="all {{ healthcheck_units | length }} units active" - return 0 diff --git a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 deleted file mode 100644 index 508c860..0000000 --- a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 +++ /dev/null @@ -1,18 +0,0 @@ - # OL means on line power. Anything else - OB (on battery), LB (low battery), - # or no answer at all - is a failure worth waking up for, because the - # hypervisor has a finite number of minutes left. - local ups="{{ healthcheck_ups_name }}" - local status charge runtime load - - status=$(upsc "${ups}@localhost" ups.status 2>/dev/null) - if [ -z "$status" ]; then - MESSAGE="cannot reach UPS ${ups} via upsd" - return 1 - fi - - charge=$(upsc "${ups}@localhost" battery.charge 2>/dev/null) - runtime=$(upsc "${ups}@localhost" battery.runtime 2>/dev/null) - load=$(upsc "${ups}@localhost" ups.load 2>/dev/null) - - MESSAGE="status=${status} charge=${charge}% runtime=${runtime}s load=${load}%" - [[ "$status" == *"OL"* ]] diff --git a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 deleted file mode 100644 index ebe88b9..0000000 --- a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 +++ /dev/null @@ -1,63 +0,0 @@ - # Five conditions, all of which have to hold. Ported from the check that - # infra/nodito/32_zfs_pool_setup_playbook.yml deployed, which was correct - - # only its reporting was tied to Uptime Kuma. - local pool="{{ healthcheck_zfs_pool }}" - local json issues="" - - json=$(zpool status -j "$pool" 2>&1) || { MESSAGE="zpool status failed: ${json}"; return 1; } - - # 1. pool state - local state - state=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].state') - [ "$state" = "ONLINE" ] || issues="${issues}${issues:+; }pool ${state}" - - # 2. every vdev and device ONLINE - local bad - bad=$(echo "$json" | jq -r --arg p "$pool" ' - .pools[$p].vdevs[] | .. | objects - | select(.state? and .state != "ONLINE") - | "\(.name // "unknown"):\(.state)"' 2>/dev/null | paste -sd, -) - [ -z "$bad" ] || issues="${issues}${issues:+; }devices ${bad}" - - # 3. resilver in progress - local fn st - fn=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.function // "NONE"') - st=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.state // "NONE"') - if [ "$fn" = "RESILVER" ] && [ "$st" = "SCANNING" ]; then - issues="${issues}${issues:+; }resilvering" - fi - - # 4. read/write/checksum errors. ZFS reports these as strings. - local errs - errs=$(echo "$json" | jq -r --arg p "$pool" ' - .pools[$p].vdevs[] | .. | objects - | select(.name? and ((.read_errors // "0" | tonumber) > 0 - or (.write_errors // "0" | tonumber) > 0 - or (.checksum_errors // "0" | tonumber) > 0)) - | "\(.name) r=\(.read_errors) w=\(.write_errors) c=\(.checksum_errors)"' 2>/dev/null | paste -sd, -) - [ -z "$errs" ] || issues="${issues}${issues:+; }errors ${errs}" - - # 5. errors from the last scrub - local scan_err - scan_err=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.errors // "0"') - if [ -n "$scan_err" ] && [ "$scan_err" != "0" ] && [ "$scan_err" != "null" ]; then - issues="${issues}${issues:+; }scan errors ${scan_err}" - fi - - # 6. Capacity. Not an error condition in `zpool status` - a 95% full pool is - # reported perfectly ONLINE - so it has to be read separately, and it is - # the failure you get warning of rather than the one you discover. - local capacity - capacity=$(zpool list -H -o capacity "$pool" 2>/dev/null | tr -dc '0-9') - if [ -z "$capacity" ]; then - issues="${issues}${issues:+; }cannot read capacity" - elif [ "$capacity" -ge {{ healthcheck_zfs_capacity_threshold }} ]; then - issues="${issues}${issues:+; }pool ${capacity}% full (>={{ healthcheck_zfs_capacity_threshold }}%)" - fi - - if [ -n "$issues" ]; then MESSAGE="$issues"; return 1; fi - - local scrub - scrub=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.start_time // "never"') - MESSAGE="${pool} ONLINE, ${capacity}% full, last scrub ${scrub}" - return 0 diff --git a/ansible/roles/healthcheck/templates/healthcheck.service.j2 b/ansible/roles/healthcheck/templates/healthcheck.service.j2 deleted file mode 100644 index 3ddd8aa..0000000 --- a/ansible/roles/healthcheck/templates/healthcheck.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description={{ healthcheck_description }} -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -User=root -ExecStart={{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -Environment=HEALTHCHECK_PUSH_BASE={{ healthcheck_push_base }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 deleted file mode 100644 index 6667878..0000000 --- a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# {{ healthcheck_name }} — {{ healthcheck_description }} -# Managed by Ansible (roles/healthcheck). Do not edit on the host. -# -# The exit code is the real answer; systemd stores it: -# systemctl is-failed {{ healthcheck_name }}-healthcheck.service -# The push below is an optional extra, and having no URL is normal. - -set -uo pipefail - -LOG_FILE="{{ healthcheck_log_dir }}/{{ healthcheck_name }}.log" -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -# Set only by checks that report several results; see healthcheck_push_base. -PUSH_BASE="${HEALTHCHECK_PUSH_BASE:-}" - -log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $*" >> "$LOG_FILE"; } - -# Report to Gatus as an external endpoint. Note this is a POST with a bearer -# token, not a GET with a query string - it is not the shape Uptime Kuma used. -report() { - local success="$1" message="$2" - - # No push URL is normal, not an error: the exit code below is a complete - # answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - - local code - code=$(curl -s -o /dev/null -w '%{http_code}' -X POST \ - --max-time 15 --retry 2 --retry-delay 3 \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null) - - if [ "$code" = "200" ]; then - log "reported success=${success}" - else - log "ERROR: report failed (HTTP ${code})" - return 1 - fi -} - -# Report to an arbitrary endpoint key under PUSH_BASE. Used by checks that -# produce one result per item rather than a single verdict. -report_key() { - local key="$1" success="$2" message="$3" - [ -n "$PUSH_BASE" ] || return 0 - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true -} - -# ── the check ──────────────────────────────────────────────────────────────── -# -# The blank line before the closing brace below is load-bearing. Jinja strips an -# included template's trailing newline, and trim_blocks (on by default in -# Ansible) then eats the newline after {% raw %}{% endif %}{% endraw %} - so without it the brace lands -# on the same line as the check body's last statement, producing `return 0}` and -# a script that dies with "syntax error: unexpected end of file". -check() { -{% if healthcheck_check %} -{% include 'checks/' ~ healthcheck_check ~ '.sh.j2' %} -{% else %} - {{ healthcheck_command }} -{% endif %} - -} - -MESSAGE="" -if check; then - log "OK${MESSAGE:+ - $MESSAGE}" - report "true" "${MESSAGE:-ok}" - exit 0 -else - log "FAILED${MESSAGE:+ - $MESSAGE}" - report "false" "${MESSAGE:-check failed}" - exit 1 -fi diff --git a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 deleted file mode 100644 index 0271735..0000000 --- a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=Run {{ healthcheck_description }} -Requires={{ healthcheck_name }}-healthcheck.service - -[Timer] -OnBootSec={{ healthcheck_boot_delay }} -{% if healthcheck_on_calendar %} -OnCalendar={{ healthcheck_on_calendar }} -{% else %} -OnUnitActiveSec={{ healthcheck_interval }} -{% endif %} -# Run a missed occurrence on the next boot rather than silently skipping it. -Persistent=true -# Spread the pushes out so twelve hosts do not all report in the same second. -RandomizedDelaySec={{ healthcheck_randomized_delay | default('30') }} - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/mempool/README.md b/ansible/roles/mempool/README.md deleted file mode 100644 index d8282cc..0000000 --- a/ansible/roles/mempool/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# `mempool` - -Deploys the [Mempool](https://mempool.space) block explorer as a three-container -Docker Compose stack — MariaDB, backend, frontend — on `mempool-box`, and keeps a -health check on each. - -Converted from `deploy_mempool_playbook.yml` (745 lines) under Plan 6. The -playbook is now 37 lines: this role, plus a second play that publishes the -frontend through Caddy on the edge host. - -## Phases - -| | | -|---|---| -| `docker.yml` | Docker engine: repo, key, packages, service | -| `deploy.yml` | directories, `docker-compose.yml`, pull, up, wait-for-healthy | -| `healthcheck.yml` | three check scripts, three services, three timers | - -## Three health checks, not one - -Mempool is three moving parts and knowing *which* one is down is the point, so -each gets its own check, unit and timer, driven by the `mempool_healthchecks` -list: - -| | checks | -|---|---| -| `mariadb` | `docker inspect` health status of `mempool-db` | -| `backend` | `GET /api/v1/backend-info` | -| `frontend` | `GET /` | - -Each records its answer in its exit code, which systemd keeps: -`systemctl is-failed mempool-backend-healthcheck.service`. Reporting elsewhere -is one field per check, `push_url`, and is the plug-in point for whatever -monitoring exists. Empty means check, exit honestly, report nowhere. The URLs -are credentials, so callers pass them from the vault. - -Nothing here is specific to a monitoring product. The embedded Python that -created monitors over the Uptime Kuma API, the `/tmp` credentials file, the -push-URL file read back and parsed, and three systemd `Environment=` rewrites -are gone. - -## MariaDB owns its own data directory - -`{{ mempool_mysql_dir }}` is bind-mounted into the container, which runs as uid -**999** and must create files there. The playbook this replaced declared -`owner: "{{ ansible_user }}"` (1000) on it, which had drifted from reality ever -since the containers were created — unnoticed, because the playbook had not been -run since. - -That was not academic. The first real run of this role pulled a newer -`mariadb:10.11` and recreated `mempool-db`; had the chown still been in place, -MariaDB would have come back to a directory it could not write. The role now -ensures the directory exists and leaves ownership to the container. - -## `mempool_frontend_port` lives in `services_config.yml` - -Two hosts need it: this role deploys the frontend on `mempool-box`, and the Caddy -play proxies to it from the edge host. A role default is invisible to the second -play, so the value lives in `service_settings.mempool.frontend_port` and the role -default derives from it. - -## Expect `changed=2` on a converged host - -`Pull Mempool images` and `Deploy Mempool containers with docker compose` are -bare `command:` tasks with no `changed_when`, so they always report changed. -That is the idempotent floor, not drift. Everything else reports `ok`. - -**`mariadb:10.11` is a moving tag**, so a run can pull a newer patch release and -recreate the database container. Pin it if that is not what you want. diff --git a/ansible/roles/mempool/defaults/main.yml b/ansible/roles/mempool/defaults/main.yml deleted file mode 100644 index 9ce8f6a..0000000 --- a/ansible/roles/mempool/defaults/main.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Mempool Configuration Variables - -# Version - Pinned to specific release -mempool_version: "v3.2.1" - -# Directories -mempool_dir: /opt/mempool -mempool_data_dir: "{{ mempool_dir }}/data" -mempool_mysql_dir: "{{ mempool_dir }}/mysql" - -# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) -bitcoin_host: "knots-box" -bitcoin_rpc_port: 8332 -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml - -# Network - Fulcrum Electrum server (via Tailnet Magic DNS) -fulcrum_host: "fulcrum-box" -fulcrum_port: 50001 -fulcrum_tls: "false" - -# Mempool network mode -mempool_network: "mainnet" - -# Container ports (internal) -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/mempool_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -mempool_frontend_port: 8080 -mempool_backend_port: 8999 - -# MariaDB settings -mariadb_database: "mempool" -mariadb_user: "mempool" -# Note: mariadb_mempool_password is loaded from group_vars/all/vault.yml - - - -# --- Health checks ---------------------------------------------------------- -# Three independent checks, because Mempool is three moving parts and knowing -# WHICH one is down is the whole point. Each records its answer in its exit -# code, which systemd keeps: -# systemctl is-failed mempool-backend-healthcheck.service -# -# push_url is where to report, and is the single plug-in point for whatever -# monitoring exists. Empty means check, exit honestly, report nowhere. -# The URLs are credentials, so callers pass them from the vault. -healthcheck_push_token: "" - -mempool_healthchecks: - - name: mariadb - label: MariaDB - push_url: "" - - name: backend - label: Backend - push_url: "" - - name: frontend - label: Frontend - push_url: "" diff --git a/ansible/roles/mempool/tasks/deploy.yml b/ansible/roles/mempool/tasks/deploy.yml deleted file mode 100644 index d98df7e..0000000 --- a/ansible/roles/mempool/tasks/deploy.yml +++ /dev/null @@ -1,75 +0,0 @@ ---- -- name: Create mempool directories - file: - path: "{{ item }}" - state: directory - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0755' - loop: - - "{{ mempool_dir }}" - - "{{ mempool_data_dir }}" - -# MariaDB owns its own data directory. The container runs as uid 999 and has to -# create files in there; this playbook declared owner: {{ ansible_user }} (1000), -# which had been drifting from reality ever since the containers were created and -# would have broken MariaDB the first time it needed a new file. It went -# unnoticed only because the playbook had not been run since. -# -# So: ensure the directory exists, and let the container own it. On a fresh -# install the mariadb image's entrypoint sets ownership itself. -- name: Ensure the MariaDB data directory exists - file: - path: "{{ mempool_mysql_dir }}" - state: directory - -- name: Create docker-compose.yml for Mempool - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ mempool_dir }}/docker-compose.yml" - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0644' -- name: Pull Mempool images - command: docker compose pull - args: - chdir: "{{ mempool_dir }}" - -- name: Deploy Mempool containers with docker compose - command: docker compose up -d - args: - chdir: "{{ mempool_dir }}" - -- name: Wait for MariaDB to be healthy - command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db - register: mariadb_health - until: mariadb_health.stdout == 'healthy' - retries: 30 - delay: 10 - changed_when: false - -- name: Wait for Mempool backend to start - uri: - url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - method: GET - status_code: 200 - timeout: 10 - register: backend_check - until: backend_check.status == 200 - retries: 30 - delay: 10 - ignore_errors: yes - -- name: Wait for Mempool frontend to be available - uri: - url: "http://localhost:{{ mempool_frontend_port }}" - method: GET - status_code: 200 - timeout: 10 - register: frontend_check - until: frontend_check.status == 200 - retries: 20 - delay: 5 - ignore_errors: yes - -# ═════════════════════════════════════════════════════════════════════════ diff --git a/ansible/roles/mempool/tasks/docker.yml b/ansible/roles/mempool/tasks/docker.yml deleted file mode 100644 index b97e63e..0000000 --- a/ansible/roles/mempool/tasks/docker.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -- name: Remove old Docker-related packages - apt: - name: - - docker.io - - docker-doc - - docker-compose - - podman-docker - - containerd - - runc - state: absent - purge: yes - autoremove: yes - -- name: Update apt cache - apt: - update_cache: yes - -- name: Install prerequisites - apt: - name: - - ca-certificates - - curl - state: present - -- name: Create directory for Docker GPG key - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - -- name: Download Docker GPG key - get_url: - url: https://download.docker.com/linux/debian/gpg - dest: /etc/apt/keyrings/docker.asc - mode: '0644' - -- name: Get Debian architecture - command: dpkg --print-architecture - register: deb_arch - changed_when: false - -- name: Add Docker repository - apt_repository: - repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" - filename: docker - state: present - update_cache: yes - -- name: Install Docker packages - apt: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present - update_cache: yes - -- name: Ensure Docker is started and enabled - systemd: - name: docker - enabled: yes - state: started - -- name: Add user to docker group - user: - name: "{{ ansible_user }}" - groups: docker - append: yes - diff --git a/ansible/roles/mempool/tasks/healthcheck.yml b/ansible/roles/mempool/tasks/healthcheck.yml deleted file mode 100644 index 1e1ac28..0000000 --- a/ansible/roles/mempool/tasks/healthcheck.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -# Three checks, one per moving part. The Uptime Kuma specifics that used to -# follow — an embedded Python script creating monitors over the API, a /tmp -# credentials file, a push-URL file read back and parsed, and three systemd -# Environment= rewrites — are gone. Where each reports is now hc.push_url. -- name: Create Mempool health check scripts - ansible.builtin.template: - src: "healthcheck-{{ hc.name }}.sh.j2" - dest: "/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh" - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" - -- name: Create systemd services for health checks - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.service" - owner: root - group: root - mode: "0600" - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" - -- name: Create systemd timers for health checks - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.timer" - owner: root - group: root - mode: '0644' - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" - -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -# Ungated on purpose: enabling a timer is deployment, not monitoring. The -# deprecation banner swept this up with the push plumbing, so Ansible stopped -# managing three timers that are in fact running on the host. -- name: Enable and start health check timers - systemd: - name: "mempool-{{ hc.name }}-healthcheck.timer" - enabled: yes - state: started - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" diff --git a/ansible/roles/mempool/tasks/main.yml b/ansible/roles/mempool/tasks/main.yml deleted file mode 100644 index 9feb652..0000000 --- a/ansible/roles/mempool/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: docker.yml -- ansible.builtin.import_tasks: deploy.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/mempool/templates/docker-compose.yml.j2 b/ansible/roles/mempool/templates/docker-compose.yml.j2 deleted file mode 100644 index fda2ad1..0000000 --- a/ansible/roles/mempool/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,75 +0,0 @@ -# All containers use host network for Tailscale MagicDNS resolution -services: - mariadb: - image: mariadb:10.11 - container_name: mempool-db - restart: unless-stopped - network_mode: host - environment: - MYSQL_DATABASE: "{{ mariadb_database }}" - MYSQL_USER: "{{ mariadb_user }}" - MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" - MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" - volumes: - - {{ mempool_mysql_dir }}:/var/lib/mysql - healthcheck: - test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - mempool-backend: - image: mempool/backend:{{ mempool_version }} - container_name: mempool-backend - restart: unless-stopped - network_mode: host - environment: - # Database (localhost since all containers share host network) - DATABASE_ENABLED: "true" - DATABASE_HOST: "127.0.0.1" - DATABASE_DATABASE: "{{ mariadb_database }}" - DATABASE_USERNAME: "{{ mariadb_user }}" - DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" - # Bitcoin Core/Knots (via Tailnet MagicDNS) - CORE_RPC_HOST: "{{ bitcoin_host }}" - CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" - CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" - CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" - # Electrum (Fulcrum via Tailnet MagicDNS) - ELECTRUM_HOST: "{{ fulcrum_host }}" - ELECTRUM_PORT: "{{ fulcrum_port }}" - ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" - # Mempool settings - MEMPOOL_NETWORK: "{{ mempool_network }}" - MEMPOOL_BACKEND: "electrum" - MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" - MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" - volumes: - - {{ mempool_data_dir }}:/backend/cache - depends_on: - mariadb: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - mempool-frontend: - image: mempool/frontend:{{ mempool_version }} - container_name: mempool-frontend - restart: unless-stopped - network_mode: host - environment: - FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" - BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" - depends_on: - - mempool-backend - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s diff --git a/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 deleted file mode 100644 index fe0f57c..0000000 --- a/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Mempool backend health check — managed by Ansible (roles/mempool) -# The exit code is the answer; systemd keeps it. Reporting is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -BACKEND_PORT="{{ mempool_backend_port }}" - -check() { - curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 -} - -report() { - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "$1" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true -} - -if check; then report up "OK"; exit 0 -else echo "Mempool backend not responding"; report down "Mempool backend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 deleted file mode 100644 index 8452202..0000000 --- a/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Mempool frontend health check — managed by Ansible (roles/mempool) -# The exit code is the answer; systemd keeps it. Reporting is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -FRONTEND_PORT="{{ mempool_frontend_port }}" - -check() { - curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 -} - -report() { - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "$1" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true -} - -if check; then report up "OK"; exit 0 -else echo "Mempool frontend not responding"; report down "Mempool frontend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 b/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 deleted file mode 100644 index 922adab..0000000 --- a/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Mempool MariaDB health check — managed by Ansible (roles/mempool) -# The exit code is the answer; systemd keeps it. Reporting is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -check() { -{% raw %} - [ "$(docker inspect --format='{{.State.Health.Status}}' mempool-db 2>/dev/null)" = "healthy" ] -{% endraw %} -} - -report() { - # No push URL is normal, not an error. The previous version logged - # "ERROR: UPTIME_KUMA_PUSH_URL not set" on every fire, once a minute. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "$1" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true -} - -if check; then report up "OK"; exit 0 -else echo "MariaDB container unhealthy"; report down "MariaDB container unhealthy"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck.service.j2 b/ansible/roles/mempool/templates/healthcheck.service.j2 deleted file mode 100644 index 76fa4e9..0000000 --- a/ansible/roles/mempool/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Mempool {{ hc.label }} Health Check -After=network.target docker.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ hc.push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/mempool/templates/healthcheck.timer.j2 b/ansible/roles/mempool/templates/healthcheck.timer.j2 deleted file mode 100644 index 868cb7c..0000000 --- a/ansible/roles/mempool/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Mempool {{ hc.name }} Health Check Timer - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/phoenixd/README.md b/ansible/roles/phoenixd/README.md deleted file mode 100644 index 3c147e3..0000000 --- a/ansible/roles/phoenixd/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# `phoenixd` - -Deploys and runs [phoenixd](https://phoenix.acinq.co/server), an ACINQ Lightning -node, on the edge host. LNBits uses it as a wallet backend. The HTTP API stays on -loopback — phoenixd is never published through Caddy. - -Converted from `deploy_phoenixd_playbook.yml` (552 lines) under Plan 6. The -playbook is now 18 lines. - -## Phases - -| | | -|---|---| -| `install.yml` | packages, system user, directories, versioned download and install | -| `service.yml` | systemd unit, start, then first-boot checks (config written, seed created) | -| `healthcheck.yml` | check script, unit, timer | - -## The seed - -`{{ phoenixd_data_dir }}/seed.dat` **is** the funds. phoenixd is deliberately -excluded from the automated backups (Plan 5, Model C): the seed is twelve fixed -words that never change, so an automated job would only manufacture more copies -of a static secret on more machines. Write them down offline, once. - -Note the live file is mode `0644`. That is phoenixd's own doing, not this role's, -and it is worth tightening. - -## Monitoring: one variable, no product knowledge - -The check asks the node itself — the service must be active **and** -`phoenix-cli getinfo` must return a `nodeId` — and records the answer in its exit -code, which systemd keeps: - -```bash -systemctl is-failed phoenixd-healthcheck.service -``` - -That is a complete answer with no monitoring system involved. To report -elsewhere, set `healthcheck_push_url` to anything accepting an HTTP ping. Gone -from this role: the embedded Python that created monitors over the Uptime Kuma -API, the `/tmp` credentials file, the push-URL file written and parsed back, and -the systemd `Environment=` rewrite. - -### Two things the conversion fixed - -**The check used to log an error once a minute.** Its `Environment=` push URL had -been empty since the decommissioning, and the script printed -`ERROR: UPTIME_KUMA_PUSH_URL not set` on every fire — roughly 1,400 times a day. -The exit code was still correct, so nothing was broken; it was pure noise, and -noise that trains you to ignore the log. An unset push URL is now normal and -silent. - -**`Enable and start phoenixd health check timer` was guarded by -`uptime_kuma_enabled`** and so had not run since the decommissioning — while the -timer itself was still live on the host from before. Ansible had quietly stopped -managing something that was still running. Ungated. diff --git a/ansible/roles/phoenixd/handlers/main.yml b/ansible/roles/phoenixd/handlers/main.yml deleted file mode 100644 index 7779684..0000000 --- a/ansible/roles/phoenixd/handlers/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Restart phoenixd - systemd: - name: phoenixd - state: restarted - daemon_reload: yes - -- name: Restart phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - state: restarted - daemon_reload: yes diff --git a/ansible/roles/phoenixd/tasks/healthcheck.yml b/ansible/roles/phoenixd/tasks/healthcheck.yml deleted file mode 100644 index d439047..0000000 --- a/ansible/roles/phoenixd/tasks/healthcheck.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -# Everything here answers "is phoenixd healthy" and records the answer. The -# Uptime Kuma specifics that used to follow it — an embedded Python script -# creating monitors over the API, a /tmp credentials file, a push-URL file read -# back and parsed, and a systemd Environment= rewrite — are gone. What reports -# where is now one variable, healthcheck_push_url. See the role README. -- name: Create phoenixd health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: "{{ phoenixd_healthcheck_script_path }}" - owner: root - group: root - mode: "0755" - validate: "bash -n %s" - -- name: Create phoenixd health check systemd service - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" - owner: root - group: root - mode: "0600" - notify: Restart phoenixd health check timer - -- name: Create phoenixd health check systemd timer - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" - owner: root - group: root - mode: "0644" - notify: Restart phoenixd health check timer - -- name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - -# Ungated on purpose. This was guarded by `uptime_kuma_enabled`, but enabling a -# timer is deployment, not monitoring — the deprecation banner swept it up with -# the push plumbing. The timer is in fact running on the host, from before the -# decommissioning, so the guard meant Ansible had stopped managing something -# that was still live. -- name: Enable and start phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - enabled: yes - state: started - -- name: Display post-install information - debug: - msg: | - ✓ phoenixd {{ phoenixd_version }} deployed - - Status: systemctl status phoenixd - Logs: journalctl -u phoenixd -f - CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo - HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) - Data dir: {{ phoenixd_data_dir }} - Health: systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service - - API password (needed to wire LNBits up to this node): - sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf - - ⚠️ BACK UP THE SEED: {{ phoenixd_data_dir }}/seed.dat - Losing it means losing the funds. phoenixd is deliberately excluded - from the automated backups (Plan 5, Model C) because the seed is 12 - fixed words — write them down offline, once: - sudo cat {{ phoenixd_data_dir }}/seed.dat diff --git a/ansible/roles/phoenixd/tasks/install.yml b/ansible/roles/phoenixd/tasks/install.yml deleted file mode 100644 index 89a4fa0..0000000 --- a/ansible/roles/phoenixd/tasks/install.yml +++ /dev/null @@ -1,125 +0,0 @@ ---- -- name: Install phoenixd runtime dependencies - apt: - name: - - unzip - - curl - state: present - update_cache: yes - -# System User and Directories -- name: Create phoenixd system group - group: - name: "{{ phoenixd_group }}" - system: yes - -- name: Create phoenixd system user - user: - name: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ phoenixd_home }}" - create_home: yes - comment: "phoenixd Lightning node" - -- name: Create phoenixd home directory - file: - path: "{{ phoenixd_home }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0750" - -- name: Create phoenixd data directory - file: - path: "{{ phoenixd_data_dir }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0700" - -# Download and Install -- name: Check if phoenixd is already installed - stat: - path: "{{ phoenixd_bin_dir }}/phoenixd" - register: phoenixd_binary - -- name: Check installed phoenixd version - command: "{{ phoenixd_bin_dir }}/phoenixd --version" - register: phoenixd_installed_version - changed_when: false - failed_when: false - when: phoenixd_binary.stat.exists - -- name: Decide whether phoenixd needs installing - set_fact: - phoenixd_needs_install: >- - {{ not phoenixd_binary.stat.exists - or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} - -- name: Download phoenixd {{ phoenixd_version }} - get_url: - url: "{{ phoenixd_url }}" - dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - mode: "0644" - when: phoenixd_needs_install | bool - -- name: Create temporary extraction directory - file: - path: /tmp/phoenixd-extract - state: directory - mode: "0755" - when: phoenixd_needs_install | bool - -- name: Extract phoenixd archive - unarchive: - src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - dest: /tmp/phoenixd-extract - remote_src: yes - when: phoenixd_needs_install | bool - -- name: Locate extracted binaries - find: - paths: /tmp/phoenixd-extract - patterns: "{{ item }}" - recurse: yes - file_type: file - register: phoenixd_extracted - loop: - - phoenixd - - phoenix-cli - when: phoenixd_needs_install | bool - -- name: Fail if the archive did not contain the expected binaries - assert: - that: - - item.files | length > 0 - fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - -- name: Install phoenixd and phoenix-cli binaries - copy: - src: "{{ item.files[0].path }}" - dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" - remote_src: yes - owner: root - group: root - mode: "0755" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - notify: Restart phoenixd - -- name: Clean up phoenixd download artifacts - file: - path: "{{ item }}" - state: absent - loop: - - "/tmp/phoenixd-{{ phoenixd_version }}.zip" - - /tmp/phoenixd-extract - diff --git a/ansible/roles/phoenixd/tasks/main.yml b/ansible/roles/phoenixd/tasks/main.yml deleted file mode 100644 index bc7ff05..0000000 --- a/ansible/roles/phoenixd/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/phoenixd/tasks/service.yml b/ansible/roles/phoenixd/tasks/service.yml deleted file mode 100644 index 79bf6f5..0000000 --- a/ansible/roles/phoenixd/tasks/service.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -- name: Build phoenixd command line arguments - set_fact: - phoenixd_args: >- - {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) - + ['--chain', phoenixd_chain] - + ['--auto-liquidity', phoenixd_auto_liquidity] - + ['--http-bind-ip', phoenixd_http_bind_ip] - + ['--http-bind-port', phoenixd_http_bind_port | string] - + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) - + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) - + ['--silent'] }} - -- name: Create phoenixd systemd service - ansible.builtin.template: - src: phoenixd.service.j2 - dest: /etc/systemd/system/phoenixd.service - owner: root - group: root - mode: "0644" - notify: Restart phoenixd - -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start phoenixd - systemd: - name: phoenixd - enabled: yes - state: started - -- name: Flush handlers so phoenixd is running before we inspect its data dir - meta: flush_handlers - -# --- First boot checks --- -- name: Wait for phoenixd to write its config file - wait_for: - path: "{{ phoenixd_data_dir }}/phoenix.conf" - state: present - timeout: 120 - -- name: Check that the seed file exists - stat: - path: "{{ phoenixd_data_dir }}/seed.dat" - register: phoenixd_seed_file - -- name: Fail if phoenixd did not create a seed - assert: - that: - - phoenixd_seed_file.stat.exists - fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" diff --git a/ansible/roles/phoenixd/templates/healthcheck.service.j2 b/ansible/roles/phoenixd/templates/healthcheck.service.j2 deleted file mode 100644 index 6ff21ab..0000000 --- a/ansible/roles/phoenixd/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=phoenixd Health Check -After=network.target phoenixd.service - -[Service] -Type=oneshot -User=root -ExecStart={{ phoenixd_healthcheck_script_path }} -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/phoenixd/templates/healthcheck.sh.j2 b/ansible/roles/phoenixd/templates/healthcheck.sh.j2 deleted file mode 100644 index a1717c5..0000000 --- a/ansible/roles/phoenixd/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# phoenixd health check — managed by Ansible (roles/phoenixd) -# -# Asks the node whether it is healthy and records the answer in the exit code, -# which systemd keeps: -# systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service -# That is a complete answer on its own. Reporting anywhere else is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" - -check_phoenixd() { - # Service must be active and the node must answer getinfo. - # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, - # but not the bind address, so pass it explicitly. - systemctl is-active --quiet phoenixd && \ - {{ phoenixd_bin_dir }}/phoenix-cli \ - --http-bind-ip {{ phoenixd_http_bind_ip }} \ - --http-bind-port {{ phoenixd_http_bind_port }} \ - getinfo 2>/dev/null | grep -q '"nodeId"' -} - -report() { - local status=$1 msg=$2 - # No push URL configured is NORMAL, not an error: the exit code below still - # answers the question. The previous version logged ERROR here on every - # single fire, once a minute, which is noise that trains you to ignore it. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true -} - -if check_phoenixd; then - report "up" "OK" - exit 0 -else - echo "phoenixd is not responding" - report "down" "phoenixd not responding" - exit 1 -fi diff --git a/ansible/roles/phoenixd/templates/healthcheck.timer.j2 b/ansible/roles/phoenixd/templates/healthcheck.timer.j2 deleted file mode 100644 index ee07257..0000000 --- a/ansible/roles/phoenixd/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=phoenixd Health Check Timer - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/phoenixd/templates/phoenixd.service.j2 b/ansible/roles/phoenixd/templates/phoenixd.service.j2 deleted file mode 100644 index 44fdb94..0000000 --- a/ansible/roles/phoenixd/templates/phoenixd.service.j2 +++ /dev/null @@ -1,31 +0,0 @@ -[Unit] -Description=phoenixd - Lightning Network Node -Documentation=https://phoenix.acinq.co/server -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User={{ phoenixd_user }} -Group={{ phoenixd_group }} -WorkingDirectory={{ phoenixd_home }} -Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} -ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} -Restart=always -RestartSec=30 -TimeoutStartSec=120 -TimeoutStopSec=120 -StandardOutput=journal -StandardError=journal - -# Hardening: the node only ever writes to its own data directory -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths={{ phoenixd_data_dir }} - -LimitNOFILE=65535 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/signal_api/README.md b/ansible/roles/signal_api/README.md deleted file mode 100644 index eadcfb7..0000000 --- a/ansible/roles/signal_api/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# signal_api - -Runs [signal-cli-rest-api](https://github.com/bbernhard/signal-cli-rest-api) on -the `observability` host. Gatus uses it to deliver alerts over Signal. - -Gatus does not speak Signal — it POSTs JSON to this service, which holds the -Signal identity and does the protocol work. - -## It is never published, and that is not optional - -**This API has no authentication of any kind.** No key, no token, no basic auth. -Anything that can reach the port can send messages as your identity and read -your Signal. So the compose file publishes **no ports at all** and there is no -Caddy vhost. - -Gatus reaches it over a shared docker network (`monitoring`) by service name: -`http://signal-api:8080`. That is also *why* a shared network is needed rather -than a published port — Gatus runs in a container, so `127.0.0.1` for Gatus is -the Gatus container, not the host. - -The network is created by an explicit Ansible task in both this role and -`gatus`, so neither stack has to be deployed before the other. - -## MODE, and why `native` - -Upstream offers `normal`, `native`, `json-rpc` and `json-rpc-native`. The -json-rpc modes keep a resident JVM daemon and upstream describes them as -"increased memory". - -**This VPS has 464 MB of RAM**, already running Gatus and Caddy. A resident JVM -is not affordable. `native` runs a precompiled GraalVM binary per request — no -daemon, no resident cost — and alerts are rare enough that paying startup cost -per alert is the right trade. - -## Linking the device — a one-time manual step - -Ansible cannot scan a QR code, so this is manual. **Do not use -`/v1/qrcodelink`** — it is broken in `native` mode. - -### The trap - -`GET /v1/qrcodelink?device_name=...` returns: - -```json -{"error":"Couldn't create QR code: no data to encode"} -``` - -The linking itself is fine: running the binary directly inside the container -emits a perfectly good provisioning URI. - -``` -$ docker exec signal-api signal-cli-native link -n gatus -sgnl://linkdevice?uuid=...&pub_key=... -``` - -It is the REST wrapper that fails to capture that output in `native` mode. - -**Do not "fix" this by switching MODE to `normal` or `json-rpc`.** That puts a -JVM in the path of *every alert* on a 464 MB host, permanently degrading the -running system to work around a step performed once. Generate the QR yourself -instead. - -### The procedure - -**`docker exec` runs as root, but the service runs as uid 1000.** Without -`--config`, signal-cli writes the linked account to `/root/.local/share/signal-cli` -— the container's ephemeral layer, NOT the mounted volume. It looks like it -worked (`Associated with: +34…`), `/v1/accounts` keeps returning `[]`, and the -account is destroyed on the next `docker compose up`. Always pass `--config`. - -1. Start the link and capture the URI. It must keep running while you scan: - - docker exec signal-api sh -c "rm -f /tmp/link.uri; \ - nohup signal-cli-native --config /home/.local/share/signal-cli \ - link -n gatus > /tmp/link.uri 2>/tmp/link.log & echo started" - sleep 10 - docker exec signal-api cat /tmp/link.uri - - Do **not** add `setsid`, and do **not** background `docker exec` itself from - the host — the first stops the URI appearing, the second is killed when the - Ansible task returns. The output is block-buffered because stdout is a file, - so the URI appears only after several seconds; `stdbuf` does not help, as the - buffering is GraalVM's, not libc's. - -2. Render the QR on your own machine and scan it: - - qrencode -o /tmp/qr.png -s 12 -m 4 "sgnl://linkdevice?uuid=...&pub_key=..." - -3. Phone: Signal → Settings → Linked devices → **+** → scan. Provisioning links - expire in a couple of minutes, so generate and scan in one sitting. - -4. Confirm — this must list the number, not `[]`: - - docker exec signal-api curl -s http://localhost:8080/v1/accounts - -5. Send a test message: - - docker exec signal-api curl -s -X POST -H "Content-Type: application/json" \ - -d '{"message":"test","number":"+34…","recipients":["+34…"]}' \ - http://localhost:8080/v2/send - -Alerts are sent **from your own number**, so sending to yourself lands in Note -to Self. If the device is ever unlinked from the phone, alerts stop silently — -which is why this service is itself monitored. - -### If the phone says "network error" - -The phone is not the problem. `chat.signal.org` resolves to AWS Global -Accelerator **dualstack** addresses with the AAAA records first, this container -has no IPv6 address at all, and this host's IPv6 path is broken — the same edge -that returned a bogus 404 for the Go tarball. signal-cli reaches for an -unreachable IPv6 address and dies with `Link request error: Connection closed!`, -while the phone can only report a failed handshake. - -That is what `gai.conf` (mounted at `/etc/gai.conf`) fixes. If linking starts -failing again, check it is still mounted and that `getent ahosts chat.signal.org` -returns an IPv4 address first. - -## Backups - -Deliberately **not** backed up. The data directory holds Signal private keys, -and the recovery path is to link again from the phone — which takes a minute and -does not depend on any stored artefact. Backing it up would copy a credential -off the host to buy nothing. - -## Verifying - -```bash -docker ps --filter name=signal-api -docker exec signal-api curl -fsS http://localhost:8080/v1/health -docker exec signal-api curl -fsS http://localhost:8080/v1/accounts -docker logs signal-api --tail 50 -``` diff --git a/ansible/roles/signal_api/defaults/main.yml b/ansible/roles/signal_api/defaults/main.yml deleted file mode 100644 index 6bb7d4f..0000000 --- a/ansible/roles/signal_api/defaults/main.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# signal-cli-rest-api: the transport Gatus uses to send Signal messages. -# -# Gatus does not speak Signal. It POSTs JSON to this service, which holds the -# actual Signal identity and does the protocol work. - -# Pinned by digest for the same reason as Gatus: a tag is mutable. -# Upstream publishes no versioned tags worth pinning to, so this pins the -# DIGEST that `latest` resolved to when this was reviewed. `latest` is a moving -# target; a digest is a content address, and `docker compose pull` either -# fetches exactly this image or fails. -signal_api_image_digest: "sha256:2399d449123cdad56c4d859277e3b9127e1a00c4d2ab4601c239882609286cf8" -signal_api_image: "bbernhard/signal-cli-rest-api@{{ signal_api_image_digest }}" - -signal_api_dir: /opt/signal-api -signal_api_data_dir: "{{ signal_api_dir }}/data" - -# MODE matters on this host. Upstream offers normal / native / json-rpc / -# json-rpc-native. json-rpc keeps a resident JVM daemon and upstream describes it -# as "increased memory" - this VPS has 464MB total and already runs Gatus and -# Caddy, so a resident JVM is not affordable. `native` runs a precompiled -# GraalVM binary per request: no daemon, no resident cost, and alerts are rare -# enough that paying startup per alert is the right trade. -signal_api_mode: native - -# Port INSIDE the shared docker network. Never published to the host: this API -# has NO AUTHENTICATION of any kind. Anyone who can reach it can send messages -# as you and read your Signal. -signal_api_port: 8080 - -# Both this and Gatus join this network so Gatus can reach the API by service -# name. Gatus runs in a container, so the host's loopback is NOT reachable from -# it - this is why a shared network is required rather than a published port. -signal_api_network: monitoring -signal_api_service_name: signal-api - -# The uid the upstream image drops to (`setpriv --reuid=1000`). The data -# directory must be owned by it or signal-cli cannot write the account. -signal_api_uid: 1000 diff --git a/ansible/roles/signal_api/tasks/main.yml b/ansible/roles/signal_api/tasks/main.yml deleted file mode 100644 index 1aa92b4..0000000 --- a/ansible/roles/signal_api/tasks/main.yml +++ /dev/null @@ -1,93 +0,0 @@ ---- -- name: Assert Docker is available - ansible.builtin.command: docker --version - register: signal_docker_check - changed_when: false - -# Created explicitly rather than by either compose file, so neither stack has to -# be deployed before the other and neither owns it. -- name: Ensure the shared monitoring network exists - ansible.builtin.command: "docker network create {{ signal_api_network }}" - register: signal_net - changed_when: "'already exists' not in signal_net.stderr" - failed_when: - - signal_net.rc != 0 - - "'already exists' not in signal_net.stderr" - -- name: Create the signal-api directory - ansible.builtin.file: - path: "{{ signal_api_dir }}" - state: directory - owner: root - group: root - mode: "0755" - -# Owned by the container's uid, NOT root. -# -# The image drops to uid 1000 (`setpriv --reuid=1000`), and a root-owned 0700 -# directory cannot be traversed by uid 1000 - signal-cli then fails to write the -# account and linking silently never completes, leaving a 39-byte accounts.json -# with no accounts and the API returning "Failed to read local accounts list". -# -# 0700 on uid 1000 is still private: only that uid and root can read the Signal -# private keys, which is the property actually wanted. -- name: Create the signal-api data directory owned by the container user - ansible.builtin.file: - path: "{{ signal_api_data_dir }}" - state: directory - owner: "{{ signal_api_uid }}" - group: "{{ signal_api_uid }}" - mode: "0700" - -- name: Write the IPv4-preference resolver config - ansible.builtin.template: - src: gai.conf.j2 - dest: "{{ signal_api_dir }}/gai.conf" - owner: root - group: root - mode: "0644" - -- name: Write the docker compose file - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ signal_api_dir }}/docker-compose.yml" - owner: root - group: root - mode: "0644" - -- name: Pull the pinned signal-api image - ansible.builtin.command: - cmd: docker compose pull - chdir: "{{ signal_api_dir }}" - register: signal_pull - changed_when: "'Downloaded newer image' in signal_pull.stderr or 'Pull complete' in signal_pull.stderr" - -- name: Start signal-api - ansible.builtin.command: - cmd: docker compose up -d --remove-orphans - chdir: "{{ signal_api_dir }}" - register: signal_up - changed_when: "'Started' in signal_up.stderr or 'Created' in signal_up.stderr or 'Recreated' in signal_up.stderr" - -- name: Wait for the API to answer - ansible.builtin.command: - cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/health" - register: signal_health - until: signal_health.rc == 0 - retries: 12 - delay: 5 - changed_when: false - -- name: Report whether an account is linked yet - ansible.builtin.command: - cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/accounts" - register: signal_accounts - changed_when: false - failed_when: false - -- name: Show the linking status - ansible.builtin.debug: - msg: >- - {{ 'Linked account(s): ' ~ signal_accounts.stdout - if (signal_accounts.stdout | default('[]') | trim) not in ['[]', '', 'null'] - else 'NO ACCOUNT LINKED YET - this is a one-time manual step, see the role README.' }} diff --git a/ansible/roles/signal_api/templates/docker-compose.yml.j2 b/ansible/roles/signal_api/templates/docker-compose.yml.j2 deleted file mode 100644 index 9825299..0000000 --- a/ansible/roles/signal_api/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ -# Managed by Ansible (roles/signal_api) -services: - {{ signal_api_service_name }}: - image: {{ signal_api_image }} - container_name: {{ signal_api_service_name }} - restart: unless-stopped - - environment: - MODE: "{{ signal_api_mode }}" - - volumes: - # Prefer IPv4. See gai.conf.j2 - without this, signal-cli reaches for - # chat.signal.org's IPv6 address, which is unreachable from here, and - # linking fails with an opaque "network error" on the phone. - - {{ signal_api_dir }}/gai.conf:/etc/gai.conf:ro - # Holds the Signal identity: the linked-device keys and registration - # state. Lose this and the device must be linked again by scanning a new - # QR code from the phone. It is also the most sensitive thing on this - # host - anyone with these keys can send and read Signal as you. - - {{ signal_api_data_dir }}:/home/.local/share/signal-cli - - networks: - - {{ signal_api_network }} - - # NO PORTS. Deliberately. - # - # This API has no authentication whatsoever - no key, no token, nothing. - # Publishing it, even on 127.0.0.1, would expose "send a Signal message as - # this identity" to anything that can reach the host. Gatus talks to it over - # the shared docker network by service name instead, which is why no port is - # published and why there is no Caddy vhost. - - healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:8080/v1/health"] - interval: 60s - timeout: 5s - retries: 3 - - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - -networks: - {{ signal_api_network }}: - external: true diff --git a/ansible/roles/signal_api/templates/gai.conf.j2 b/ansible/roles/signal_api/templates/gai.conf.j2 deleted file mode 100644 index 9b2cca4..0000000 --- a/ansible/roles/signal_api/templates/gai.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -# Managed by Ansible (roles/signal_api) -# -# Prefer IPv4 over IPv6 in getaddrinfo. -# -# chat.signal.org resolves to AWS Global Accelerator dualstack addresses, and -# DNS returns the AAAA records first. This container has NO IPv6 address at all, -# and this host's IPv6 path is unreliable anyway - the same edge that made -# Google's IPv6 endpoint return a confident 404 for the Go tarball. -# -# signal-cli would connect to the AAAA address, fail, and report -# Link request error: Connection closed! -# while the phone showed a bare "network error" - a failure with no obvious -# cause on either end. -# -# This line flips the precedence so IPv4-mapped addresses sort first, which is -# the standard glibc fix. It does NOT disable IPv6; it only changes the order. -precedence ::ffff:0:0/96 100 diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml new file mode 100644 index 0000000..c9bd7ca --- /dev/null +++ b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml @@ -0,0 +1,38 @@ +# Bitcoin Knots Configuration Variables + +# Version - REQUIRED: Specify exact version/tag to build +bitcoin_knots_version: "v29.2.knots20251110" # Must specify exact version/tag +bitcoin_knots_version_short: "29.2.knots20251110" # Version without 'v' prefix (for tarball URLs) + +# Directories +bitcoin_knots_dir: /opt/bitcoin-knots +bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" +bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets +bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) +bitcoin_conf_dir: /etc/bitcoin + +# Network +bitcoin_rpc_port: 8332 +bitcoin_p2p_port: 8333 +bitcoin_rpc_bind: "0.0.0.0" + +# Build options +bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores +bitcoin_build_prefix: /usr/local + +# Configuration options +bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) +bitcoin_max_connections: 125 +# dbcache will be calculated as 90% of host RAM automatically in playbook + +# ZMQ Configuration +bitcoin_zmq_enabled: true +bitcoin_zmq_bind: "tcp://0.0.0.0" +bitcoin_zmq_port_rawblock: 28332 +bitcoin_zmq_port_rawtx: 28333 +bitcoin_zmq_port_hashblock: 28334 +bitcoin_zmq_port_hashtx: 28335 + +# Service user +bitcoin_user: bitcoin +bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index d67006b..1ae5823 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -1,25 +1,757 @@ ---- -# Bitcoin Knots: full node built from source, with PGP signature and SHA256 -# verification of the release tarball. The build is guarded by a binary-exists -# check, so a converged host skips the whole 30-60 minute compile. -# -# The chain lives in bitcoin_large_data_dir (/mnt/knots_data, ~875 GB). Nothing -# here touches it; the only `state: absent` tasks target the build tree under -# /opt/bitcoin-knots and run only when the binary is missing. - name: Build and Deploy Bitcoin Knots from Source hosts: bitcoin become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./bitcoin_knots_vars.yml vars: - # Preserves the push URL this check has been reporting to. The role knows - # nothing about Uptime Kuma — this is just "a URL that accepts a ping". - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_bitcoin-knots/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - bitcoin_knots + bitcoin_repo_url: "https://github.com/bitcoinknots/bitcoin.git" + bitcoin_sigs_base_url: "https://raw.githubusercontent.com/bitcoinknots/guix.sigs/knots" + bitcoin_version_major: "{{ bitcoin_knots_version_short | regex_replace('^(\\d+)\\..*', '\\1') }}" + bitcoin_source_tarball_url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Calculate 90% of system RAM for dbcache + set_fact: + bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" + changed_when: false + + - name: Display calculated dbcache value + debug: + msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" + + + - name: Install build dependencies + apt: + name: + - build-essential + - libtool + - autotools-dev + - automake + - pkg-config + - bsdmainutils + - python3 + - python3-pip + - libevent-dev + - libboost-system-dev + - libboost-filesystem-dev + - libboost-test-dev + - libboost-thread-dev + - libboost-chrono-dev + - libboost-program-options-dev + - libboost-dev + - libssl-dev + - libdb-dev + - libminiupnpc-dev + - libzmq3-dev + - libnatpmp-dev + - libsqlite3-dev + - git + - curl + - wget + - cmake + state: present + update_cache: yes + + - name: Create bitcoin group + group: + name: "{{ bitcoin_group }}" + system: yes + state: present + + - name: Create bitcoin user + user: + name: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ bitcoin_data_dir }}" + create_home: yes + state: present + + - name: Create bitcoin-knots directory + file: + path: "{{ bitcoin_knots_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create bitcoin-knots source directory + file: + path: "{{ bitcoin_knots_source_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create bitcoin data directory (for config, logs, wallets) + file: + path: "{{ bitcoin_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + + - name: Create bitcoin large data directory (for blockchain) + file: + path: "{{ bitcoin_large_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + + - name: Create bitcoin config directory + file: + path: "{{ bitcoin_conf_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Check if bitcoind binary already exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_binary_exists + changed_when: false + + - name: Install gnupg for signature verification + apt: + name: gnupg + state: present + when: not bitcoind_binary_exists.stat.exists + + - name: Import Luke Dashjr's Bitcoin Knots signing key + command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 + register: key_import + changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" + when: not bitcoind_binary_exists.stat.exists + failed_when: key_import.rc != 0 + + - name: Display imported key fingerprint + command: gpg --fingerprint 90C8019E36C2E964 + register: key_fingerprint + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Download SHA256SUMS file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + + - name: Download SHA256SUMS.asc signature file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + + - name: Verify PGP signature on SHA256SUMS file + command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS + register: sha256sums_verification + changed_when: false + failed_when: false # Don't fail here - check for 'Good signature' in next task + when: not bitcoind_binary_exists.stat.exists + + + - name: Display SHA256SUMS verification result + debug: + msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" + when: not bitcoind_binary_exists.stat.exists + + - name: Fail if SHA256SUMS signature verification failed + fail: + msg: "SHA256SUMS signature verification failed. Aborting build." + when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) + + - name: Remove any existing tarball to force fresh download + file: + path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Download Bitcoin Knots source tarball + get_url: + url: "{{ bitcoin_source_tarball_url }}" + dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + mode: '0644' + validate_certs: yes + force: yes + when: not bitcoind_binary_exists.stat.exists + + - name: Calculate SHA256 checksum of downloaded tarball + command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + register: tarball_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Extract expected checksum from SHA256SUMS file + shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' + register: expected_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + failed_when: expected_checksum.stdout == "" + + - name: Display checksum comparison + debug: + msg: + - "Expected: {{ expected_checksum.stdout | trim }}" + - "Actual: {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists + + - name: Verify tarball checksum matches SHA256SUMS + fail: + msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] + + - name: Remove existing source directory if it exists (to force fresh extraction) + file: + path: "{{ bitcoin_knots_source_dir }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Remove extracted directory if it exists (from previous runs) + file: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Extract verified source tarball + unarchive: + src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + dest: "{{ bitcoin_knots_dir }}" + remote_src: yes + when: not bitcoind_binary_exists.stat.exists + + - name: Check if extracted directory exists + stat: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + register: extracted_dir_stat + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Rename extracted directory to expected name + command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" + when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists + + - name: Check if CMakeLists.txt exists + stat: + path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" + register: cmake_exists + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Create CMake build directory + file: + path: "{{ bitcoin_knots_source_dir }}/build" + state: directory + mode: '0755' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + + - name: Configure Bitcoin Knots build with CMake + command: > + cmake + -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} + -DBUILD_BITCOIN_WALLET=OFF + -DCMAKE_BUILD_TYPE=Release + -DWITH_ZMQ=ON + .. + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: configure_result + changed_when: true + + - name: Verify CMake enabled ZMQ + shell: | + set -e + cd "{{ bitcoin_knots_source_dir }}/build" + cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: zmq_check + changed_when: false + + - name: Fail if CMakeLists.txt not found + fail: + msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." + when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) + + - name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) + command: cmake --build . -j{{ bitcoin_build_jobs }} + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + async: 3600 + poll: 0 + register: build_result + changed_when: true + + - name: Check build status + async_status: + jid: "{{ build_result.ansible_job_id }}" + register: build_job_result + until: build_job_result.finished + retries: 120 + delay: 60 + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined + + - name: Fail if build failed + fail: + msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) + + - name: Install Bitcoin Knots binaries + command: cmake --install . + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + changed_when: true + + - name: Verify bitcoind binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_installed + changed_when: false + + - name: Verify bitcoin-cli binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" + register: bitcoin_cli_installed + changed_when: false + + - name: Fail if binaries not found + fail: + msg: "Bitcoin Knots binaries not found after installation" + when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists + + - name: Create bitcoin.conf configuration file + copy: + dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" + content: | + # Bitcoin Knots Configuration + # Generated by Ansible + + # Data directory (blockchain storage) + datadir={{ bitcoin_large_data_dir }} + + # RPC Configuration + server=1 + rpcuser={{ bitcoin_rpc_user }} + rpcpassword={{ bitcoin_rpc_password }} + rpcbind={{ bitcoin_rpc_bind }} + rpcport={{ bitcoin_rpc_port }} + rpcallowip=0.0.0.0/0 + + # Network Configuration + listen=1 + port={{ bitcoin_p2p_port }} + maxconnections={{ bitcoin_max_connections }} + + # Performance + dbcache={{ bitcoin_dbcache_mb }} + + # Transaction Index (optional) + {% if bitcoin_enable_txindex %} + txindex=1 + {% endif %} + + # Logging (to journald via systemd) + logtimestamps=1 + printtoconsole=1 + + # ZMQ Configuration + {% if bitcoin_zmq_enabled | default(false) %} + zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} + zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} + zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} + zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} + {% endif %} + + # Security + disablewallet=1 + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0640' + notify: Restart bitcoind + + - name: Create systemd service file for bitcoind + copy: + dest: /etc/systemd/system/bitcoind.service + content: | + [Unit] + Description=Bitcoin Knots daemon + After=network.target + + [Service] + Type=simple + User={{ bitcoin_user }} + Group={{ bitcoin_group }} + ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf + Restart=always + RestartSec=10 + TimeoutStopSec=600 + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + notify: Restart bitcoind + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start bitcoind service + systemd: + name: bitcoind + enabled: yes + state: started + + - name: Wait for bitcoind RPC to be available + uri: + url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" + method: POST + body_format: json + body: + jsonrpc: "1.0" + id: "healthcheck" + method: "getblockchaininfo" + params: [] + user: "{{ bitcoin_rpc_user }}" + password: "{{ bitcoin_rpc_password }}" + status_code: 200 + timeout: 10 + register: rpc_check + until: rpc_check.status == 200 + retries: 30 + delay: 5 + ignore_errors: yes + + - name: Display RPC connection status + debug: + msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Bitcoin Knots health check and push script + when: uptime_kuma_enabled | default(false) + copy: + dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh + content: | + #!/bin/bash + # + # Bitcoin Knots Health Check and Push to Uptime Kuma + # Checks if bitcoind RPC is responding and pushes status to Uptime Kuma + # + + RPC_HOST="{{ bitcoin_rpc_bind }}" + RPC_PORT={{ bitcoin_rpc_port }} + RPC_USER="{{ bitcoin_rpc_user }}" + RPC_PASSWORD="{{ bitcoin_rpc_password }}" + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + # Check if bitcoind RPC is responding + check_bitcoind() { + local response + response=$(curl -s --max-time 30 \ + --user "${RPC_USER}:${RPC_PASSWORD}" \ + --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ + --header 'Content-Type: application/json' \ + "http://${RPC_HOST}:${RPC_PORT}" 2>&1) + + if [ $? -eq 0 ]; then + # Check if response contains a non-null error + # Successful responses have "error": null, failures have "error": {...} + if echo "$response" | grep -q '"error":null\|"error": null'; then + return 0 + else + return 1 + fi + else + return 1 + fi + } + + # Push status to Uptime Kuma + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + + # URL encode spaces in message + local encoded_msg="${msg// /%20}" + + if ! curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then + echo "ERROR: Failed to push to Uptime Kuma" + return 1 + fi + } + + # Main health check + if check_bitcoind; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "bitcoind RPC not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Install curl for health check script + apt: + name: curl + state: present + + - name: Create systemd timer for Bitcoin Knots health check + copy: + dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer + content: | + [Unit] + Description=Bitcoin Knots Health Check Timer + Requires=bitcoind.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for Bitcoin Knots health check + when: uptime_kuma_enabled | default(false) + copy: + dest: /etc/systemd/system/bitcoin-knots-healthcheck.service + content: | + [Unit] + Description=Bitcoin Knots Health Check and Push to Uptime Kuma + After=network.target bitcoind.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + + - name: Enable and start Bitcoin Knots health check timer + systemd: + name: bitcoin-knots-healthcheck.timer + enabled: yes + state: started + + - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_bitcoin_knots_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for new monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "Bitcoin Knots" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_bitcoin_knots_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Extract push URL from monitor setup output + set_fact: + uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" + delegate_to: localhost + become: no + when: monitor_setup.stdout is defined + + - name: Display extracted push URL + debug: + msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" + when: uptime_kuma_push_url | default('') != '' + + - name: Set push URL in systemd service environment + lineinfile: + path: /etc/systemd/system/bitcoin-knots-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" + state: present + insertafter: '^\[Service\]' + when: uptime_kuma_push_url | default('') != '' + + - name: Reload systemd daemon after push URL update + systemd: + daemon_reload: yes + when: uptime_kuma_push_url | default('') != '' + + - name: Restart health check timer to pick up new environment + systemd: + name: bitcoin-knots-healthcheck.timer + state: restarted + when: uptime_kuma_push_url | default('') != '' + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_bitcoin_knots_monitor.py + - /tmp/ansible_config.yml + + handlers: + - name: Restart bitcoind + when: uptime_kuma_enabled | default(false) + systemd: + name: bitcoind + state: restarted + - name: Setup public Bitcoin P2P forwarding on the edge host hosts: edge become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./bitcoin_knots_vars.yml + vars: + bitcoin_tailscale_hostname: "knots-box" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + tasks: - name: Expose Bitcoin P2P through a socket proxy ansible.builtin.include_role: @@ -27,9 +759,134 @@ vars: socket_proxy_name: bitcoin-p2p socket_proxy_description: "Bitcoin P2P" - socket_proxy_listen_port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}" - socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" + socket_proxy_listen_port: "{{ bitcoin_p2p_port }}" + socket_proxy_upstream_host: "{{ bitcoin_tailscale_hostname }}" + # These four were added by hand on vipy and were NOT in this playbook; + # writing the unit without them would have dropped FreeBind, which lets + # the socket bind before the address is up. socket_proxy_documentation: "https://github.com/bitcoin/bitcoin" socket_proxy_free_bind: true socket_proxy_timeout_stop_sec: 5 socket_proxy_log_to_journal: true + + - name: Display public endpoint + when: uptime_kuma_enabled | default(false) + debug: + msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public P2P + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_bitcoin_p2p_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_bitcoin_p2p_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for TCP monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_bitcoin_p2p_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ bitcoin_p2p_port }} + monitor_name: "Bitcoin Knots P2P Public" + mode: '0644' + + - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up TCP monitor temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_bitcoin_p2p_tcp_monitor.py + - /tmp/ansible_bitcoin_p2p_config.yml + diff --git a/ansible/services/datum-gateway/datum_gateway_vars.yml b/ansible/services/datum-gateway/datum_gateway_vars.yml new file mode 100644 index 0000000..48bb5b7 --- /dev/null +++ b/ansible/services/datum-gateway/datum_gateway_vars.yml @@ -0,0 +1,43 @@ +# DATUM Gateway Configuration Variables +# https://github.com/OCEAN-xyz/datum_gateway + +# Version - pin to a specific tag +datum_gateway_version: "v0.4.1beta" + +# Directories +datum_gateway_dir: /opt/datum-gateway +datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" +datum_gateway_config_dir: /etc/datum-gateway +datum_gateway_log_dir: /var/log/datum-gateway + +# Binary +datum_gateway_bin_path: /usr/local/bin/datum_gateway + +# Ports +datum_gateway_stratum_port: 23334 # Miners connect here via Stratum v1 +datum_gateway_api_port: 7152 # Web dashboard / API + +# Stratum settings +datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) + +# Service user +datum_gateway_user: datum +datum_gateway_group: datum + +# Build options +datum_gateway_build_jobs: 4 + +# Bitcoin node connection +# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. +# datum_bitcoin_rpc_url should include http:// and port. +datum_bitcoin_rpc_url: "http://127.0.0.1:8332" +# Note: bitcoin_rpc_user and bitcoin_rpc_password come from infra_secrets.yml + +# Mining config +datum_coinbase_tag_primary: "DATUM" +datum_coinbase_tag_secondary: "BY ORDER OF BIP110" + +datum_pool_pass_workers: true +datum_pool_pass_full_users: true +datum_pooled_mining_only: true + diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 2e88a27..0c53e9d 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -1,49 +1,667 @@ --- -# DATUM Gateway: solo/pooled mining gateway, built from source on knots-box. +# DATUM Gateway Deployment Playbook # -# This is HALF OF A SYSTEM. The Bitcoin Knots node on the same host feeds it via -# `blocknotify=killall -USR1 datum_gateway` in bitcoin.conf (see -# roles/bitcoin_knots/README.md). Changing either config means thinking about -# both. Interrupting Stratum costs mining shares, so check for connected miners -# before restarting: -# ss -tn state established '( sport = :23334 )' -- name: Deploy DATUM Gateway on the bitcoin host +# Deploys DATUM Gateway (https://github.com/OCEAN-xyz/datum_gateway) on the +# Bitcoin Knots host so it has direct localhost RPC access to bitcoind. +# +# What this does: +# 1. Installs build deps and compiles datum_gateway from source +# 2. Creates a dedicated system user and config/log directories +# 3. Writes /etc/datum-gateway/config.json from vars/secrets +# 4. Patches bitcoin.conf with the required blockmaxsize/blocknotify lines +# 5. Creates and enables a systemd service +# 6. Creates a push-monitor health check script + systemd timer +# 7. Registers a push monitor in Uptime Kuma +# +# Separate play: adds a Caddy reverse proxy on vipy for the dashboard. +# +# Stratum port (23334) is bound on knots_box_local. Expose it to miners via +# a firewall rule, Tailscale, or a socket proxy on vipy — not handled here. +# +# Required secrets in infra_secrets.yml: +# datum_mining_address - Bitcoin address for block rewards +# datum_gateway_admin_password - Password for the /api admin endpoint +# bitcoin_rpc_user - Shared with the bitcoin-knots deployment +# bitcoin_rpc_password - Shared with the bitcoin-knots deployment + +- name: Deploy DATUM Gateway on knots_box_local hosts: bitcoin become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./datum_gateway_vars.yml vars: - # Preserves the push URL this check reports to. The role knows nothing about - # Uptime Kuma — this is just "a URL that accepts a ping". - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_datum-gateway/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - datum_gateway + datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" + datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" -- name: Configure Caddy reverse proxy for the DATUM Gateway dashboard on the edge host + tasks: + # =========================================== + # Build Dependencies + # =========================================== + - name: Install DATUM Gateway build dependencies + apt: + name: + - cmake + - build-essential + - git + - libjansson-dev + - libmicrohttpd-dev + - libsodium-dev + - libcurl4-openssl-dev + # Runtime-only (netcat for health check) + - netcat-openbsd + state: present + update_cache: yes + + # =========================================== + # System User and Directories + # =========================================== + - name: Create datum system user + user: + name: "{{ datum_gateway_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ datum_gateway_dir }}" + create_home: no + comment: "DATUM Gateway" + + - name: Create DATUM Gateway directories + file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ datum_gateway_group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + + # =========================================== + # Build from Source + # =========================================== + - name: Clone DATUM Gateway repository at {{ datum_gateway_version }} + git: + repo: https://github.com/OCEAN-xyz/datum_gateway.git + dest: "{{ datum_gateway_source_dir }}" + version: "{{ datum_gateway_version }}" + force: yes + register: git_clone + + - name: Configure cmake build + command: cmake . -DCMAKE_BUILD_TYPE=Release + args: + chdir: "{{ datum_gateway_source_dir }}" + + - name: Compile datum_gateway + command: make -j{{ datum_gateway_build_jobs }} + args: + chdir: "{{ datum_gateway_source_dir }}" + + - name: Install datum_gateway binary + copy: + src: "{{ datum_gateway_source_dir }}/datum_gateway" + dest: "{{ datum_gateway_bin_path }}" + remote_src: yes + owner: root + group: root + mode: "0755" + notify: Restart datum-gateway + + # =========================================== + # Configuration + # =========================================== + - name: Write DATUM Gateway config.json + copy: + dest: "{{ datum_gateway_config_dir }}/config.json" + content: | + { + "bitcoind": { + "rpcuser": "{{ bitcoin_rpc_user }}", + "rpcpassword": "{{ bitcoin_rpc_password }}", + "rpcurl": "{{ datum_bitcoin_rpc_url }}", + "notify_fallback": true + }, + "stratum": { + "listen_port": {{ datum_gateway_stratum_port }}, + "vardiff_min": {{ datum_vardiff_min }} + }, + "mining": { + "pool_address": "{{ datum_mining_address }}", + "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", + "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" + }, + "api": { + "admin_password": "{{ datum_gateway_admin_password }}", + "listen_port": {{ datum_gateway_api_port }}, + "modify_conf": false + }, + "logger": { + "log_to_console": true, + "log_to_file": true, + "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", + "log_rotate_daily": true, + "log_level_console": 2, + "log_level_file": 1 + }, + "datum": { + "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, + "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, + "pooled_mining_only": {{ datum_pooled_mining_only | lower }} + } + } + owner: "{{ datum_gateway_user }}" + group: "{{ datum_gateway_group }}" + mode: "0640" + notify: Restart datum-gateway + + # =========================================== + # Systemd Service + # =========================================== + - name: Create datum-gateway systemd service + copy: + dest: /etc/systemd/system/datum-gateway.service + content: | + [Unit] + Description=DATUM Gateway - Bitcoin Mining Gateway + Documentation=https://github.com/OCEAN-xyz/datum_gateway + After=network.target bitcoind.service + Wants=bitcoind.service + + [Service] + User={{ datum_gateway_user }} + Group={{ datum_gateway_group }} + Type=simple + ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json + Restart=on-failure + RestartSec=10 + StandardOutput=journal + StandardError=journal + + # Prevent config from being read by other users + ReadWritePaths={{ datum_gateway_log_dir }} + ReadOnlyPaths={{ datum_gateway_config_dir }} + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + notify: Restart datum-gateway + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start datum-gateway + systemd: + name: datum-gateway + enabled: yes + state: started + + # =========================================== + # Health Check Script + Systemd Timer + # =========================================== + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create DATUM Gateway health check script + when: uptime_kuma_enabled | default(false) + copy: + dest: /usr/local/bin/datum-gateway-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + STRATUM_PORT={{ datum_gateway_stratum_port }} + + check_datum() { + # Service must be active and stratum port must be listening + systemctl is-active --quiet datum-gateway && \ + nc -z 127.0.0.1 "${STRATUM_PORT}" + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_datum; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "DATUM Gateway not responding" + exit 1 + fi + owner: root + group: root + mode: "0755" + + - name: Create datum-gateway health check systemd service + copy: + dest: /etc/systemd/system/datum-gateway-healthcheck.service + content: | + [Unit] + Description=DATUM Gateway Health Check + After=network.target datum-gateway.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + + - name: Create datum-gateway health check systemd timer + copy: + dest: /etc/systemd/system/datum-gateway-healthcheck.timer + content: | + [Unit] + Description=DATUM Gateway Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: "0644" + + - name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + + - name: Enable and start datum-gateway health check timer + when: uptime_kuma_enabled | default(false) + systemd: + name: datum-gateway-healthcheck.timer + enabled: yes + state: started + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for DATUM Gateway + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_datum_gateway_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + + try: + import socketio.exceptions + except ImportError: + pass + + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_datum_gateway_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + push_url = None + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + else: + print(f"Creating push monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=90, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + # socketio timeout: add_monitor may have succeeded server-side + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + + api.disconnect() + + if push_url: + print(f"PUSH_URL={push_url}") + with open('/tmp/datum_gateway_push_url.txt', 'w') as f: + f.write(push_url) + + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for push monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_datum_gateway_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "DATUM Gateway" + mode: "0644" + + - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_datum_gateway_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URL from file + when: uptime_kuma_enabled | default(false) + slurp: + src: /tmp/datum_gateway_push_url.txt + delegate_to: localhost + become: no + register: push_url_file + ignore_errors: yes + + - name: Parse push URL + set_fact: + datum_push_url: "{{ push_url_file.content | b64decode | trim }}" + when: push_url_file.content is defined + + - name: Update health check service with push URL + lineinfile: + path: /etc/systemd/system/datum-gateway-healthcheck.service + regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" + line: "Environment=UPTIME_KUMA_PUSH_URL={{ datum_push_url }}" + when: datum_push_url is defined + notify: Restart datum-gateway health check timer + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_datum_gateway_monitor.py + - /tmp/ansible_datum_gateway_config.yml + - /tmp/datum_gateway_push_url.txt + + handlers: + - name: Restart datum-gateway + systemd: + name: datum-gateway + state: restarted + daemon_reload: yes + + - name: Restart datum-gateway health check timer + when: uptime_kuma_enabled | default(false) + systemd: + name: datum-gateway-healthcheck.timer + state: restarted + daemon_reload: yes + + +# =========================================== +# Caddy Reverse Proxy for DATUM Dashboard (on vipy) +# =========================================== +- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on the edge host hosts: edge become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./datum_gateway_vars.yml + vars: + datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" + datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + tasks: - name: Publish the DATUM Gateway dashboard through Caddy ansible.builtin.include_role: name: caddy_site vars: caddy_site_name: datum-gateway - caddy_site_domain: "{{ subdomains.datum_gateway }}.{{ root_domain }}" - caddy_site_upstream: "{{ hostvars['knots_box_local'].ansible_host }}:{{ hostvars['knots_box_local'].datum_gateway_api_port }}" + caddy_site_domain: "{{ datum_gateway_domain }}" + caddy_site_upstream: "knots-box:{{ datum_gateway_api_port }}" caddy_site_resolvers: "100.100.100.100" caddy_site_basic_auth: - user: "{{ datum_dashboard_username }}" hash: "{{ datum_dashboard_password_hash }}" - # The role validates its own site fragment; this re-validates the whole - # assembled Caddyfile, which is the only thing that catches a conflict - # between two sites. + # The role validates the site fragment on its own. This re-validates the + # whole assembled Caddyfile, which is the only thing that catches a + # conflict between this site and another. Kept from the hand-rolled + # version; the other nine services never had it. - name: Validate the assembled Caddyfile ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile changed_when: false + - name: Display DATUM Gateway dashboard URL + when: uptime_kuma_enabled | default(false) + debug: + msg: "DATUM Gateway dashboard: https://{{ datum_gateway_domain }}" + + # =========================================== + # Uptime Kuma HTTP Monitor for Public Dashboard + # =========================================== + - name: Create Uptime Kuma HTTP monitor setup script for DATUM dashboard + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_datum_http_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_datum_http_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + else: + print(f"Creating HTTP monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for HTTP monitor + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_datum_http_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ datum_gateway_domain }}" + monitor_name: "DATUM Gateway Dashboard" + mode: "0644" + + - name: Run Uptime Kuma HTTP monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_datum_http_monitor.py + delegate_to: localhost + become: no + register: http_monitor_setup + changed_when: "'SUCCESS' in http_monitor_setup.stdout" + ignore_errors: yes + + - name: Display HTTP monitor setup output + debug: + msg: "{{ http_monitor_setup.stdout_lines }}" + when: http_monitor_setup.stdout is defined + + - name: Clean up HTTP monitor temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_datum_http_monitor.py + - /tmp/ansible_datum_http_config.yml + + +# =========================================== +# Stratum Port Forwarding on vipy via systemd-socket-proxyd +# Miners connect to vipy:23334; traffic is forwarded to knots-box:23334 +# over the Tailscale network, matching the Bitcoin P2P proxy pattern. +# =========================================== - name: Setup public Stratum port forwarding on the edge host hosts: edge become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./datum_gateway_vars.yml + vars: + datum_tailscale_hostname: "knots-box" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + tasks: - name: Expose the DATUM Stratum port through a socket proxy ansible.builtin.include_role: @@ -51,8 +669,134 @@ vars: socket_proxy_name: datum-stratum socket_proxy_description: "DATUM Stratum" - socket_proxy_listen_port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}" - socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" - # Matches the UFW comment already on the edge host; the derived default - # would say "DATUM Stratum" and rewrite the rule. + socket_proxy_listen_port: "{{ datum_gateway_stratum_port }}" + socket_proxy_upstream_host: "{{ datum_tailscale_hostname }}" + # Matches the UFW comment already on vipy; the derived default would + # have said "DATUM Stratum" and rewritten the rule. socket_proxy_ufw_comment: "DATUM Gateway Stratum public access" + + - name: Display public Stratum endpoint + when: uptime_kuma_enabled | default(false) + debug: + msg: "DATUM Stratum public endpoint: {{ ansible_host }}:{{ datum_gateway_stratum_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public Stratum + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Stratum + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_datum_stratum_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_datum_stratum_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for Stratum TCP monitor + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_datum_stratum_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ datum_gateway_stratum_port }} + monitor_name: "DATUM Stratum (public)" + mode: "0644" + + - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_datum_stratum_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up Stratum TCP monitor temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_datum_stratum_tcp_monitor.py + - /tmp/ansible_datum_stratum_config.yml + diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index 04081ff..bdc8428 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -1,13 +1,409 @@ ---- - name: Install Forgejo Runner on Debian 13 hosts: ci_runner become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./forgejo_runner_vars.yml vars: - # Preserves the push URL this host has been reporting to all along, so the - # move to a role changes no behaviour. The role itself knows nothing about - # Uptime Kuma — this is just "a URL that accepts a ping", and whatever - # replaces it sets the same variable. - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_forgejo-runner/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - forgejo_runner + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + healthcheck_interval_seconds: 60 + healthcheck_timeout_seconds: 90 + healthcheck_retries: 1 + healthcheck_script_dir: /opt/forgejo-runner-healthcheck + healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" + healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" + healthcheck_service_name: forgejo-runner-healthcheck + + tasks: + # ── 1. Assert Docker is available ────────────────────────────────── + - name: Check if Docker is installed + command: docker --version + register: docker_check + changed_when: false + failed_when: docker_check.rc != 0 + + - name: Fail if Docker is not available + assert: + that: + - docker_check.rc == 0 + fail_msg: > + Docker is not installed or not in PATH. + Please install Docker before running this playbook. + + # ── 2. Download forgejo-runner binary ────────────────────────────── + - name: Download forgejo-runner binary + get_url: + url: "{{ forgejo_runner_url }}" + dest: "{{ forgejo_runner_bin_path }}" + mode: '0755' + + # ── 3. Create runner system user ─────────────────────────────────── + - name: Create runner system user + user: + name: "{{ forgejo_runner_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ forgejo_runner_dir }}" + create_home: no + groups: docker + append: yes + comment: 'Forgejo Runner' + + # ── 4. Create working directory ──────────────────────────────────── + - name: Create forgejo-runner working directory + file: + path: "{{ forgejo_runner_dir }}" + state: directory + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + mode: '0750' + + # ── 5. Generate default config ───────────────────────────────────── + - name: Check if config already exists + stat: + path: "{{ forgejo_runner_config_path }}" + register: config_stat + + - name: Generate default config + shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not config_stat.stat.exists + + - name: Set config file ownership + file: + path: "{{ forgejo_runner_config_path }}" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not config_stat.stat.exists + + # ── 6. Register runner ───────────────────────────────────────────── + - name: Check if runner is already registered + stat: + path: "{{ forgejo_runner_dir }}/.runner" + register: runner_stat + + - name: Register runner with Forgejo instance + command: > + {{ forgejo_runner_bin_path }} register --no-interactive + --instance {{ forgejo_instance_url }} + --token {{ forgejo_runner_registration_token }} + --name forgejo-runner-box + --labels "{{ forgejo_runner_labels }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not runner_stat.stat.exists + + - name: Set runner registration file ownership + file: + path: "{{ forgejo_runner_dir }}/.runner" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not runner_stat.stat.exists + + # ── 7. Create systemd service ────────────────────────────────────── + - name: Create forgejo-runner systemd service + copy: + dest: /etc/systemd/system/forgejo-runner.service + content: | + [Unit] + Description=Forgejo Runner + Documentation=https://forgejo.org/docs/latest/admin/actions/ + After=docker.service + Requires=docker.service + + [Service] + Type=simple + User={{ forgejo_runner_user }} + Group={{ forgejo_runner_user }} + WorkingDirectory={{ forgejo_runner_dir }} + ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} + Restart=on-failure + RestartSec=10 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + # ── 8. Reload systemd, enable and start ──────────────────────────── + - name: Reload systemd + systemd: + daemon_reload: yes + + - name: Enable and start forgejo-runner service + systemd: + name: forgejo-runner + enabled: yes + state: started + + # ── 9. Verify runner is active ───────────────────────────────────── + - name: Verify forgejo-runner is active + command: systemctl is-active forgejo-runner + register: runner_active + changed_when: false + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Assert runner is running + when: uptime_kuma_enabled | default(false) + assert: + that: + - runner_active.stdout == "active" + fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" + + # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── + - name: Create Uptime Kuma push monitor setup script + when: uptime_kuma_enabled | default(false) + copy: + dest: /tmp/setup_forgejo_runner_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) + command: > + {{ ansible_playbook_python }} + /tmp/setup_forgejo_runner_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "services" + "forgejo-runner-healthcheck" + "Forgejo Runner healthcheck - ping every {{ healthcheck_interval_seconds }}s" + "{{ healthcheck_timeout_seconds }}" + "{{ healthcheck_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + when: uptime_kuma_enabled | default(false) + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL + when: uptime_kuma_enabled | default(false) + set_fact: + uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Create healthcheck script directory + file: + path: "{{ healthcheck_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create forgejo-runner healthcheck script + when: uptime_kuma_enabled | default(false) + copy: + dest: "{{ healthcheck_script_path }}" + content: | + #!/bin/bash + + # Forgejo Runner Healthcheck Script + # Checks if forgejo-runner is active and pings Uptime Kuma on success + + LOG_FILE="{{ healthcheck_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_push_url }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + main() { + if systemctl is-active --quiet forgejo-runner; then + log_message "forgejo-runner is active, sending ping" + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=forgejo-runner%20is%20active" 2>&1) + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Ping sent successfully (HTTP $http_code)" + else + log_message "ERROR: Failed to send ping (HTTP $http_code)" + exit 1 + fi + else + log_message "ERROR: forgejo-runner is not active" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create healthcheck systemd service + copy: + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" + content: | + [Unit] + Description=Forgejo Runner Healthcheck + After=network.target + + [Service] + Type=oneshot + ExecStart={{ healthcheck_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create healthcheck systemd timer + copy: + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" + content: | + [Unit] + Description=Run Forgejo Runner Healthcheck every minute + Requires={{ healthcheck_service_name }}.service + + [Timer] + OnBootSec=30sec + OnUnitActiveSec={{ healthcheck_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd for healthcheck units + systemd: + daemon_reload: yes + + - name: Enable and start healthcheck timer + systemd: + name: "{{ healthcheck_service_name }}.timer" + enabled: yes + state: started + + - name: Test healthcheck script + command: "{{ healthcheck_script_path }}" + register: healthcheck_test + changed_when: false + + - name: Verify healthcheck script works + assert: + that: + - healthcheck_test.rc == 0 + fail_msg: "Healthcheck script failed to execute properly" + + - name: Display deployment summary + debug: + msg: | + Forgejo Runner deployed successfully! + + Runner Name: forgejo-runner-box + Instance: {{ forgejo_instance_url }} + Working Directory: {{ forgejo_runner_dir }} + Service: forgejo-runner.service ({{ runner_active.stdout }}) + + Healthcheck Monitor: forgejo-runner-healthcheck + Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s + Timeout: {{ healthcheck_timeout_seconds }}s + + - name: Clean up temporary monitor setup script + when: uptime_kuma_enabled | default(false) + file: + path: /tmp/setup_forgejo_runner_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/services/forgejo-runner/forgejo_runner_vars.yml b/ansible/services/forgejo-runner/forgejo_runner_vars.yml new file mode 100644 index 0000000..e618fca --- /dev/null +++ b/ansible/services/forgejo-runner/forgejo_runner_vars.yml @@ -0,0 +1,9 @@ +forgejo_runner_version: "6.3.1" +forgejo_runner_arch: "linux-amd64" +forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" +forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" +forgejo_runner_user: "runner" +forgejo_runner_dir: "/opt/forgejo-runner" +forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" +forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" +forgejo_instance_url: "https://forgejo.contrapeso.xyz" diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index f908586..db78e95 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -2,10 +2,14 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./forgejo_vars.yml vars: forgejo_subdomain: "{{ subdomains.forgejo }}" forgejo_domain: "{{ forgejo_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure required packages are installed @@ -90,3 +94,123 @@ caddy_site_name: forgejo caddy_site_domain: "{{ forgejo_domain }}" caddy_site_upstream: "localhost:{{ forgejo_port }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Uptime Kuma monitor setup script for Forgejo + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_forgejo_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ forgejo_domain }}/api/healthz" + monitor_name: "Forgejo" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_forgejo_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_forgejo_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/forgejo/forgejo_vars.yml b/ansible/services/forgejo/forgejo_vars.yml index 7bba0ed..2a24133 100644 --- a/ansible/services/forgejo/forgejo_vars.yml +++ b/ansible/services/forgejo/forgejo_vars.yml @@ -9,7 +9,7 @@ forgejo_url: "https://codeberg.org/forgejo/forgejo/releases/download/v{{ forgejo forgejo_bin_path: "/usr/local/bin/forgejo" forgejo_user: "git" -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # Remote access remote_host_name: "{{ groups['edge'] | first }}" @@ -18,3 +18,6 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/forgejo-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/forgejo_backup.sh" diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml index 5329e2a..d6aafef 100644 --- a/ansible/services/forgejo/setup_backup_forgejo.yml +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -9,6 +9,7 @@ hosts: edge become: yes vars_files: + - ../../group_vars/all/main.yml - ./forgejo_vars.yml tasks: @@ -22,7 +23,3 @@ backup_source_stop_service: forgejo backup_source_retention_days: 2 backup_source_on_calendar: "*-*-* 02:30:00" - # Reported to Gatus as backup-dump_forgejo. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_forgejo/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml new file mode 100644 index 0000000..c3ff8d6 --- /dev/null +++ b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml @@ -0,0 +1,122 @@ +--- +- name: Configure local backup for Forgejo from remote + hosts: control + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./forgejo_vars.yml + vars: + remote_data_path: "{{ forgejo_data_dir }}" + remote_config_path: "{{ forgejo_config_dir }}" + forgejo_service_name: "forgejo" + gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" + gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" + + tasks: + - name: Debug Forgejo backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "remote_config_path='{{ remote_config_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + - "gpg_recipient={{ gpg_recipient }}" + - "gpg_key_id={{ gpg_key_id }}" + + - name: Ensure local backup directory exists + ansible.builtin.file: + path: "{{ local_backup_dir }}" + state: directory + mode: "0755" + + - name: Ensure ~/.local/bin exists + ansible.builtin.file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: "0755" + + - name: Create Forgejo backup script + ansible.builtin.copy: + dest: "{{ backup_script_path }}" + mode: "0750" + content: | + #!/bin/bash + set -euo pipefail + + if [ -z "{{ gpg_recipient }}" ]; then + echo "GPG recipient is not configured. Aborting." + exit 1 + fi + + TIMESTAMP=$(date +'%Y-%m-%d') + ENCRYPTED_BACKUP="{{ local_backup_dir }}/forgejo-backup-$TIMESTAMP.tar.gz.gpg" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + echo "Stopping Forgejo service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop {{ forgejo_service_name }}" + + echo "Creating encrypted backup archive..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo tar -czf - {{ remote_data_path }} {{ remote_config_path }}" | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" + + echo "Starting Forgejo service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start {{ forgejo_service_name }}" + + # Rotate old backups (keep 3 days) + # Calculate cutoff date (3 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '3 days ago' +'%Y-%m-%d') + for backup_file in "{{ local_backup_dir }}"/forgejo-backup-*.tar.gz.gpg; do + if [ -f "$backup_file" ]; then + # Extract date from filename: forgejo-backup-YYYY-MM-DD.tar.gz.gpg + file_date=$(basename "$backup_file" | sed -n 's/forgejo-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$backup_file" + fi + fi + done + + echo "Backup completed successfully" + + - name: Ensure cronjob for Forgejo backup exists + ansible.builtin.cron: + name: "Forgejo backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run Forgejo backup script to create initial backup + ansible.builtin.command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check if backup file exists + stat: + path: "{{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg" + register: backup_file_stat + + - name: Verify backup file exists + assert: + that: + - backup_file_stat.stat.exists + - backup_file_stat.stat.isreg + fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" + success_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists" + + - name: Verify backup file is not empty + assert: + that: + - backup_file_stat.stat.size > 0 + fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" + success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 1b7b963..01e9f17 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -1,21 +1,556 @@ ---- -# Fulcrum: Electrum server indexing the Bitcoin Knots node. -# The index takes days to rebuild, so nothing here touches its data directory -# beyond asserting that it exists. - name: Deploy Fulcrum Electrum Server hosts: electrum become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./fulcrum_vars.yml vars: - # Preserves the push URL this check has been configured with. The role knows - # nothing about Uptime Kuma — this is just "a URL that accepts a ping". - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_fulcrum/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - fulcrum + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Calculate 75% of system RAM for db_mem + set_fact: + fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" + changed_when: false + + - name: Display calculated db_mem value + debug: + msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" + + - name: Display Fulcrum version to install + debug: + msg: "Installing Fulcrum version {{ fulcrum_version }}" + + - name: Install required packages + apt: + name: + - curl + - wget + - openssl + state: present + update_cache: yes + + - name: Create fulcrum group + group: + name: "{{ fulcrum_group }}" + system: yes + state: present + + - name: Create fulcrum user + user: + name: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + system: yes + shell: /usr/sbin/nologin + home: /home/{{ fulcrum_user }} + create_home: yes + state: present + + - name: Create Fulcrum database directory (heavy data on special mount) + file: + path: "{{ fulcrum_db_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Create Fulcrum config directory + file: + path: "{{ fulcrum_config_dir }}" + state: directory + owner: root + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Create Fulcrum lib directory (for banner and other data files) + file: + path: "{{ fulcrum_lib_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + + # =========================================== + # SSL Certificate Generation + # =========================================== + - name: Check if SSL certificate already exists + stat: + path: "{{ fulcrum_ssl_cert_path }}" + register: fulcrum_ssl_cert_exists + when: fulcrum_ssl_enabled | default(false) + + - name: Generate self-signed SSL certificate for Fulcrum + command: > + openssl req -x509 -newkey rsa:4096 + -keyout {{ fulcrum_ssl_key_path }} + -out {{ fulcrum_ssl_cert_path }} + -sha256 -days {{ fulcrum_ssl_cert_days }} + -nodes + -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" + args: + creates: "{{ fulcrum_ssl_cert_path }}" + when: fulcrum_ssl_enabled | default(false) + notify: Restart fulcrum + + - name: Set SSL certificate permissions + file: + path: "{{ fulcrum_ssl_cert_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) + + - name: Set SSL key permissions + file: + path: "{{ fulcrum_ssl_key_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0600' + when: fulcrum_ssl_enabled | default(false) + + - name: Check if Fulcrum binary already exists + stat: + path: "{{ fulcrum_binary_path }}" + register: fulcrum_binary_exists + changed_when: false + + - name: Download Fulcrum binary tarball + get_url: + url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + mode: '0644' + when: not fulcrum_binary_exists.stat.exists + + - name: Extract Fulcrum binary + unarchive: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp" + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + + - name: Install Fulcrum binary + copy: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" + dest: "{{ fulcrum_binary_path }}" + owner: root + group: root + mode: '0755' + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + + - name: Verify Fulcrum binary installation + command: "{{ fulcrum_binary_path }} --version" + register: fulcrum_version_check + changed_when: false + + - name: Display Fulcrum version + debug: + msg: "{{ fulcrum_version_check.stdout_lines }}" + + - name: Create Fulcrum banner file + copy: + dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" + content: | + counterinfra + + PER ASPERA AD ASTRA + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + + - name: Create Fulcrum configuration file + copy: + dest: "{{ fulcrum_config_dir }}/fulcrum.conf" + content: | + # Fulcrum Configuration + # Generated by Ansible + + # Bitcoin Core/Knots RPC settings + bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} + rpcuser = {{ bitcoin_rpc_user }} + rpcpassword = {{ bitcoin_rpc_password }} + + # Fulcrum server general settings + datadir = {{ fulcrum_db_dir }} + tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} + peering = {{ 'true' if fulcrum_peering else 'false' }} + zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} + + # SSL/TLS Configuration + {% if fulcrum_ssl_enabled | default(false) %} + ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} + cert = {{ fulcrum_ssl_cert_path }} + key = {{ fulcrum_ssl_key_path }} + {% endif %} + + # Anonymize client IP addresses and TxIDs in logs + anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} + + # Max RocksDB Memory in MiB + db_mem = {{ fulcrum_db_mem_mb }}.0 + + # Banner + banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0640' + notify: Restart fulcrum + + - name: Create systemd service file for Fulcrum + copy: + dest: /etc/systemd/system/fulcrum.service + content: | + # MiniBolt: systemd unit for Fulcrum + # /etc/systemd/system/fulcrum.service + + [Unit] + Description=Fulcrum + After=network.target + + StartLimitBurst=2 + StartLimitIntervalSec=20 + + [Service] + ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf + + User={{ fulcrum_user }} + Group={{ fulcrum_group }} + + # Process management + #################### + Type=simple + KillSignal=SIGINT + TimeoutStopSec=300 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + notify: Restart fulcrum + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum service + systemd: + name: fulcrum + enabled: yes + state: started + + - name: Wait for Fulcrum to start + wait_for: + port: "{{ fulcrum_tcp_port }}" + host: "{{ fulcrum_tcp_bind }}" + delay: 5 + timeout: 30 + ignore_errors: yes + + - name: Check Fulcrum service status + systemd: + name: fulcrum + register: fulcrum_service_status + changed_when: false + + - name: Display Fulcrum service status + debug: + msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Fulcrum health check and push script + when: uptime_kuma_enabled | default(false) + copy: + dest: /usr/local/bin/fulcrum-healthcheck-push.sh + content: | + #!/bin/bash + # + # Fulcrum Health Check and Push to Uptime Kuma + # Checks if Fulcrum TCP port is responding and pushes status to Uptime Kuma + # + + FULCRUM_HOST="{{ fulcrum_tcp_bind }}" + FULCRUM_PORT={{ fulcrum_tcp_port }} + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + # Check if Fulcrum TCP port is responding + check_fulcrum() { + # Try to connect to TCP port + timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null + return $? + } + + # Push status to Uptime Kuma + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + + # URL encode spaces in message + local encoded_msg="${msg// /%20}" + + if ! curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then + echo "ERROR: Failed to push to Uptime Kuma" + return 1 + fi + } + + # Main health check + if check_fulcrum; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Fulcrum TCP port not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create systemd timer for Fulcrum health check + copy: + dest: /etc/systemd/system/fulcrum-healthcheck.timer + content: | + [Unit] + Description=Fulcrum Health Check Timer + Requires=fulcrum.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for Fulcrum health check + when: uptime_kuma_enabled | default(false) + copy: + dest: /etc/systemd/system/fulcrum-healthcheck.service + content: | + [Unit] + Description=Fulcrum Health Check and Push to Uptime Kuma + After=network.target fulcrum.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum health check timer + systemd: + name: fulcrum-healthcheck.timer + enabled: yes + state: started + + - name: Create Uptime Kuma push monitor setup script for Fulcrum + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_fulcrum_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for new monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "Fulcrum" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_fulcrum_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Extract push URL from monitor setup output + set_fact: + uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" + delegate_to: localhost + become: no + when: monitor_setup.stdout is defined + + - name: Display extracted push URL + debug: + msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" + when: uptime_kuma_push_url | default('') != '' + + - name: Set push URL in systemd service environment + lineinfile: + path: /etc/systemd/system/fulcrum-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" + state: present + insertafter: '^\[Service\]' + when: uptime_kuma_push_url | default('') != '' + + - name: Reload systemd daemon after push URL update + systemd: + daemon_reload: yes + when: uptime_kuma_push_url | default('') != '' + + - name: Restart health check timer to pick up new environment + systemd: + name: fulcrum-healthcheck.timer + state: restarted + when: uptime_kuma_push_url | default('') != '' + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_fulcrum_monitor.py + - /tmp/ansible_config.yml + - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz + - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux + + handlers: + - name: Restart fulcrum + when: uptime_kuma_enabled | default(false) + systemd: + name: fulcrum + state: restarted + - name: Setup public Fulcrum SSL forwarding on the edge host hosts: edge become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./fulcrum_vars.yml + vars: + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + tasks: - name: Expose Fulcrum SSL through a socket proxy ansible.builtin.include_role: @@ -23,5 +558,128 @@ vars: socket_proxy_name: fulcrum-ssl socket_proxy_description: "Fulcrum SSL" - socket_proxy_listen_port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}" - socket_proxy_upstream_host: "{{ hostvars['fulcrum_box_local'].ansible_host }}" + socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" + + - name: Display public endpoint + when: uptime_kuma_enabled | default(false) + debug: + msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public SSL Port + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_fulcrum_ssl_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_fulcrum_ssl_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for TCP monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_fulcrum_ssl_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ fulcrum_ssl_port }} + monitor_name: "Fulcrum SSL Public" + mode: '0644' + + - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up TCP monitor temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_fulcrum_ssl_tcp_monitor.py + - /tmp/ansible_fulcrum_ssl_config.yml + + diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml new file mode 100644 index 0000000..8486b14 --- /dev/null +++ b/ansible/services/fulcrum/fulcrum_vars.yml @@ -0,0 +1,51 @@ +# Fulcrum Configuration Variables + +# Version - Pinned to specific release +fulcrum_version: "2.1.0" # Fulcrum version to install + +# Directories +fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) +fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) +fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk +fulcrum_binary_path: /usr/local/bin/Fulcrum + +# Network - Bitcoin RPC connection +# Bitcoin Knots is on a different host (knots_box_local) +# Using RPC user/password authentication (credentials from infra_secrets.yml) +bitcoin_rpc_host: "192.168.1.140" # Bitcoin Knots RPC host (IP of knots_box_local) +bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml + +# Network - Fulcrum server +fulcrum_tcp_port: 50001 +fulcrum_ssl_port: 50002 +# Binding address for Fulcrum TCP/SSL server: +# - "127.0.0.1" = localhost only (use when Caddy is on the same box) +# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) +# - Specific IP = bind to specific network interface +fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) +fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port +# If Caddy is on a different box, set this to the IP address that Caddy will use to connect + +# SSL/TLS Configuration +fulcrum_ssl_enabled: true +fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" +fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" +fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert + +# Port forwarding configuration (for public access via VPS) +fulcrum_tailscale_hostname: "fulcrum-box" + +# Performance +# db_mem will be calculated as 75% of available RAM automatically in playbook +fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache + +# Configuration options +fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs +fulcrum_peering: false # Disable peering with other Fulcrum servers +fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications + +# Service user +fulcrum_user: fulcrum +fulcrum_group: fulcrum + diff --git a/ansible/services/gatus/deploy_gatus_playbook.yml b/ansible/services/gatus/deploy_gatus_playbook.yml deleted file mode 100644 index d6a5ac1..0000000 --- a/ansible/services/gatus/deploy_gatus_playbook.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -# Gatus: health checks, status page and alerting for the whole estate. -# -# Built from source and run under systemd - upstream publishes no binaries, and -# their Dockerfile shows the runtime needs nothing but the static binary and a -# CA bundle. See roles/gatus/README.md. -- name: Deploy Gatus on the observability host - hosts: observability - become: yes - vars: - gatus_alerting: - signal: - # NOTE: the key is `api-url`, not `url` as upstream's own README table - # says - see alerting/provider/signal/signal.go. Gatus appends /v2/send - # itself if the suffix is missing. - # - # Reached by service name over the shared docker network. Gatus runs in - # a container, so 127.0.0.1 here would be the Gatus container, not the - # host - and the Signal API deliberately publishes no ports because it - # has no authentication. - api-url: "http://signal-api:8080" - number: "{{ signal_number }}" - recipients: "{{ signal_recipients }}" - default-alert: - # Overridden per group by the registration playbooks; these are the - # values that apply if a caller sets nothing. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - # An ongoing outage should not become an ongoing phone buzz. - minimum-reminder-interval: 6h - - roles: - - gatus - -# The dashboard is bound to loopback; Caddy publishes it. -# -# Auth is done HERE, at the edge, and not with Gatus's own `security.basic`. -# Gatus's security middleware protects exactly four routes (api/api.go): -# -# /api/v1/endpoints/statuses -# /api/v1/endpoints/:key/statuses -# /api/v1/suites/statuses -# /api/v1/suites/:key/statuses -# -# Everything else is registered on the UNPROTECTED router, including -# /api/v1/config, every badge, and - the part that matters - -# /api/v1/endpoints/:key/uptimes/:duration and .../response-times/:duration/history, -# which return real per-endpoint data to anyone who can guess a key. Keys are -# just "_". So Gatus's own auth makes the dashboard render empty -# while leaving the data readable, which is worse than it looks. -# -# The one route that must NOT sit behind basic auth is the external-endpoint -# push API. It authenticates with `Authorization: Bearer `, and basic -# auth wants `Authorization: Basic <...>` - same header, two schemes, and the -# push clients lose. It is not actually unauthenticated: the handler 401s on a -# missing prefix, an empty token, or a token that does not match that endpoint's -# own. Upstream's comment on the route says exactly that. -- name: Publish the Gatus status page through Caddy - hosts: observability - become: yes - tasks: - - name: Require the dashboard credentials to be set - ansible.builtin.assert: - that: - - gatus_dashboard_username is defined - - gatus_dashboard_username | length > 0 - - gatus_dashboard_password_hash is defined - - gatus_dashboard_password_hash.startswith('$2') - fail_msg: >- - gatus_dashboard_username and gatus_dashboard_password_hash must be in - the vault. Generate the hash on the observability host, which runs - Caddy natively, so the bcrypt cost and format match what verifies it: - caddy hash-password --plaintext 'your-password' - then: ansible-vault edit group_vars/all/vault.yml - - - name: Configure the Caddy vhost for Gatus - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: gatus - caddy_site_domain: "{{ subdomains.gatus }}.{{ root_domain }}" - # caddy_site_body rather than caddy_site_upstream + caddy_site_basic_auth, - # because that pair applies auth to the whole site with no way to carve - # out the push path. `handle` blocks are mutually exclusive and first - # match wins, so the push API gets a route of its own. - caddy_site_body: | - @push { - path /api/v1/endpoints/*/external - method POST - } - - # Push API: Bearer-authenticated by Gatus itself. No basic auth here, - # or the Authorization header collides. - handle @push { - reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} - } - - # Everything else: the dashboard, the config endpoint, the badges and - # the uptime/response-time history. - handle { - basic_auth { - {{ gatus_dashboard_username }} {{ gatus_dashboard_password_hash }} - } - reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} - } diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index e181f24..527e4c2 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -2,11 +2,16 @@ hosts: vpn_control become: no vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./headscale_vars.yml vars: headscale_subdomain: "{{ subdomains.headscale }}" headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" headscale_base_domain: "tailnet.{{ root_domain }}" + headscale_namespace: "{{ service_settings.headscale.namespace }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Install required packages @@ -253,6 +258,125 @@ # All API operations require a valid Bearer token in the Authorization header reverse_proxy * http://localhost:{{ headscale_port }} + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Uptime Kuma monitor setup script for Headscale + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_headscale_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ headscale_domain }}/health" + monitor_name: "Headscale" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_headscale_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_headscale_monitor.py + - /tmp/ansible_config.yml + handlers: - name: Restart headscale become: yes diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index c3fb948..05fc040 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -1,5 +1,5 @@ # Headscale service configuration -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (subdomain and caddy_sites_dir now in services_config.yml) headscale_port: 8080 headscale_grpc_port: 50443 @@ -10,7 +10,7 @@ headscale_version: "0.26.1" # Data directory headscale_data_dir: /var/lib/headscale -# Namespace is headscale_namespace in group_vars/all/main.yml +# Namespace now configured in services_config.yml under service_settings.headscale.namespace # Remote access remote_host_name: "{{ groups['vpn_control'] | first }}" @@ -19,5 +19,9 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/headscale-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/headscale_backup.sh" + # Headplane (headscale admin UI), proxied at /admin* behind Caddy basic auth headplane_port: 3000 diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml index 15ba5b7..9c72d1d 100644 --- a/ansible/services/headscale/setup_backup_headscale.yml +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -3,6 +3,7 @@ hosts: vpn_control become: yes vars_files: + - ../../group_vars/all/main.yml - ./headscale_vars.yml tasks: @@ -18,7 +19,3 @@ backup_source_stop_service: headscale backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" - # Reported to Gatus as backup-dump_headscale. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_headscale/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml new file mode 100644 index 0000000..acc82ce --- /dev/null +++ b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml @@ -0,0 +1,75 @@ +- name: Configure local backup for Headscale from remote + hosts: control + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./headscale_vars.yml + vars: + remote_data_path: "{{ headscale_data_dir }}" + remote_config_path: "/etc/headscale" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "remote_config_path='{{ remote_config_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + # Stop headscale service for consistent backup + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop headscale" + + # Backup data directory + rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/data/" + + # Backup config directory + rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_config_path }}/ "$BACKUP_DIR/config/" + + # Start headscale service again + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start headscale" + + # Rotate old backups (keep 14 days) + find "{{ local_backup_dir }}" -maxdepth 1 -type d -name '20*' -mtime +13 -exec rm -rf {} \; + + - name: Ensure cronjob for backup exists + cron: + name: "Headscale backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index 7b89f33..65bdfbf 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -2,10 +2,14 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./lnbits_vars.yml vars: lnbits_subdomain: "{{ subdomains.lnbits }}" lnbits_domain: "{{ lnbits_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create lnbits directory @@ -151,3 +155,123 @@ caddy_site_upstream: "localhost:{{ lnbits_port }}" caddy_site_headers_up: X-Forwarded-Host: "{{ lnbits_domain }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Uptime Kuma monitor setup script for LNBits + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_lnbits_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ lnbits_domain }}/api/v1/health" + monitor_name: "LNBits" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_lnbits_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_lnbits_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/lnbits/lnbits_vars.yml b/ansible/services/lnbits/lnbits_vars.yml index 466ce78..eabd6fc 100644 --- a/ansible/services/lnbits/lnbits_vars.yml +++ b/ansible/services/lnbits/lnbits_vars.yml @@ -3,7 +3,7 @@ lnbits_dir: /opt/lnbits lnbits_data_dir: "{{ lnbits_dir }}/data" lnbits_port: 8765 -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # Remote access remote_host_name: "{{ groups['edge'] | first }}" @@ -12,3 +12,6 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/lnbits-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/lnbits_backup.sh" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml index 2da0ac0..46abf31 100644 --- a/ansible/services/lnbits/setup_backup_lnbits.yml +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -9,6 +9,7 @@ hosts: edge become: yes vars_files: + - ../../group_vars/all/main.yml - ./lnbits_vars.yml tasks: @@ -22,7 +23,3 @@ backup_source_stop_service: lnbits backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:20:00" - # Reported to Gatus as backup-dump_lnbits. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_lnbits/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml new file mode 100644 index 0000000..48296cb --- /dev/null +++ b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml @@ -0,0 +1,104 @@ +- name: Configure local backup for LNBits from remote + hosts: control + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./lnbits_vars.yml + vars: + remote_data_path: "{{ lnbits_data_dir }}" + remote_lnbits_dir: "{{ lnbits_dir }}/lnbits" + gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" + gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + - "gpg_recipient={{ gpg_recipient }}" + - "gpg_key_id={{ gpg_key_id }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + ENCRYPTED_BACKUP="{{ local_backup_dir }}/lnbits-backup-$TIMESTAMP.tar.gz.gpg" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + # Stop LNBits service before backup + echo "Stopping LNBits service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop lnbits.service" + + # Create encrypted backup on the fly + # First, create a tar archive of the data directory and pipe it through gpg + echo "Creating backup..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "cd {{ remote_data_path }} && tar -czf - ." | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" + + # Also backup the .env file separately (smaller, might need quick access) + $SSH_CMD {{ remote_user }}@{{ remote_host }} "cat {{ remote_lnbits_dir }}/.env" | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "{{ local_backup_dir }}/lnbits-env-$TIMESTAMP.gpg" + + # Start LNBits service after backup + echo "Starting LNBits service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start lnbits.service" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for backup_file in "{{ local_backup_dir }}"/lnbits-backup-*.tar.gz.gpg; do + if [ -f "$backup_file" ]; then + # Extract date from filename: lnbits-backup-YYYY-MM-DD.tar.gz.gpg + file_date=$(basename "$backup_file" | sed -n 's/lnbits-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$backup_file" + fi + fi + done + for env_file in "{{ local_backup_dir }}"/lnbits-env-*.gpg; do + if [ -f "$env_file" ]; then + # Extract date from filename: lnbits-env-YYYY-MM-DD.gpg + file_date=$(basename "$env_file" | sed -n 's/lnbits-env-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$env_file" + fi + fi + done + + echo "Backup completed successfully" + + - name: Ensure cronjob for backup exists + cron: + name: "LNBits backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index 8b21b85..04e99cf 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -2,6 +2,9 @@ hosts: memos become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" @@ -62,6 +65,14 @@ owner: root group: root + - name: Clean up temporary files + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/memos.tar.gz + - /tmp/memos + - name: Create memos environment file copy: dest: "{{ memos_config_dir }}/memos.env" @@ -128,7 +139,18 @@ msg: "Memos is running on port {{ memos_port }}. Access via Tailscale at http://{{ memos_tailscale_hostname }}:{{ memos_port }}" handlers: + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Restart memos + when: uptime_kuma_enabled | default(false) systemd: name: memos state: restarted @@ -138,10 +160,14 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Publish Memos through Caddy (via Tailscale) @@ -152,3 +178,117 @@ caddy_site_domain: "{{ memos_domain }}" caddy_site_upstream: "{{ memos_tailscale_hostname }}:{{ memos_port }}" caddy_site_resolvers: "100.100.100.100" + + - name: Create Uptime Kuma monitor setup script for Memos + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_memos_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_memos_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_memos_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ memos_domain }}/healthz" + monitor_name: "Memos" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_memos_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_memos_monitor.py + - /tmp/ansible_memos_config.yml diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index 94c6de7..99618db 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -12,7 +12,15 @@ memos_url: "https://github.com/usememos/memos/releases/download/v{{ memos_versio memos_tailscale_hostname: "memos-box" memos_tailscale_ip: "100.64.0.4" -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain in services_config.yml) # Remote access (for backup from lapy via Tailscale) +backup_host: "{{ memos_tailscale_hostname }}" +backup_user: "counterweight" +backup_key_file: "~/.ssh/counterganzua" +backup_port: 22 + +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/memos_backup.sh" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml index 53ab85d..2131924 100644 --- a/ansible/services/memos/setup_backup_memos.yml +++ b/ansible/services/memos/setup_backup_memos.yml @@ -7,6 +7,7 @@ hosts: memos become: yes vars_files: + - ../../group_vars/all/main.yml - ./memos_vars.yml tasks: @@ -23,7 +24,3 @@ backup_source_stop_service: memos backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:00:00" - # Reported to Gatus as backup-dump_memos. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_memos/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml new file mode 100644 index 0000000..c32c332 --- /dev/null +++ b/ansible/services/memos/setup_backup_memos_to_lapy.yml @@ -0,0 +1,106 @@ +- name: Configure local backup for Memos from memos-box + hosts: control + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./memos_vars.yml + vars: + backup_data_path: "{{ memos_data_dir }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "backup_host={{ backup_host }}" + - "backup_user={{ backup_user }}" + - "backup_data_path='{{ backup_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if backup_key_file %} + SSH_CMD="ssh -i {{ backup_key_file }} -p {{ backup_port }}" + {% else %} + SSH_CMD="ssh -p {{ backup_port }}" + {% endif %} + + rsync -az -e "$SSH_CMD" --rsync-path="sudo rsync" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for dir in "{{ local_backup_dir }}"/20*; do + if [ -d "$dir" ]; then + dir_date=$(basename "$dir") + if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then + rm -rf "$dir" + fi + fi + done + + - name: Ensure cronjob for backup exists + cron: + name: "Memos backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 15 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check backup directory exists and contains files + stat: + path: "{{ local_backup_dir }}/{{ today_date.stdout }}" + register: backup_dir_stat + + - name: Verify backup directory exists + assert: + that: + - backup_dir_stat.stat.exists + - backup_dir_stat.stat.isdir + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" + success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" + + - name: Check if backup directory contains files + find: + paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" + recurse: yes + register: backup_files + + - name: Verify backup directory is not empty + assert: + that: + - backup_files.files | length > 0 + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" + success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" + diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 0042fb0..e9cab0e 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -1,27 +1,620 @@ ---- - name: Deploy Mempool Block Explorer with Docker hosts: mempool become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./mempool_vars.yml vars: - # Preserves the three push URLs these checks have been reporting to all - # along, so the move to a role changes no behaviour. The role knows nothing - # about Uptime Kuma — these are just "URLs that accept a ping", and whatever - # replaces it sets the same values. - mempool_healthchecks: - - {name: mariadb, label: MariaDB, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-mariadb/external"} - - {name: backend, label: Backend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-backend/external"} - - {name: frontend, label: Frontend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-frontend/external"} - # One token for all three components: they run on the same host, so the - # blast radius is already that host. - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - mempool + mempool_subdomain: "{{ subdomains.mempool }}" + mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + # =========================================== + # Docker Installation (from 910_docker_playbook.yml) + # =========================================== + - name: Remove old Docker-related packages + apt: + name: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + purge: yes + autoremove: yes + + - name: Update apt cache + apt: + update_cache: yes + + - name: Install prerequisites + apt: + name: + - ca-certificates + - curl + state: present + + - name: Create directory for Docker GPG key + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Download Docker GPG key + get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + + - name: Get Debian architecture + command: dpkg --print-architecture + register: deb_arch + changed_when: false + + - name: Add Docker repository + apt_repository: + repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + filename: docker + state: present + update_cache: yes + + - name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + + - name: Ensure Docker is started and enabled + systemd: + name: docker + enabled: yes + state: started + + - name: Add user to docker group + user: + name: "{{ ansible_user }}" + groups: docker + append: yes + + # =========================================== + # Mempool Deployment + # =========================================== + - name: Create mempool directories + file: + path: "{{ item }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + loop: + - "{{ mempool_dir }}" + - "{{ mempool_data_dir }}" + - "{{ mempool_mysql_dir }}" + + - name: Create docker-compose.yml for Mempool + copy: + dest: "{{ mempool_dir }}/docker-compose.yml" + content: | + # All containers use host network for Tailscale MagicDNS resolution + services: + mariadb: + image: mariadb:10.11 + container_name: mempool-db + restart: unless-stopped + network_mode: host + environment: + MYSQL_DATABASE: "{{ mariadb_database }}" + MYSQL_USER: "{{ mariadb_user }}" + MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" + MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" + volumes: + - {{ mempool_mysql_dir }}:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + mempool-backend: + image: mempool/backend:{{ mempool_version }} + container_name: mempool-backend + restart: unless-stopped + network_mode: host + environment: + # Database (localhost since all containers share host network) + DATABASE_ENABLED: "true" + DATABASE_HOST: "127.0.0.1" + DATABASE_DATABASE: "{{ mariadb_database }}" + DATABASE_USERNAME: "{{ mariadb_user }}" + DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" + # Bitcoin Core/Knots (via Tailnet MagicDNS) + CORE_RPC_HOST: "{{ bitcoin_host }}" + CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" + CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" + CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" + # Electrum (Fulcrum via Tailnet MagicDNS) + ELECTRUM_HOST: "{{ fulcrum_host }}" + ELECTRUM_PORT: "{{ fulcrum_port }}" + ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" + # Mempool settings + MEMPOOL_NETWORK: "{{ mempool_network }}" + MEMPOOL_BACKEND: "electrum" + MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" + volumes: + - {{ mempool_data_dir }}:/backend/cache + depends_on: + mariadb: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + mempool-frontend: + image: mempool/frontend:{{ mempool_version }} + container_name: mempool-frontend + restart: unless-stopped + network_mode: host + environment: + FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" + BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" + depends_on: + - mempool-backend + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' + + - name: Pull Mempool images + command: docker compose pull + args: + chdir: "{{ mempool_dir }}" + + - name: Deploy Mempool containers with docker compose + command: docker compose up -d + args: + chdir: "{{ mempool_dir }}" + + - name: Wait for MariaDB to be healthy + command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db + register: mariadb_health + until: mariadb_health.stdout == 'healthy' + retries: 30 + delay: 10 + changed_when: false + + - name: Wait for Mempool backend to start + uri: + url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + method: GET + status_code: 200 + timeout: 10 + register: backend_check + until: backend_check.status == 200 + retries: 30 + delay: 10 + ignore_errors: yes + + - name: Wait for Mempool frontend to be available + uri: + url: "http://localhost:{{ mempool_frontend_port }}" + method: GET + status_code: 200 + timeout: 10 + register: frontend_check + until: frontend_check.status == 200 + retries: 20 + delay: 5 + ignore_errors: yes + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Display deployment status + when: uptime_kuma_enabled | default(false) + debug: + msg: + - "Mempool deployment complete!" + - "Frontend: http://localhost:{{ mempool_frontend_port }}" + - "Backend API: http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + - "Backend check: {{ 'OK' if backend_check.status == 200 else 'Still initializing...' }}" + - "Frontend check: {{ 'OK' if frontend_check.status == 200 else 'Still initializing...' }}" + + # =========================================== + # Health Check Scripts for Uptime Kuma Push Monitors + # =========================================== + - name: Create Mempool MariaDB health check script + when: uptime_kuma_enabled | default(false) + copy: + dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + check_container() { + local status=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db 2>/dev/null) + [ "$status" = "healthy" ] + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_container; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "MariaDB container unhealthy" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create Mempool backend health check script + when: uptime_kuma_enabled | default(false) + copy: + dest: /usr/local/bin/mempool-backend-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + BACKEND_PORT={{ mempool_backend_port }} + + check_backend() { + curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_backend; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Backend API not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create Mempool frontend health check script + when: uptime_kuma_enabled | default(false) + copy: + dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + FRONTEND_PORT={{ mempool_frontend_port }} + + check_frontend() { + curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_frontend; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Frontend not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + # =========================================== + # Systemd Timers for Health Checks + # =========================================== + - name: Create systemd services for health checks + copy: + dest: "/etc/systemd/system/mempool-{{ item.name }}-healthcheck.service" + content: | + [Unit] + Description=Mempool {{ item.label }} Health Check + After=network.target docker.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/mempool-{{ item.name }}-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + loop: + - { name: "mariadb", label: "MariaDB" } + - { name: "backend", label: "Backend" } + - { name: "frontend", label: "Frontend" } + + - name: Create systemd timers for health checks + copy: + dest: "/etc/systemd/system/mempool-{{ item }}-healthcheck.timer" + content: | + [Unit] + Description=Mempool {{ item }} Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + loop: + - mariadb + - backend + - frontend + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start health check timers + when: uptime_kuma_enabled | default(false) + systemd: + name: "mempool-{{ item }}-healthcheck.timer" + enabled: yes + state: started + loop: + - mariadb + - backend + - frontend + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for Mempool + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_mempool_monitors.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_mempool_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitors_to_create = config['monitors'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + results = {} + for monitor_name in monitors_to_create: + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + results[monitor_name] = f"{url}/api/push/{push_token}" + print(f"Push URL ({monitor_name}): {results[monitor_name]}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=90, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + results[monitor_name] = f"{url}/api/push/{push_token}" + print(f"Push URL ({monitor_name}): {results[monitor_name]}") + + api.disconnect() + print("SUCCESS") + + # Write results to file for Ansible to read + with open('/tmp/mempool_push_urls.yml', 'w') as f: + yaml.dump(results, f) + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_mempool_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitors: + - "Mempool MariaDB" + - "Mempool Backend" + - "Mempool Frontend" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_mempool_monitors.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URLs from file + when: uptime_kuma_enabled | default(false) + slurp: + src: /tmp/mempool_push_urls.yml + delegate_to: localhost + become: no + register: push_urls_file + ignore_errors: yes + + - name: Parse push URLs + set_fact: + push_urls: "{{ push_urls_file.content | b64decode | from_yaml }}" + when: push_urls_file.content is defined + ignore_errors: yes + + - name: Update MariaDB health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-mariadb-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool MariaDB'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool MariaDB'] is defined + + - name: Update Backend health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-backend-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Backend'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool Backend'] is defined + + - name: Update Frontend health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-frontend-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Frontend'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool Frontend'] is defined + + - name: Reload systemd after push URL updates + systemd: + daemon_reload: yes + when: push_urls is defined + + - name: Restart health check timers + systemd: + name: "mempool-{{ item }}-healthcheck.timer" + state: restarted + loop: + - mariadb + - backend + - frontend + when: push_urls is defined + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_mempool_monitors.py + - /tmp/ansible_mempool_config.yml + - /tmp/mempool_push_urls.yml - name: Configure Caddy reverse proxy for Mempool on the edge host hosts: edge become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./mempool_vars.yml vars: - mempool_domain: "{{ subdomains.mempool }}.{{ root_domain }}" + mempool_subdomain: "{{ subdomains.mempool }}" + mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" + tasks: - name: Publish Mempool through Caddy (via Tailscale) ansible.builtin.include_role: @@ -29,5 +622,124 @@ vars: caddy_site_name: mempool caddy_site_domain: "{{ mempool_domain }}" - caddy_site_upstream: "{{ hostvars['mempool_box_local'].ansible_host }}:{{ hostvars['mempool_box_local'].mempool_frontend_port }}" + caddy_site_upstream: "mempool-box:{{ mempool_frontend_port }}" caddy_site_resolvers: "100.100.100.100" + + - name: Display Mempool URL + when: uptime_kuma_enabled | default(false) + debug: + msg: "Mempool is now available at https://{{ mempool_domain }}" + + # =========================================== + # Uptime Kuma HTTP Monitor for Public Endpoint + # =========================================== + - name: Create Uptime Kuma HTTP monitor setup script for Mempool + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_mempool_http_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_mempool_http_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating HTTP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for HTTP monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_mempool_http_config.yml + content: | + uptime_kuma_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ mempool_domain }}" + monitor_name: "Mempool Public Access" + mode: '0644' + + - name: Run Uptime Kuma HTTP monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_mempool_http_monitor.py + delegate_to: localhost + become: no + register: http_monitor_setup + changed_when: "'SUCCESS' in http_monitor_setup.stdout" + ignore_errors: yes + + - name: Display HTTP monitor setup output + debug: + msg: "{{ http_monitor_setup.stdout_lines }}" + when: http_monitor_setup.stdout is defined + + - name: Clean up HTTP monitor temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_mempool_http_monitor.py + - /tmp/ansible_mempool_http_config.yml + diff --git a/ansible/services/mempool/mempool_vars.yml b/ansible/services/mempool/mempool_vars.yml new file mode 100644 index 0000000..d3051c3 --- /dev/null +++ b/ansible/services/mempool/mempool_vars.yml @@ -0,0 +1,33 @@ +# Mempool Configuration Variables + +# Version - Pinned to specific release +mempool_version: "v3.2.1" + +# Directories +mempool_dir: /opt/mempool +mempool_data_dir: "{{ mempool_dir }}/data" +mempool_mysql_dir: "{{ mempool_dir }}/mysql" + +# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) +bitcoin_host: "knots-box" +bitcoin_rpc_port: 8332 +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml + +# Network - Fulcrum Electrum server (via Tailnet Magic DNS) +fulcrum_host: "fulcrum-box" +fulcrum_port: 50001 +fulcrum_tls: "false" + +# Mempool network mode +mempool_network: "mainnet" + +# Container ports (internal) +mempool_frontend_port: 8080 +mempool_backend_port: 8999 + +# MariaDB settings +mariadb_database: "mempool" +mariadb_user: "mempool" +# Note: mariadb_mempool_password is loaded from infra_secrets.yml + + diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index 58f0348..7379d5f 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -2,23 +2,18 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml + - ../../infra_secrets.yml + - ../../services_config.yml - ./ntfy_emergency_app_vars.yml vars: ntfy_emergency_app_subdomain: "{{ subdomains.ntfy_emergency_app }}" ntfy_emergency_app_domain: "{{ ntfy_emergency_app_subdomain }}.{{ root_domain }}" - # ⚠ UNRESOLVED: this app sends its notifications to an ntfy server, and the - # server it points at ran on watchtower, which is being destroyed. This was - # derived from subdomains.ntfy, which is now gone with it. - # - # Until an ntfy server exists again this URL is dead, and the app fails - # silently at exactly the moment it matters - it is an EMERGENCY notifier. - # Three ways out, none of them automatic: - # * stand ntfy up somewhere else (the monitoring VPS, or vipy) - # * point this at the public ntfy.sh - # * retire the app - ntfy_emergency_app_ntfy_url: "{{ ntfy_service_url }}" + ntfy_service_domain: "{{ subdomains.ntfy }}.{{ root_domain }}" + ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" ntfy_emergency_app_ntfy_user: "{{ ntfy_username | default('') }}" ntfy_emergency_app_ntfy_password: "{{ ntfy_password | default('') }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create ntfy-emergency-app directory @@ -60,3 +55,127 @@ caddy_site_name: ntfy-emergency-app caddy_site_domain: "{{ ntfy_emergency_app_domain }}" caddy_site_upstream: "localhost:{{ ntfy_emergency_app_port }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_ntfy_emergency_app_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ ntfy_emergency_app_domain }}" + monitor_name: "ntfy-emergency-app" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_ntfy_emergency_app_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_ntfy_emergency_app_monitor.py + - /tmp/ansible_config.yml diff --git a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml index bfa4bd1..a3bb480 100644 --- a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml +++ b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml @@ -2,7 +2,7 @@ ntfy_emergency_app_dir: /opt/ntfy-emergency-app ntfy_emergency_app_port: 3000 -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # ntfy configuration ntfy_emergency_app_topic: "emergencia" @@ -14,9 +14,3 @@ remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_h remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" - -# Where the emergency notifications are sent. This pointed at ntfy.contrapeso.xyz -# on watchtower; that host is being destroyed, so this MUST be repointed before -# the app can work again. Left at the old value so the break is visible rather -# than silently defaulted to something plausible. -ntfy_service_url: "https://ntfy.contrapeso.xyz" diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml new file mode 100644 index 0000000..61fafe1 --- /dev/null +++ b/ansible/services/ntfy/deploy_ntfy_playbook.yml @@ -0,0 +1,100 @@ +- name: Deploy ntfy and configure Caddy reverse proxy + hosts: monitoring + become: yes + vars_files: + - ../../infra_vars.yml + - ../../infra_secrets.yml + - ../../services_config.yml + - ./ntfy_vars.yml + vars: + ntfy_subdomain: "{{ subdomains.ntfy }}" + ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" + + tasks: + - name: Ensure /etc/apt/keyrings exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Download and dearmor ntfy GPG key + shell: curl -fsSL https://archive.heckel.io/apt/pubkey.txt | gpg --dearmor -o /etc/apt/keyrings/archive.heckel.io.gpg + args: + creates: /etc/apt/keyrings/archive.heckel.io.gpg + + - name: Add ntfy APT repository + copy: + dest: /etc/apt/sources.list.d/archive.heckel.io.list + content: | + deb [arch=amd64 signed-by=/etc/apt/keyrings/archive.heckel.io.gpg] https://archive.heckel.io/apt debian main + mode: '0644' + + - name: Update APT cache + apt: + update_cache: yes + + - name: Install ntfy + apt: + name: ntfy + state: present + + - name: Ensure ntfy cache directories exist + file: + path: "{{ item }}" + state: directory + owner: ntfy + group: ntfy + mode: '0755' + loop: + - /var/cache/ntfy + - /var/cache/ntfy/attachments + + - name: Deploy ntfy configuration file + copy: + dest: /etc/ntfy/server.yml + content: | + base-url: "http://{{ ntfy_domain }}" + listen-http: ":{{ ntfy_port }}" + cache-file: "/var/cache/ntfy/cache.db" + attachment-cache-dir: "/var/cache/ntfy/attachments" + behind-proxy: true + auth-file: "/var/lib/ntfy/user.db" + auth-default-access: "deny-all" + owner: root + group: root + mode: '0644' + notify: Restart ntfy + + - name: Enable and start ntfy service + systemd: + name: ntfy + enabled: yes + state: started + + - name: Create ntfy admin user + shell: | + (echo "{{ ntfy_password }}"; echo "{{ ntfy_password }}") | ntfy user add --role=admin "{{ ntfy_username }}" + + - name: Publish ntfy through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: ntfy + caddy_site_domain: "{{ ntfy_domain }}, http://{{ ntfy_domain }}" + # Raw body: ntfy needs a plain-HTTP listener for its CLI/app clients, + # with only GETs to the docs and topic paths redirected to HTTPS. + caddy_site_body: | + reverse_proxy 127.0.0.1:{{ ntfy_port }} + + @httpget { + protocol http + method GET + path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) + } + redir @httpget https://{host}{uri} + + handlers: + - name: Restart ntfy + systemd: + name: ntfy + state: restarted \ No newline at end of file diff --git a/ansible/services/ntfy/ntfy_vars.yml b/ansible/services/ntfy/ntfy_vars.yml new file mode 100644 index 0000000..5ebec37 --- /dev/null +++ b/ansible/services/ntfy/ntfy_vars.yml @@ -0,0 +1,3 @@ +ntfy_port: 6674 + +# ntfy_topic now lives in services_config.yml under service_settings.ntfy.topic \ No newline at end of file diff --git a/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml similarity index 97% rename from archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml rename to ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml index 97d6c9c..2d3d22a 100644 --- a/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml +++ b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml @@ -12,13 +12,17 @@ # What was being monitored: archive/uptime_kuma/MONITORS.md # ═════════════════════════════════════════════════════════════════════════════ - name: Setup ntfy as Uptime Kuma Notification Channel - hosts: observability + hosts: monitoring become: no vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./ntfy_vars.yml vars: ntfy_subdomain: "{{ subdomains.ntfy }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" uptime_kuma_subdomain: "{{ subdomains.uptime_kuma }}" ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" ntfy_server_url: "https://{{ ntfy_domain }}" diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index 21d8b43..96d030f 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -2,10 +2,14 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./personal_blog_vars.yml vars: personal_blog_subdomain: "{{ subdomains.personal_blog }}" personal_blog_domain: "{{ personal_blog_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure user is in www-data group @@ -53,3 +57,123 @@ caddy_site_name: personal-blog caddy_site_domain: "{{ personal_blog_domain }}" caddy_site_root: "{{ personal_blog_web_root }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Uptime Kuma monitor setup script for Personal Blog + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_personal_blog_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ personal_blog_domain }}" + monitor_name: "Personal Blog" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_personal_blog_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_personal_blog_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml index bd9b715..99f9b34 100644 --- a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml +++ b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml @@ -2,6 +2,7 @@ hosts: control gather_facts: no vars_files: + - ../../infra_vars.yml - ./personal_blog_vars.yml vars: bashrc_path: "{{ lookup('env', 'HOME') }}/.bashrc" diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index b9cf243..8b30a1f 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -1,15 +1,552 @@ --- -# phoenixd: Lightning node on the edge host, used by LNBits as a wallet backend. -# Never exposed through Caddy — the HTTP API stays on loopback. +# phoenixd Deployment Playbook +# +# Deploys phoenixd (https://phoenix.acinq.co/server), the server version of the +# Phoenix Lightning wallet, on vipy so LNBits can use it as a wallet backend +# over loopback. +# +# What this does: +# 1. Downloads the pinned phoenixd release and installs phoenixd + phoenix-cli +# 2. Creates a dedicated system user and a 0700 data directory +# 3. Creates and enables a systemd service +# 4. Creates a push-monitor health check script + systemd timer +# 5. Registers a push monitor in Uptime Kuma +# +# The HTTP API stays bound to 127.0.0.1 and is NOT proxied by Caddy: phoenixd +# holds funds and its API is protected by a single password. Anything that needs +# it either runs on this host or reaches it over the Tailscale mesh. +# +# ⚠️ After the first run, back up {{ phoenixd_data_dir }}/seed.dat. Losing it +# means losing the funds. See setup_backup_phoenixd_to_lapy.yml. + - name: Deploy phoenixd on the edge host hosts: edge become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./phoenixd_vars.yml vars: - # phoenixd's health check has never reported anywhere since the Uptime Kuma - # decommissioning — its systemd Environment= was left empty. Leaving it empty - # preserves that; the check still runs and its exit code is still the answer. - # Set this to plug in whatever monitoring replaces it. - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_phoenixd/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - phoenixd + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + # =========================================== + # Prerequisites + # =========================================== + - name: Install phoenixd runtime dependencies + apt: + name: + - unzip + - curl + state: present + update_cache: yes + + # =========================================== + # System User and Directories + # =========================================== + - name: Create phoenixd system group + group: + name: "{{ phoenixd_group }}" + system: yes + + - name: Create phoenixd system user + user: + name: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ phoenixd_home }}" + create_home: yes + comment: "phoenixd Lightning node" + + - name: Create phoenixd home directory + file: + path: "{{ phoenixd_home }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0750" + + - name: Create phoenixd data directory + file: + path: "{{ phoenixd_data_dir }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0700" + + # =========================================== + # Download and Install + # =========================================== + - name: Check if phoenixd is already installed + stat: + path: "{{ phoenixd_bin_dir }}/phoenixd" + register: phoenixd_binary + + - name: Check installed phoenixd version + command: "{{ phoenixd_bin_dir }}/phoenixd --version" + register: phoenixd_installed_version + changed_when: false + failed_when: false + when: phoenixd_binary.stat.exists + + - name: Decide whether phoenixd needs installing + set_fact: + phoenixd_needs_install: >- + {{ not phoenixd_binary.stat.exists + or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} + + - name: Download phoenixd {{ phoenixd_version }} + get_url: + url: "{{ phoenixd_url }}" + dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + mode: "0644" + when: phoenixd_needs_install | bool + + - name: Create temporary extraction directory + file: + path: /tmp/phoenixd-extract + state: directory + mode: "0755" + when: phoenixd_needs_install | bool + + - name: Extract phoenixd archive + unarchive: + src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + dest: /tmp/phoenixd-extract + remote_src: yes + when: phoenixd_needs_install | bool + + - name: Locate extracted binaries + find: + paths: /tmp/phoenixd-extract + patterns: "{{ item }}" + recurse: yes + file_type: file + register: phoenixd_extracted + loop: + - phoenixd + - phoenix-cli + when: phoenixd_needs_install | bool + + - name: Fail if the archive did not contain the expected binaries + assert: + that: + - item.files | length > 0 + fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + + - name: Install phoenixd and phoenix-cli binaries + copy: + src: "{{ item.files[0].path }}" + dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" + remote_src: yes + owner: root + group: root + mode: "0755" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + notify: Restart phoenixd + + - name: Clean up phoenixd download artifacts + file: + path: "{{ item }}" + state: absent + loop: + - "/tmp/phoenixd-{{ phoenixd_version }}.zip" + - /tmp/phoenixd-extract + + # =========================================== + # Systemd Service + # =========================================== + - name: Build phoenixd command line arguments + set_fact: + phoenixd_args: >- + {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) + + ['--chain', phoenixd_chain] + + ['--auto-liquidity', phoenixd_auto_liquidity] + + ['--http-bind-ip', phoenixd_http_bind_ip] + + ['--http-bind-port', phoenixd_http_bind_port | string] + + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) + + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) + + ['--silent'] }} + + - name: Create phoenixd systemd service + copy: + dest: /etc/systemd/system/phoenixd.service + content: | + [Unit] + Description=phoenixd - Lightning Network Node + Documentation=https://phoenix.acinq.co/server + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User={{ phoenixd_user }} + Group={{ phoenixd_group }} + WorkingDirectory={{ phoenixd_home }} + Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} + ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} + Restart=always + RestartSec=30 + TimeoutStartSec=120 + TimeoutStopSec=120 + StandardOutput=journal + StandardError=journal + + # Hardening: the node only ever writes to its own data directory + NoNewPrivileges=true + PrivateTmp=true + ProtectSystem=strict + ProtectHome=read-only + ReadWritePaths={{ phoenixd_data_dir }} + + LimitNOFILE=65535 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + notify: Restart phoenixd + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start phoenixd + systemd: + name: phoenixd + enabled: yes + state: started + + - name: Flush handlers so phoenixd is running before we inspect its data dir + meta: flush_handlers + + # =========================================== + # First Boot Checks + # =========================================== + - name: Wait for phoenixd to write its config file + wait_for: + path: "{{ phoenixd_data_dir }}/phoenix.conf" + state: present + timeout: 120 + + - name: Check that the seed file exists + stat: + path: "{{ phoenixd_data_dir }}/seed.dat" + register: phoenixd_seed_file + + - name: Fail if phoenixd did not create a seed + assert: + that: + - phoenixd_seed_file.stat.exists + fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" + + # =========================================== + # Health Check Script + Systemd Timer + # =========================================== + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create phoenixd health check script + when: uptime_kuma_enabled | default(false) + copy: + dest: "{{ phoenixd_healthcheck_script_path }}" + content: | + #!/bin/bash + # Checks phoenixd and pushes the result to Uptime Kuma. + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" + + check_phoenixd() { + # Service must be active and the node must answer getinfo. + # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, + # but not the bind address, so pass it explicitly. + systemctl is-active --quiet phoenixd && \ + {{ phoenixd_bin_dir }}/phoenix-cli \ + --http-bind-ip {{ phoenixd_http_bind_ip }} \ + --http-bind-port {{ phoenixd_http_bind_port }} \ + getinfo 2>/dev/null | grep -q '"nodeId"' + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_phoenixd; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "phoenixd not responding" + exit 1 + fi + owner: root + group: root + mode: "0755" + + - name: Create phoenixd health check systemd service + copy: + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" + content: | + [Unit] + Description=phoenixd Health Check + After=network.target phoenixd.service + + [Service] + Type=oneshot + User=root + ExecStart={{ phoenixd_healthcheck_script_path }} + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + + - name: Create phoenixd health check systemd timer + copy: + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" + content: | + [Unit] + Description=phoenixd Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: "0644" + + - name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + + - name: Enable and start phoenixd health check timer + when: uptime_kuma_enabled | default(false) + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + enabled: yes + state: started + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for phoenixd + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_phoenixd_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_phoenixd_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + push_url = None + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + else: + print(f"Creating push monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=90, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + # socketio timeout: add_monitor may have succeeded server-side + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + + api.disconnect() + + if push_url: + print(f"PUSH_URL={push_url}") + with open('/tmp/phoenixd_push_url.txt', 'w') as f: + f.write(push_url) + + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for push monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_phoenixd_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "{{ phoenixd_monitor_name }}" + mode: "0644" + + - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_phoenixd_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URL from file + when: uptime_kuma_enabled | default(false) + slurp: + src: /tmp/phoenixd_push_url.txt + delegate_to: localhost + become: no + register: push_url_file + ignore_errors: yes + + - name: Parse push URL + set_fact: + phoenixd_push_url: "{{ push_url_file.content | b64decode | trim }}" + when: push_url_file.content is defined + + - name: Update health check service with push URL + lineinfile: + path: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" + regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" + line: "Environment=UPTIME_KUMA_PUSH_URL={{ phoenixd_push_url }}" + when: phoenixd_push_url is defined + notify: Restart phoenixd health check timer + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_phoenixd_monitor.py + - /tmp/ansible_phoenixd_config.yml + - /tmp/phoenixd_push_url.txt + + # =========================================== + # Post-install Notes + # =========================================== + - name: Display post-install information + debug: + msg: | + ✓ phoenixd {{ phoenixd_version }} deployed + + Status: systemctl status phoenixd + Logs: journalctl -u phoenixd -f + CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo + HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) + Data dir: {{ phoenixd_data_dir }} + + API password (needed to wire LNBits up to this node): + sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf + + ⚠️ BACK UP THE SEED NOW: {{ phoenixd_data_dir }}/seed.dat + Losing it means losing the funds. Run + services/phoenixd/setup_backup_phoenixd_to_lapy.yml and also keep + the 12 words somewhere offline. + + handlers: + - name: Restart phoenixd + systemd: + name: phoenixd + state: restarted + daemon_reload: yes + + - name: Restart phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/phoenixd/defaults/main.yml b/ansible/services/phoenixd/phoenixd_vars.yml similarity index 63% rename from ansible/roles/phoenixd/defaults/main.yml rename to ansible/services/phoenixd/phoenixd_vars.yml index 2fcf1ea..f67c302 100644 --- a/ansible/roles/phoenixd/defaults/main.yml +++ b/ansible/services/phoenixd/phoenixd_vars.yml @@ -35,22 +35,18 @@ phoenixd_http_bind_port: 9740 # Optional webhook for payment events. Leave empty to disable. phoenixd_webhook_url: "" - - - -# --- Health check ----------------------------------------------------------- -# The check asks phoenixd itself whether it is healthy (service active AND the -# node answers getinfo with a nodeId) and records the answer in its exit code, -# which systemd keeps: -# systemctl is-failed phoenixd-healthcheck.service -# That is a complete answer with no monitoring system involved. +# Monitoring phoenixd_healthcheck_script_path: /usr/local/bin/phoenixd-healthcheck-push.sh phoenixd_healthcheck_service_name: phoenixd-healthcheck +phoenixd_monitor_name: "Phoenixd" -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. -# Empty means check, log, exit honestly, report nowhere. Any endpoint that -# accepts an HTTP ping works; nothing here is specific to a monitoring product. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" +# Remote access +remote_host_name: "{{ groups['edge'] | first }}" +remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" +remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" +remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" +remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" + +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/phoenixd-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/phoenixd_backup.sh" diff --git a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml new file mode 100644 index 0000000..7474af1 --- /dev/null +++ b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml @@ -0,0 +1,136 @@ +--- +# Backs up the phoenixd seed and config from vipy to Lapy, gpg encrypted. +# +# Only seed.dat and phoenix.conf are backed up, on purpose: +# - seed.dat is what actually recovers the funds. phoenixd keeps its channel +# state with the ACINQ peer, so a wallet is restored from the seed alone. +# - restoring a *stale* channel database to a running Lightning node is +# dangerous (it can trigger a force close and a penalty), so we do not keep +# copies of phoenix.db around to be tempted by. +# - phoenix.conf holds the http api password, which is what LNBits and any +# other local consumer authenticate with. +# +# Because both files are static, phoenixd does not need to be stopped. + +- name: Configure local backup for phoenixd from remote + hosts: control + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./phoenixd_vars.yml + vars: + remote_data_path: "{{ phoenixd_data_dir }}" + gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" + gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" + + tasks: + - name: Debug phoenixd backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + - "gpg_recipient={{ gpg_recipient }}" + - "gpg_key_id={{ gpg_key_id }}" + + - name: Ensure local backup directory exists + ansible.builtin.file: + path: "{{ local_backup_dir }}" + state: directory + mode: "0700" + + - name: Ensure ~/.local/bin exists + ansible.builtin.file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: "0755" + + - name: Create phoenixd backup script + ansible.builtin.copy: + dest: "{{ backup_script_path }}" + mode: "0750" + content: | + #!/bin/bash + set -euo pipefail + + if [ -z "{{ gpg_recipient }}" ]; then + echo "GPG recipient is not configured. Aborting." + exit 1 + fi + + TIMESTAMP=$(date +'%Y-%m-%d') + ENCRYPTED_BACKUP="{{ local_backup_dir }}/phoenixd-backup-$TIMESTAMP.tar.gz.gpg" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + # seed.dat + phoenix.conf only, see the header of the playbook. + echo "Creating encrypted backup archive..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} \ + "sudo tar -czf - -C {{ remote_data_path }} seed.dat phoenix.conf" | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" + + chmod 600 "$ENCRYPTED_BACKUP" + + # Rotate old backups (keep 14 days) + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for backup_file in "{{ local_backup_dir }}"/phoenixd-backup-*.tar.gz.gpg; do + if [ -f "$backup_file" ]; then + # Extract date from filename: phoenixd-backup-YYYY-MM-DD.tar.gz.gpg + file_date=$(basename "$backup_file" | sed -n 's/phoenixd-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$backup_file" + fi + fi + done + + echo "Backup completed successfully" + + - name: Ensure cronjob for phoenixd backup exists + ansible.builtin.cron: + name: "phoenixd backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 15 + hour: "9" + + - name: Run phoenixd backup script to create initial backup + ansible.builtin.command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check if backup file exists + stat: + path: "{{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg" + register: backup_file_stat + + - name: Verify backup file exists + assert: + that: + - backup_file_stat.stat.exists + - backup_file_stat.stat.isreg + fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" + success_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists" + + - name: Verify backup file is not empty + assert: + that: + - backup_file_stat.stat.size > 0 + fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" + success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" + + - name: Remind about the offline seed copy + debug: + msg: | + These encrypted backups are only as safe as your GPG key. + Also write the 12 words down offline once: + ssh {{ remote_user }}@{{ remote_host }} "sudo cat {{ remote_data_path }}/seed.dat" diff --git a/ansible/services/signal-api/deploy_signal_api_playbook.yml b/ansible/services/signal-api/deploy_signal_api_playbook.yml deleted file mode 100644 index d4690f9..0000000 --- a/ansible/services/signal-api/deploy_signal_api_playbook.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -# The Signal transport for Gatus alerts. -# -# Deliberately NOT published and NOT fronted by Caddy: the API has no -# authentication of any kind, so it is reachable only from the shared docker -# network that Gatus is on. See roles/signal_api/README.md, including the -# one-time manual step to link the device. -- name: Deploy the Signal API on the observability host - hosts: observability - become: yes - roles: - - signal_api - - # post_tasks, not a second play: the registration below needs the role's - # defaults (service name, port) in scope, and a separate play would not have - # them. - post_tasks: - # Monitored, because a dead alert transport is the worst kind of dead: every - # check could be failing and nothing would tell you. Gatus polls it over the - # shared network - the same path the alerts take - so this proves the actual - # delivery route rather than merely that a container is running. - # - # Deliberately NOT backed up: the data directory holds Signal private keys, - # and the recovery path is to link the device again from the phone. Backing - # it up would copy a credential off the host to buy nothing. - - name: Register the signal-api health endpoint with Gatus - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_name: signal-api - gatus_endpoint_pulled: - - name: signal-api - group: infrastructure - url: "http://{{ signal_api_service_name }}:{{ signal_api_port }}/v1/health" - interval: 5m - # Deliberately NOT alerted via Signal: if this endpoint is down, - # Signal is exactly what cannot deliver the alert. It is visible on - # the dashboard, and its failure shows up indirectly as every other - # alert going missing. - conditions: - # /v1/health answers 204 No Content, not 200 - checked live. Any - # 2xx is asserted rather than the exact code, so an upstream - # change from 204 to 200 does not read as an outage. - - "[STATUS] < 300" diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 282fc2e..74e87d8 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -2,10 +2,14 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./vaultwarden_vars.yml vars: vaultwarden_subdomain: "{{ subdomains.vaultwarden }}" vaultwarden_domain: "{{ vaultwarden_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create vaultwarden directory @@ -83,3 +87,128 @@ caddy_site_name: vaultwarden caddy_site_domain: "{{ vaultwarden_domain }}" caddy_site_upstream: "localhost:{{ vaultwarden_port }}" + + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ + - name: Create Uptime Kuma monitor setup script for Vaultwarden + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_vaultwarden_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ vaultwarden_domain }}/alive" + monitor_name: "Vaultwarden" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) + command: python3 /tmp/setup_vaultwarden_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_vaultwarden_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml index bccc2cd..eebb214 100644 --- a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml +++ b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml @@ -2,6 +2,7 @@ hosts: edge become: yes vars_files: + - ../../infra_vars.yml - ./vaultwarden_vars.yml tasks: diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml index d0dfdfa..8430475 100644 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -6,6 +6,7 @@ hosts: edge become: yes vars_files: + - ../../group_vars/all/main.yml - ./vaultwarden_vars.yml tasks: @@ -22,7 +23,3 @@ backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" backup_source_retention_days: 7 backup_source_on_calendar: "*-*-* 02:10:00" - # Reported to Gatus as backup-dump_vaultwarden. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_vaultwarden/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml new file mode 100644 index 0000000..096f9d3 --- /dev/null +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml @@ -0,0 +1,105 @@ +- name: Configure local backup for Vaultwarden from remote + hosts: control + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./vaultwarden_vars.yml + vars: + remote_data_path: "{{ vaultwarden_data_dir }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for dir in "{{ local_backup_dir }}"/20*; do + if [ -d "$dir" ]; then + dir_date=$(basename "$dir") + if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then + rm -rf "$dir" + fi + fi + done + + - name: Ensure cronjob for backup exists + cron: + name: "Vaultwarden backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check backup directory exists and contains files + stat: + path: "{{ local_backup_dir }}/{{ today_date.stdout }}" + register: backup_dir_stat + + - name: Verify backup directory exists + assert: + that: + - backup_dir_stat.stat.exists + - backup_dir_stat.stat.isdir + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" + success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" + + - name: Check if backup directory contains files + find: + paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" + recurse: yes + register: backup_files + + - name: Verify backup directory is not empty + assert: + that: + - backup_files.files | length > 0 + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" + success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" diff --git a/ansible/services/vaultwarden/vaultwarden_vars.yml b/ansible/services/vaultwarden/vaultwarden_vars.yml index 9edc2cd..e60f1d4 100644 --- a/ansible/services/vaultwarden/vaultwarden_vars.yml +++ b/ansible/services/vaultwarden/vaultwarden_vars.yml @@ -3,7 +3,7 @@ vaultwarden_dir: /opt/vaultwarden vaultwarden_data_dir: "{{ vaultwarden_dir }}/data" vaultwarden_port: 8222 -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # Remote access remote_host_name: "{{ groups['edge'] | first }}" @@ -12,3 +12,6 @@ remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counter remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/vaultwarden-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/vaultwarden_backup.sh" diff --git a/ansible/services_config.yml b/ansible/services_config.yml new file mode 100644 index 0000000..a20c853 --- /dev/null +++ b/ansible/services_config.yml @@ -0,0 +1,42 @@ +# Centralized Services Configuration +# Subdomains and Caddy settings for all services + +# Edit these subdomains to match your preferences +subdomains: + # Monitoring Services (on watchtower) + ntfy: ntfy + # DEPRECATED 2026-09-11 — Uptime Kuma is decommissioned and this subdomain no + # longer resolves to anything. Kept only because the deprecated monitoring + # blocks still template it into uptime_kuma_api_url. See archive/uptime_kuma/. + uptime_kuma: uptime + + # VPN Infrastructure (on spacey) + headscale: headscale + + # Core Services (on vipy) + vaultwarden: vault + forgejo: forgejo + lnbits: wallet + + # Secondary Services (on vipy) + ntfy_emergency_app: avisame + personal_blog: pablohere + + # Memos (on memos-box) + memos: memos + + # Mempool Block Explorer (on mempool_box, proxied via vipy) + mempool: mempool + + # DATUM Gateway dashboard (on knots_box, proxied via vipy) + datum_gateway: datum + +# Caddy configuration +caddy_sites_dir: /etc/caddy/sites-enabled + +# Service-specific settings shared across playbooks +service_settings: + ntfy: + topic: alerts + headscale: + namespace: counter-net diff --git a/ansible/site.yml b/ansible/site.yml deleted file mode 100644 index 35f5867..0000000 --- a/ansible/site.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -# Everything, in the order it has to happen. -# -# This file is a TABLE OF CONTENTS, not a second source of truth. It says what -# runs and in what order. It does NOT say which hosts get what — that stays on -# the `hosts:` line inside each playbook, exactly where it is today. Nothing -# moves; this file only makes the set readable in one place. -# -# What runs on a host? ansible-playbook site.yml --limit --list-hosts -# Who gets thing Y? the `hosts:` line in Y's own playbook -# What is a host? ansible-inventory --graph -# -# Run a slice with --limit, or run one playbook directly as before. Nothing here -# changes how any individual playbook behaves. - -# ── Baseline: every managed machine ───────────────────────────────────────── -- import_playbook: infra/01_user_and_access_setup_playbook.yml -- import_playbook: infra/02_firewall_and_fail2ban_playbook.yml -- import_playbook: infra/900_install_rsync.yml -- import_playbook: infra/920_join_headscale_mesh.yml -# Idempotent and kept permanently: guarantees a rebuilt or restored host cannot -# quietly bring the Uptime-Kuma-era monitoring back. -- import_playbook: infra/409_remove_legacy_monitoring.yml - -# ── Monitoring ────────────────────────────────────────────────────────────── -# Gatus first: the three plays below register endpoints with it, and registering -# against a host that is not serving yet would simply fail. -- import_playbook: services/gatus/deploy_gatus_playbook.yml -# The Signal transport for Gatus alerts. Shares a docker network with Gatus and -# publishes no ports - the API has no authentication. Needs a one-time manual -# device link; see roles/signal_api/README.md. -- import_playbook: services/signal-api/deploy_signal_api_playbook.yml -- import_playbook: infra/400_host_monitoring.yml -- import_playbook: infra/401_service_monitoring.yml -- import_playbook: infra/402_public_monitoring.yml -# Registers where the per-service probes report. The probes themselves are -# deployed by each service's own playbook further down; the endpoints must exist -# before the first push arrives. -- import_playbook: infra/403_service_probe_registration.yml -# 910_docker says `hosts: managed`, but only 5 of 11 managed hosts have or need -# Docker. Left out until it has a [docker] group — see the note in PLAN_7. - -# ── The hypervisor ────────────────────────────────────────────────────────── -- import_playbook: infra/nodito/31_proxmox_community_repos_playbook.yml -- import_playbook: infra/nodito/32_zfs_pool_setup_playbook.yml -- import_playbook: infra/nodito/34_nut_ups_setup_playbook.yml - -# ── Reverse proxy, before anything that registers a vhost ─────────────────── -- import_playbook: services/caddy_playbook.yml - -# ── Services ──────────────────────────────────────────────────────────────── -- import_playbook: services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml -- import_playbook: services/fulcrum/deploy_fulcrum_playbook.yml -- import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml -- import_playbook: services/mempool/deploy_mempool_playbook.yml -- import_playbook: services/memos/deploy_memos_playbook.yml -- import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml -- import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml -- import_playbook: services/headscale/deploy_headscale_playbook.yml -- import_playbook: services/vaultwarden/deploy_vaultwarden_playbook.yml -- import_playbook: services/forgejo/deploy_forgejo_playbook.yml -- import_playbook: services/lnbits/deploy_lnbits_playbook.yml -- import_playbook: services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml -- import_playbook: services/personal-blog/deploy_personal_blog_playbook.yml - -# ── Backups: each source dumps itself, the box pulls ──────────────────────── -- import_playbook: services/headscale/setup_backup_headscale.yml -- import_playbook: services/vaultwarden/setup_backup_vaultwarden.yml -- import_playbook: services/forgejo/setup_backup_forgejo.yml -- import_playbook: services/lnbits/setup_backup_lnbits.yml -- import_playbook: services/memos/setup_backup_memos.yml -- import_playbook: playbooks/backups.yml - -# Deliberately not here. Every playbook in the repo is either imported above or -# listed below, so this file accounts for all of them: -# -# infra/910_docker_playbook.yml says `hosts: managed`, but Docker is on 5 -# of 11 managed hosts and those 5 are exactly -# the ones that need it. Running it would -# install Docker on the Bitcoin node and the -# hypervisor. Needs a [docker] group first. -# -# infra/nodito/30_proxmox_bootstrap one-shot: bare-metal bootstrap, run once -# infra/nodito/33_..._cloud_template one-shot: builds the VM template -# -# -# services/vaultwarden/disable_ deliberate manual actions, not convergence -# vaultwarden_sign_ups_playbook.yml -# services/personal-blog/setup_ -# deploy_alias_lapy.yml diff --git a/tofu/nodito/.terraform.lock.hcl b/tofu/nodito/.terraform.lock.hcl deleted file mode 100644 index fadff38..0000000 --- a/tofu/nodito/.terraform.lock.hcl +++ /dev/null @@ -1,24 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/telmate/proxmox" { - version = "3.0.2-rc05" - constraints = "3.0.2-rc05" - hashes = [ - "h1:QfHovHn8h9uJXdJ+urOuiD7R46OXmdZQiRcCBaV6AD4=", - "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f", - "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc", - "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321", - "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3", - "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98", - "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb", - "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e", - "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9", - "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d", - "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3", - "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e", - "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269", - "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1", - "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123", - ] -} diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md index bff6585..3a0b18f 100644 --- a/tofu/nodito/README.md +++ b/tofu/nodito/README.md @@ -27,17 +27,16 @@ This directory lets you declare VMs on the `nodito` Proxmox node and apply with - The Ansible template exists: `debian-13-cloud-init` (VMID 9001 by default). ### Provider Auth -Credentials are the only thing not in git. Copy `terraform.tfvars.example` to -`terraform.tfvars` (gitignored) and set: +Create a `terraform.tfvars` (copy from `terraform.tfvars.example`) and set: - `proxmox_api_url` (e.g. `https://nodito:8006/api2/json`) - `proxmox_api_token_id` (e.g. `root@pam!tofu`) - `proxmox_api_token_secret` +- `ssh_authorized_keys` (your public key content) -Alternatively, export them as `TF_VAR_proxmox_api_token_secret` etc. +Alternatively, you can export env vars and reference them in a tfvars file. ### Declare VMs -VMs are declared in `vms.auto.tfvars`, which is committed. `*.auto.tfvars` is -loaded automatically, so it needs no `-var-file`. Example entry: +Edit `terraform.tfvars` and fill the `vms` map. Example entry: ``` vms = { web1 = { @@ -55,24 +54,15 @@ All VM disks are created on `zfs_storage_name` (defaults to `proxmox-tank-1`). N ### Usage ``` tofu init -tofu plan -tofu apply +tofu plan -var-file=terraform.tfvars +tofu apply -var-file=terraform.tfvars ``` -`terraform.tfvars` and `vms.auto.tfvars` are both auto-loaded. > VMs are created once and then protected: the module sets `lifecycle.prevent_destroy = true` and ignores subsequent config changes. After the initial apply, manage day‑2 changes directly in Proxmox (or remove the lifecycle block if you need OpenTofu to own ongoing updates). ### Notes - Clones are full clones by default (`full_clone = true`). - Cloud-init injects `cloud_init_user` and `ssh_authorized_keys`. -- `.terraform.lock.hcl` is committed: it pins the provider hashes, which matters - because `versions.tf` tracks a release candidate (`3.0.2-rc05`). -- State is local (`terraform.tfstate`, gitignored) and has no remote backend, so - it exists only on the machine that last ran `tofu apply`. -- The map is not a complete inventory of nodito: state also manages - `bastion-box` (1100) and `nonkeiwaisi-box` (3300), which are not declared in - `vms.auto.tfvars`. `tofu plan` is clean today, but relaxing the `lifecycle` - block without first declaring them would put them up for destruction. - Disks use `scsi0` on ZFS with `discard` enabled. diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example index 37b6d7a..cc88b3f 100644 --- a/tofu/nodito/terraform.tfvars.example +++ b/tofu/nodito/terraform.tfvars.example @@ -1,7 +1,35 @@ -# Provider credentials. Copy to terraform.tfvars (gitignored) and fill in. -# Everything else — the node, storage, template, SSH key and the vms map — -# is committed in vms.auto.tfvars and needs no local copy. +proxmox_api_url = "https://nodito:8006/api2/json" +proxmox_api_token_id = "root@pam!tofu" +proxmox_api_token_secret = "REPLACE_ME" + +proxmox_node = "nodito" +zfs_storage_name = "proxmox-tank-1" +template_name = "debian-13-cloud-init" +cloud_init_user = "counterweight" + +# paste your ~/.ssh/id_ed25519.pub or similar +ssh_authorized_keys = <