Compare commits
No commits in common. "f6656b0ff78845740c9a3733265c55350b25efd5" and "394f2519ff261f518473f6df171c736d51a164a2" have entirely different histories.
f6656b0ff7
...
394f2519ff
206 changed files with 8068 additions and 6646 deletions
11
.gitignore
vendored
11
.gitignore
vendored
|
|
@ -1,16 +1,13 @@
|
|||
# OpenTofu / Terraform
|
||||
.terraform/
|
||||
.tofu/
|
||||
.terraform.lock.hcl
|
||||
.tofu.lock.hcl
|
||||
terraform.tfstate
|
||||
terraform.tfstate.*
|
||||
crash.log
|
||||
|
||||
# Provider credentials only. Non-secret infra config (the vms map) is committed
|
||||
# as *.auto.tfvars, and *.lock.hcl is committed on purpose so provider hashes
|
||||
# are pinned.
|
||||
terraform.tfvars
|
||||
terraform.tfvars.json
|
||||
*secrets.auto.tfvars
|
||||
*.tfvars
|
||||
*.tfvars.json
|
||||
|
||||
venv/*
|
||||
.env
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ root_domain: contrapeso.xyz
|
|||
# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the
|
||||
# playbooks are kept deliberately — the check logic is meant to be rewired to
|
||||
# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
|
||||
uptime_kuma_enabled: false
|
||||
|
||||
# age recipient for all backup artefacts
|
||||
age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza"
|
||||
|
|
@ -13,54 +14,3 @@ age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr
|
|||
# Public key small-backups-box pulls with
|
||||
# Authorised on each source host for an unprivileged, dedicated user only
|
||||
backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Subdomains. Global because the edge host proxies for services that live on
|
||||
# other machines, so no single inventory group covers the readers. Combine with
|
||||
# root_domain above to build an FQDN.
|
||||
#
|
||||
# Moved here from services_config.yml, which 30 plays had to remember to name in
|
||||
# vars_files: - a file everyone must opt into is a file someone will forget.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
subdomains:
|
||||
# Monitoring
|
||||
gatus: status
|
||||
|
||||
# VPN infrastructure (spacey)
|
||||
headscale: headscale
|
||||
|
||||
# Core services (vipy)
|
||||
vaultwarden: vault
|
||||
forgejo: forgejo
|
||||
lnbits: wallet
|
||||
|
||||
# Secondary services (vipy)
|
||||
ntfy_emergency_app: avisame
|
||||
personal_blog: pablohere
|
||||
|
||||
# Memos (memos-box)
|
||||
memos: memos
|
||||
|
||||
# Mempool block explorer (mempool-box, proxied via vipy)
|
||||
mempool: mempool
|
||||
|
||||
# DATUM Gateway dashboard (knots-box, proxied via vipy)
|
||||
datum_gateway: datum
|
||||
|
||||
# Read by plays across several groups, so global rather than group_vars/<group>.
|
||||
headscale_namespace: counter-net
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Domains whose registration expiry is monitored (infra/402_public_monitoring).
|
||||
#
|
||||
# Registration renewal is a manual act at the registrar, and losing a domain is
|
||||
# not recoverable in the way losing a host is - so these are checked daily and
|
||||
# alarm with two weeks of runway.
|
||||
#
|
||||
# root_domain is the estate's own domain; the rest are domains we own that are
|
||||
# served from it or from a host in the inventory.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
monitored_domains:
|
||||
- "{{ root_domain }}"
|
||||
- arbret.com
|
||||
|
|
|
|||
|
|
@ -1,164 +1,60 @@
|
|||
$ANSIBLE_VAULT;1.1;AES256
|
||||
34383033613438623432656438313763613639313139626432303637383363643339613166626665
|
||||
6537663336353661373437303636613364643736663031300a623461353135623537343763383333
|
||||
37323731646262613436646231656263396532616132316237353762376363383062646132303435
|
||||
3435316666316235660a323163613932616661303935336633656332353766393536393736356539
|
||||
61396164386266303464373262373862363361363365363235383363613335663933313062663565
|
||||
39613466353161356462656539613335613536363532333431393935303430373435373737363537
|
||||
62383266323339373062313134316464303263313830646161333530383736336637333865373764
|
||||
31313165396366653763343466643561383236643836616662383439666165343139326232326364
|
||||
32633133373863616163343365656231353939366435346534383462396134653064663566343566
|
||||
66343464373865383864313264646635316233336133346365396634626261613561373965643339
|
||||
30353831393037316334313632376537393361336366326561343832373531303139323937613531
|
||||
34386164303430316162363536386137316330336638303365393462386465633862663932376563
|
||||
34323862636435306563333065383134343362333733623066656439613339376666353331373665
|
||||
32353938663364313036373237323062393430306661633732356434333232376338313835663330
|
||||
32306536393666323464663238306562613936376236613935616664653865373330626365626330
|
||||
64613066653632646638306338393331376430383165316363323437373362613866313962663432
|
||||
62356663636630356239613831353066303530363966396135376134633936313530643262353965
|
||||
38383330613962313935366633353635363638303937316362303866343433353437363334396431
|
||||
32393134626133353564643233616632376633303934653065613262353436333533376137303862
|
||||
35303861383833326537376433336634633730373833313036643365373431666438653033653930
|
||||
33346230396661636333613031353133353631306331666362386366633062666534303539316238
|
||||
39366634656333646634643032393738333330636230663366623265666533356534623465316665
|
||||
35336535326237343231653661613736626239316262386363666264613636326332366232333964
|
||||
38353635616662663133376534306366643037613732343233373336653166316330643638353539
|
||||
35353532643534613864333839393365313439343663333337373639633832383630316164306132
|
||||
63636162373034363839656638353534613733663737626432636436373164303239316436313566
|
||||
66623466396663353136323565643262383865323830616232373466373431373261306537646463
|
||||
61353966623062356438626635663461396165343366393132393965383734633632666335313030
|
||||
37333631373239303231396264393031373162373936303462363736313538343562343737356332
|
||||
63663762633032356130303964386664366137373432353533386566613037613163383963623732
|
||||
32343765376535343630656633333362393765666131653632613361363163383038323138333537
|
||||
37373333316466316562613439356365316163336366623435666166653835616531653563343664
|
||||
30303464663838646333373837363961356465613234636138633162366636633030383337643131
|
||||
62636236663066316630396132333934353139383465363034363033656161383663656462313939
|
||||
62353738396132343734343363636164323163616237613861643862303433633934656165343263
|
||||
35313737373333386335663731643664393630653839396464376639306231653934386336353630
|
||||
61393735336662663435613431386361373561363531643232303831363163636139653538393931
|
||||
66313965646562616237383439343637623835303065333730613865666638383131616261346463
|
||||
31396464306537613135653830653138323539393731626264326335336432666333333735646534
|
||||
30616439393464326631633635363466303336613135346231626232313361303666323661616638
|
||||
64363937343231363536353034363966326132333734386638613737636130646232363666643633
|
||||
61353563636466356630346232613761636432303430336461663434333636623962643764336639
|
||||
35303565613861323431393036303534303061666437366538306236373930313439353330313632
|
||||
61363363303562316664303065613863613339626632643931386438326330373938613762303334
|
||||
33316430363931393262373661623137633835656136313235613666613932313236343966306331
|
||||
36343031303632323432366337633637336335343638393564313738386162386164613339343662
|
||||
62383939663265373265613932633265626263623939303638383838646531343433393864363235
|
||||
37313761396436623833376435393537376136373162393465343764326533666533393061643965
|
||||
61306365346562333737313962633764313232323161623861336566343735353737633539656162
|
||||
34323939306661373662663331653333346264323930663636633134666532623438323537383165
|
||||
66343033363962633766303331623866306462623235373838616565653066646132633034363535
|
||||
31613462356361613535383963643662373162363363653334393937313039666266366363653537
|
||||
64356166396139393262313565303731346534646462333638316661383139346362333364613466
|
||||
38303561316263636136303431393366343936653161336238613439366266346136636465303337
|
||||
61373634376464333037303862616335623031656133636165636264386265643261373735663266
|
||||
39396265366138363430343035356337303438346165366361316230656239326633653537626339
|
||||
39613334316563376133343336343563363564653237633764393532336437303334363830396635
|
||||
31333963666462313830333163646465356337643263636462356363613266323630393866626535
|
||||
35336436663063333364353663616564626535396133386461326433323936323161613633353539
|
||||
66323935626433353162666335613839346561343264323763303034613366663233613037343538
|
||||
66376236373462383430313030373235373337653338333033306431616562353435656433356365
|
||||
37653063363435366532306334373438306632306435653334396630343863623938666335656164
|
||||
37386634366430383762316435396237313362313965353634613266616532633465646464313662
|
||||
35303433363236393465396161346431306231666432666531626136623966663864376663373562
|
||||
37653239633732346536323265643434616136326561666330633435383264333937353366653736
|
||||
61353430633363323263323163646639663330663632343038623964396433626337353232373334
|
||||
32343938356539303938666137323831646562393233323733336464316165623232326132316362
|
||||
61313631626234336164656533616231393663363364623130313439623466396231383537656562
|
||||
35393163323131383838356130643863633736613635306635306135666261623731613239303962
|
||||
61373432336638353339646665653561653738353731373032366361313562306466373365653333
|
||||
35306236336331333961613864346165613763383266613236343066623432303766333434333331
|
||||
66376263363132616132366165393235626336323665373135653462653533376138346632393363
|
||||
35396165383966326635333134663138366136336430393532373935373264616530613762386430
|
||||
37643862316232613261343430616438333831623835663239656531356666613032653837373531
|
||||
30386232643537306530323936386561646265616339376265353763663833623539653831656634
|
||||
63626564633964343663656435636635656562666236386639353332356566316631643431353330
|
||||
32643238623932643732353332633535643738663066383830376432346463393464333236386163
|
||||
64376165333266323166613564313061643832656239333165393035343836376434626162643061
|
||||
31373861393534636236383765313834356638613332626666653762343537363339383939666263
|
||||
61366230623930643534353565316165656132396664393138326139613536343738396330653735
|
||||
37623463343834626133313231393130646134363331393238663065643930383237326439343831
|
||||
32393536613361616637373761393361303936363863333838666633343063386235316237323531
|
||||
62633362316436623761616366383336613765373362393161303765613464643632666462343565
|
||||
63616430396164346263666436336463623638313130656231643131336663336565616135373433
|
||||
39326533643733333564373434333738343634383033353036646534646632383730613161393130
|
||||
62653836376435663435353739373536366466646332653030653332353761626466353631663163
|
||||
38613734383634663662376464653733306564313761336566336336656366653536643834653631
|
||||
35333963306562393234623039613338643266333762333236366331333431343830393864623830
|
||||
34323439386133613035363861623332663933313734313637623739613163663730316134303434
|
||||
35363239363836666463366334356435666632396132306163303037663430383535393934323537
|
||||
30353266373864346662333035393761646632313032383738346439343232663666343833333565
|
||||
35363564623038376666363436663830356165323234653230636264303066666230643632356237
|
||||
66633734346633366533623966393462313438333965623033343337323135353730613561363236
|
||||
30636333613034626162393137323762306432663761393766303163393536656338366664643432
|
||||
34636630323965396166356538623165663335333739323034333237306531393734306138343436
|
||||
36613937373932633234303336373530663533646635396664343935326334333239646534343336
|
||||
35396666353938326337646431353965653836323731313861613031356561376262363965623934
|
||||
65336164623134366365303062393161353965363937646363396564313138323938656330333964
|
||||
64663862363766613630333531623463616331363533343962653164376665303463336134346662
|
||||
66376238663632663932656230323638323135653666363736643065336330636236323064386361
|
||||
37646631303138616565383665663634333338613336346438366333636165366635313337373635
|
||||
62656236653961393061373262326534393339363236353336613339656462336661663739396261
|
||||
35363636653237316436656662633462306333313336313865333037636661383731336466666536
|
||||
36646537653431666261353232653233653930346538383438633466346636656432383963333765
|
||||
62346363633265306665366630383831303434336139343837616361666539656262633735363064
|
||||
30376138386531336430613563366666333964663231363465326230316233323532366632663930
|
||||
63653164373931353861653833326130643365373530663039363166366331303134366336613438
|
||||
63633336363539636265396331653630633162316366316130326531363463616464376635376538
|
||||
32376638353337343639633434613663666437313564333264383766623561393933333561663235
|
||||
37666163626538353736383435373530663438363636643665626266316338336532623235353337
|
||||
61653762383630616266353438386137653137616532633066623165383663646437343565366237
|
||||
33333463613539346136613135386466656135616136663036313661323738363066633038656139
|
||||
38353230313861666463383737303038613731316334333031643438343463666164633536313031
|
||||
39613663376436643165333362636462303230656336333234363336633262336363643736336231
|
||||
66316630613762383832613836333263303038306235663662323162633932653733626130356665
|
||||
65316630346534356261373961353130393334396365376237353439326261336339393336303030
|
||||
63323439666439313831626639323735653730356565643938376564316337613333616664643865
|
||||
30633530383865663731383635366566613565393632336337306334663937396135303539363536
|
||||
65656331386532306438383031313164626237393762303636663831666239336637666465633630
|
||||
63383961336333663537666561376532376631343132363638633364616531306363623432396332
|
||||
31343864626335613131653762363338393738343531626330663663376531336230656365363661
|
||||
64363530333965386439376161313534333435393966656231613161633034303964653564363436
|
||||
63396131623562303962383137666439373132633330363237353031306561306362666561376435
|
||||
66383535333233663736363762323662363264393365643135323834366565653932313863353437
|
||||
38363536323739336165356561613335653464363637636234313637383066633865353239393961
|
||||
31613532373361383062393238303430353662383930373764373237383035633364323661623661
|
||||
62653661623234366666616532663563316234663137616338393036333937633963323635366165
|
||||
61613032363531666338353132346365323661626265626461396362646338373536306362303338
|
||||
31646564343434633433663039366533656665643235336136333134366231376336343465336539
|
||||
37363139346239666232313366613432646265626564313838346364306133306434313337626562
|
||||
32346430316532623037633838303363346130623636386134313566383333613565316135373264
|
||||
38393261313435323433326130303032393538333963646430646366653363613830613534353832
|
||||
32643434613964613239363966376138376661636335373130356430336631333461623734383535
|
||||
35356461326136636234663738343738303064386536336462303632303461303733333666383036
|
||||
38383563393739343134363038623638393766316533373163336439336238613734656132623163
|
||||
32626564666430663339316163393934323133306238353562323866633738363737643937376435
|
||||
66306430623334643766623564363239346361393666663766306637313265313833396435626234
|
||||
35366239616132623863343365663833363934316362636638303536656631633364646235366566
|
||||
39386637613735373761643339396132323031323438316633363464336636316534643435323861
|
||||
62663732366430646538626663313035616235643537643234356434323635373962343633336266
|
||||
30323031306438656537626430356231393466313334633934623632663661306361313330633339
|
||||
33396234386536336331616161363132343765306562303932313963623037366633383765663134
|
||||
62636466636433666530363930633765626531613539363832313361666565323635326334326533
|
||||
36336439363836663033643262336139613437623633616138656564393032393263393132303031
|
||||
31626336663039633362366663306662333432643261613464303939326562653562346239383261
|
||||
37373966383635626433363831653936333964626262363839383936356634633233336365623765
|
||||
62313031353538636464303234383865613932323164386336316362323731303263346637346534
|
||||
32353734343130343364646161643431353230336364366330363261326334613936633234646264
|
||||
39623163346663613431323630303034383761663835373565663166366239303130386139396236
|
||||
62393565353761636636366462353038366664663430616332326465373364396264323064666536
|
||||
61653035633562316634336263636332363733663666626131383161656535383133613939623034
|
||||
66363632613733306336323165336534346337386163656631343332323636353539353737316131
|
||||
34616336343865656565646338653037333838663736333330376330663834373138363739633064
|
||||
61646439636365323838376131663266636235333062616532613936616339633661303634323531
|
||||
63356335386534353639323566376134373565353137333134363761666532366431633634333430
|
||||
39326662633765626230326130386333393463323433366162363432613234336634663439313339
|
||||
33653737333830336264343839396563316462613032376335356634383436353962613636333866
|
||||
32333032623635323430356636623138366635386533646133626164316438393937626462313239
|
||||
31636635363064363763396566306234363965346438653738333961623435303233396634643763
|
||||
65343763386464366636383734336335646464306262623363303934393734333666636635356365
|
||||
65353931303862343765343665373139376530656263356365366333323135383838383030613166
|
||||
30303963383833643937636539656164336537383337346436303534303035313935323633663765
|
||||
333464336235373230316337333632326138
|
||||
61356165613635386631393135656434646436303665313031346566323336313138353433316463
|
||||
3363323534613064643132663335623238366431393062340a346538396662306537663163623366
|
||||
38626166383933616331623231373137306562623637313263333237633661663436666266616433
|
||||
3862346438643638650a306634333535653633613534646630386131333236366538333765323333
|
||||
38656163303837303732663561373232393132343331663164656262393730326434373731333636
|
||||
36613538646431396536363936336562616431656665653965373864633366663836353434626434
|
||||
34383932373461333564303439623565383661646365386665393831383463663662356536356236
|
||||
36373964666236626465366161636135393734356536633466383262326537343833636630343738
|
||||
62353066316131613737373162643363653662656261363465386364323962656537373061373032
|
||||
33343763353464383438363438343965653532393831343930393562633630383932653862623637
|
||||
38386239353237356631646436356166373961333464396639383538383662326534343339313330
|
||||
65373339356364636634616532363832386631323062363530313861336238353261353334306235
|
||||
62616338396431316537346638656365356564346666366366343638356261623664393263323937
|
||||
30343261363562383332323462336435376664386134646562643836363834313237373631353731
|
||||
35653535663864643266313332356635363262363533663232656531373130336539633066376139
|
||||
32316665393831663035623962656364363831333563366135636164346335383738363336663566
|
||||
63356532643563393939383635386462663561386434323939303431653438653131363538383034
|
||||
33623933333464363032656636643033326162626163353633343062633966343332383138363963
|
||||
36383831663562616533316436366566323061386535343538393861383462333166343562316633
|
||||
30623665623035393537393965626363323132656433313339396233356666346634316332616336
|
||||
37363635363330323230373332326565343530653335383437373230366563366237633665626331
|
||||
66623336626230663361636439316337393865383035326136653264666438666566646132353036
|
||||
38616264313833316536623238633339373466613866626366383835656638623863323838653030
|
||||
63303938376164653966356435386333363731656666313234663535666165646233313137343563
|
||||
36386437393139656438333262383437656666343831313239323961373637653163643664356565
|
||||
62346431393133656530316262303763646165643336396661666431383436323562336137653031
|
||||
62626538373839613734396366653065306534636630346338316237616161613037616364356431
|
||||
38643965376133616161336633383664326230383435363334353137303162663738313331346238
|
||||
39356161616533616134356231323530306338333162343363353531303263636632613036386638
|
||||
63633136386232306234323936303563646466313935326631396565383432386130656638616266
|
||||
32383334363237336539396665336366643764633131643663643137376438323666326435626461
|
||||
64346633636431393137633537306431646564386565303933636434386462346630626537346438
|
||||
32623333666133303061646564366366326665363163396262633164323631636337346130303239
|
||||
38373936663337356134666132303165393365663763396362623434633737373538653566646134
|
||||
62316164396438303532616266313062326666633130656338653139376634306664633031333037
|
||||
31636166306565353334633435656233336233363664306264626237623366336161303134353433
|
||||
63383739666462623336386537346662633666626466393039653439346436653937633537396436
|
||||
35393339383066326630353066623132333034656539363561346462626265363263303535343961
|
||||
39316461616630326539613731303039613736393633373338646266323938326162373831346336
|
||||
30656130343463366534323030646238313465306266383034623065636665623366333063383736
|
||||
38373063393837306462303564643962373334343139626338623935336435643730646532633630
|
||||
66363730386636633639346463363365343239373265303738353732653633653437636130636664
|
||||
39386365353334653765343335303263363461313965383664326563333734626533376436626530
|
||||
63353163323637303730353564383733653365613635353764333266393532653663326533646132
|
||||
39313763373735323835626437306435373238653432393936643165663663656665316132653330
|
||||
35383038353532656434366336346235363563636264303734633138323963396562306232646236
|
||||
38366536306561653937336333373434336164663336613839353439356435333833396363636437
|
||||
38323934613735643363656233333037336465336564313966623063376566663030303230323262
|
||||
65643564666534326234306164343365383632333061316238623565353538313538396364313337
|
||||
35336230313339643736653238386231343661623337306236383665356632366236356335323530
|
||||
66373831616239376231636361333430343433303233393066323865663434643433303832373262
|
||||
36366534646161323130393931626362626139663139643263366639656531613436313533363130
|
||||
65643736363833333939613566663339623964333262643863333030623138633464386238613934
|
||||
31323639333234336264313663376465323737353766643839303665313737336534386665363034
|
||||
30326439666433373232306136306365343764643434306561343339353132346430646436343362
|
||||
32333765616262363930353435616563333736313533653339656231316230346166363335356638
|
||||
37333561366339613136306438306130343230663732333862663838396463623661303961336433
|
||||
65646332373139303462303633346432366530643130366133363937653739653036366136373434
|
||||
343864373734666431326239373866313734
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- forgejo-runner
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# fulcrum-box: the Electrum server.
|
||||
#
|
||||
# Read by the fulcrum role here and by the socket-proxy play on the edge host,
|
||||
# which publishes the port. See host_vars/knots_box_local/main.yml for why this
|
||||
# lives in host_vars rather than in the role's defaults.
|
||||
fulcrum_ssl_port: 50002
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- fulcrum
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
# knots-box: Bitcoin Knots and the DATUM Gateway.
|
||||
#
|
||||
# These ports are read twice: by the role that deploys the service here, and by
|
||||
# the socket-proxy / Caddy plays that run on the EDGE host and publish them.
|
||||
# A role default is invisible to that second play, which is why these live in
|
||||
# host_vars rather than roles/<svc>/defaults/ - the edge play reads them as
|
||||
# hostvars['knots_box_local'].<name>, and the role picks them up automatically
|
||||
# because host_vars outranks role defaults.
|
||||
#
|
||||
# They used to live in services_config.yml, a file 30 plays had to remember to
|
||||
# name in vars_files: and that four role defaults silently depended on.
|
||||
bitcoin_p2p_port: 8333
|
||||
datum_gateway_api_port: 7152
|
||||
datum_gateway_stratum_port: 23334
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- bitcoind
|
||||
- datum-gateway
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- memos
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
# mempool-box: the Mempool block explorer.
|
||||
#
|
||||
# Read by the mempool role here and by the Caddy play on the edge host, which
|
||||
# proxies to it. See host_vars/knots_box_local/main.yml for why this lives in
|
||||
# host_vars rather than in the role's defaults.
|
||||
mempool_frontend_port: 8080
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- caddy
|
||||
|
|
@ -14,12 +14,7 @@ systemd_service_name: nodito-cpu-temp-monitor
|
|||
|
||||
# ZFS Pool Configuration
|
||||
zfs_pool_name: "proxmox-tank-1"
|
||||
# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the
|
||||
# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently
|
||||
# replaced and the repo never caught up. Pool creation is guarded by
|
||||
# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been
|
||||
# wrong on any disaster-recovery run.
|
||||
zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ" # First disk for RAID 1 mirror
|
||||
zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror
|
||||
zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror
|
||||
zfs_pool_mountpoint: "/var/lib/vz"
|
||||
|
||||
|
|
@ -31,15 +26,3 @@ ups_port: auto
|
|||
ups_user: counterweight
|
||||
ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power
|
||||
ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- nut-server
|
||||
- nut-monitor
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
$ANSIBLE_VAULT;1.1;AES256
|
||||
30333035323663393939343061323234336164396465623665346165393534646366333332376463
|
||||
3364373463333664363334373964323838336531353364310a636636373539623464336630666164
|
||||
61376532616339376562373238383436306664313564663266303534346461666466383965323538
|
||||
3163313239626663310a613033336332653165333537313366636361663036383031376561613761
|
||||
31313563373062333033323037653939663762343161656264633436343361663737626366663732
|
||||
39366666626338323436383134646263643538333564313566346336323563663534653161396136
|
||||
39333565393538366238643563323630346166643461643063393631643665363566623631373762
|
||||
36643866646637306231653837363838656163613766636265383139333838396535626335343163
|
||||
32613935353330636263616333666230323436663935326133636362343836323535623237646235
|
||||
6266316363366335323162663039366137633865396237373632
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
---
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- headscale
|
||||
- caddy
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
|
||||
# Systemd services deployed on this host, monitored every 5 minutes.
|
||||
#
|
||||
# The fact lives with the machine rather than in a central map, for the same
|
||||
# reason the cross-host ports do: "what runs here" is a property of the host,
|
||||
# and a central list is one more thing to forget to update when a service moves.
|
||||
#
|
||||
# Only units WE deploy belong here. Distro units (ssh, cron) have their own
|
||||
# supervision and would be noise.
|
||||
monitored_services:
|
||||
- forgejo
|
||||
- lnbits
|
||||
- caddy
|
||||
- phoenixd
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
- name: Secure Debian
|
||||
hosts: managed
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
- name: Secure Debian
|
||||
hosts: managed
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
|
|
|||
|
|
@ -1,216 +0,0 @@
|
|||
---
|
||||
# Host-level monitoring for the whole estate, reported to Gatus.
|
||||
#
|
||||
# Every check here PUSHES. Gatus never reaches out, which matters because nodito
|
||||
# and its VMs sit behind NAT, and because four of the five checks are internal
|
||||
# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool
|
||||
# health and UPS mains status cannot be observed from outside the machine.
|
||||
#
|
||||
# Liveness is a push too, and that is a choice rather than a limitation. A
|
||||
# heartbeat proves the host is running AND can reach the internet; an ICMP probe
|
||||
# from one vantage point only proves it answers pings from there. And because
|
||||
# Gatus alerts when a heartbeat window expires, a check that stops running
|
||||
# raises the alarm by itself - a dead timer looks exactly like a dead host,
|
||||
# which is the correct reading.
|
||||
#
|
||||
# Each host has ONE bearer token, shared across its own checks: a token can only
|
||||
# write results for that host's endpoints, so a compromised host can lie about
|
||||
# itself, which it could do anyway.
|
||||
#
|
||||
# The push URL must use Gatus's own key format (config/key/key.go):
|
||||
# key = sanitize(group) + "_" + sanitize(name)
|
||||
# where sanitize lowercases and replaces / _ . , space # + & with "-". So
|
||||
# knots_box_local becomes knots-box-local in the URL but stays readable in the
|
||||
# name. host_key below is the Jinja equivalent; do not hand-write these.
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Register everything with Gatus.
|
||||
#
|
||||
# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the
|
||||
# host plays below take minutes, so every endpoint exists before its first push
|
||||
# arrives. Registering afterwards would 404 every first report.
|
||||
#
|
||||
# Heartbeat windows are several times the check interval, so one missed run - a
|
||||
# slow apt run, a reboot - does not raise an alarm, but a check that has
|
||||
# genuinely stopped does.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Alerting thresholds, and why they differ by check type.
|
||||
#
|
||||
# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
|
||||
# different amount of wall-clock time per check:
|
||||
#
|
||||
# push/heartbeat endpoints a failure is produced once per heartbeat window
|
||||
# pulled endpoints a failure is produced once per interval
|
||||
#
|
||||
# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
|
||||
# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
|
||||
# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
|
||||
# threshold of 3 on top triples a tolerance that was already chosen.
|
||||
#
|
||||
# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
|
||||
# have no built-in tolerance, so the threshold is where it belongs for them.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
- name: Register the host checks with Gatus
|
||||
hosts: observability
|
||||
become: yes
|
||||
vars:
|
||||
monitored: "{{ groups['managed'] | sort }}"
|
||||
|
||||
tasks:
|
||||
- name: Build the liveness endpoint list
|
||||
ansible.builtin.set_fact:
|
||||
liveness_endpoints: "{{ liveness_endpoints | default([]) + [{
|
||||
'name': item,
|
||||
'group': 'liveness',
|
||||
'token': gatus_push_tokens[item],
|
||||
'heartbeat': '11m'}] }}"
|
||||
loop: "{{ monitored }}"
|
||||
|
||||
- name: Build the disk endpoint list
|
||||
ansible.builtin.set_fact:
|
||||
disk_endpoints: "{{ disk_endpoints | default([]) + [{
|
||||
'name': item,
|
||||
'group': 'disk',
|
||||
'token': gatus_push_tokens[item],
|
||||
'heartbeat': '7h'}] }}"
|
||||
loop: "{{ monitored }}"
|
||||
|
||||
- name: Register liveness endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
gatus_endpoint_default_alerts:
|
||||
- type: signal
|
||||
# 1, not 3: the heartbeat window is the tolerance. See the note above.
|
||||
failure-threshold: 1
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
minimum-reminder-interval: 6h
|
||||
gatus_endpoint_name: liveness
|
||||
gatus_endpoint_external: "{{ liveness_endpoints }}"
|
||||
|
||||
- name: Register disk endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
gatus_endpoint_default_alerts:
|
||||
- type: signal
|
||||
# 1, not 3: the heartbeat window is the tolerance. See the note above.
|
||||
failure-threshold: 1
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
minimum-reminder-interval: 6h
|
||||
gatus_endpoint_name: disk
|
||||
gatus_endpoint_external: "{{ disk_endpoints }}"
|
||||
|
||||
- name: Register the hypervisor endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
gatus_endpoint_default_alerts:
|
||||
- type: signal
|
||||
# 1, not 3: the heartbeat window is the tolerance. See the note above.
|
||||
failure-threshold: 1
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
minimum-reminder-interval: 6h
|
||||
gatus_endpoint_name: hypervisor
|
||||
gatus_endpoint_external:
|
||||
- name: cpu
|
||||
group: hypervisor
|
||||
token: "{{ gatus_push_tokens['nodito'] }}"
|
||||
heartbeat: "11m"
|
||||
- name: zfs
|
||||
group: hypervisor
|
||||
token: "{{ gatus_push_tokens['nodito'] }}"
|
||||
heartbeat: "7h"
|
||||
- name: ups
|
||||
group: hypervisor
|
||||
token: "{{ gatus_push_tokens['nodito'] }}"
|
||||
heartbeat: "11m"
|
||||
|
||||
- name: Deploy host liveness and disk checks
|
||||
hosts: managed
|
||||
become: yes
|
||||
vars:
|
||||
gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
|
||||
host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}"
|
||||
host_token: "{{ gatus_push_tokens[inventory_hostname] }}"
|
||||
|
||||
tasks:
|
||||
- name: Is the host up?
|
||||
ansible.builtin.include_role:
|
||||
name: healthcheck
|
||||
vars:
|
||||
healthcheck_name: liveness
|
||||
healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}"
|
||||
healthcheck_check: liveness
|
||||
healthcheck_interval: "5min"
|
||||
healthcheck_boot_delay: "1min"
|
||||
healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external"
|
||||
healthcheck_push_token: "{{ host_token }}"
|
||||
|
||||
- name: Is the disk packed?
|
||||
ansible.builtin.include_role:
|
||||
name: healthcheck
|
||||
vars:
|
||||
healthcheck_name: disk-usage
|
||||
healthcheck_description: "Disk usage for {{ inventory_hostname }}"
|
||||
healthcheck_check: disk-usage
|
||||
# Every 6h rather than daily. Disk usage itself moves slowly, but the
|
||||
# heartbeat can only be as tight as the push frequency - a daily push
|
||||
# forces a >24h window, and a stuck check then hides for a day and a
|
||||
# half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the
|
||||
# hosts so twelve boxes do not all report in the same second.
|
||||
healthcheck_on_calendar: "*-*-* 00/6:00:00"
|
||||
healthcheck_randomized_delay: "900"
|
||||
healthcheck_boot_delay: "5min"
|
||||
healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external"
|
||||
healthcheck_push_token: "{{ host_token }}"
|
||||
|
||||
- name: Deploy the hypervisor-only checks
|
||||
hosts: hypervisor
|
||||
become: yes
|
||||
vars:
|
||||
gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
|
||||
host_token: "{{ gatus_push_tokens[inventory_hostname] }}"
|
||||
|
||||
tasks:
|
||||
- name: Is the CPU hot?
|
||||
ansible.builtin.include_role:
|
||||
name: healthcheck
|
||||
vars:
|
||||
healthcheck_name: cpu-temp
|
||||
healthcheck_description: "CPU temperature for {{ inventory_hostname }}"
|
||||
healthcheck_check: cpu-temp
|
||||
healthcheck_packages: [curl, lm-sensors]
|
||||
healthcheck_interval: "5min"
|
||||
healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external"
|
||||
healthcheck_push_token: "{{ host_token }}"
|
||||
|
||||
- name: Is ZFS broken?
|
||||
ansible.builtin.include_role:
|
||||
name: healthcheck
|
||||
vars:
|
||||
healthcheck_name: zfs-health
|
||||
healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}"
|
||||
healthcheck_check: zfs-health
|
||||
healthcheck_packages: [curl, jq]
|
||||
healthcheck_zfs_pool: "{{ zfs_pool_name }}"
|
||||
healthcheck_on_calendar: "*-*-* 00/6:20:00"
|
||||
healthcheck_boot_delay: "10min"
|
||||
healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external"
|
||||
healthcheck_push_token: "{{ host_token }}"
|
||||
|
||||
- name: Is the UPS online?
|
||||
ansible.builtin.include_role:
|
||||
name: healthcheck
|
||||
vars:
|
||||
healthcheck_name: ups-status
|
||||
healthcheck_description: "UPS mains status for {{ ups_name }}"
|
||||
healthcheck_check: ups-status
|
||||
healthcheck_ups_name: "{{ ups_name }}"
|
||||
healthcheck_interval: "5min"
|
||||
healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external"
|
||||
healthcheck_push_token: "{{ host_token }}"
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
---
|
||||
# Is each systemd-deployed service actually running?
|
||||
#
|
||||
# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before
|
||||
# it alarms, so a reboot or a slow check does not page anyone, but a host that
|
||||
# stops reporting does.
|
||||
#
|
||||
# This closes the gap that let a real bug run unnoticed: a backup script left
|
||||
# forgejo, lnbits, headscale and memos stopped, and NOTHING caught it. The dumps
|
||||
# exited 0, the artefacts were correct, the deploy said failed=0, and liveness
|
||||
# only proves the HOST is up - not that anything on it is serving.
|
||||
#
|
||||
# One endpoint PER UNIT, not per host. A host running four services needs four
|
||||
# endpoints, or a single red light says "something on vipy is down" without
|
||||
# saying which - and that is the question you actually have at 3am. But only ONE
|
||||
# timer per host: the check iterates that host's units and pushes a result for
|
||||
# each, the same way check-backups.sh reports per source. Four units on vipy
|
||||
# would otherwise mean four scripts, four services and four timers.
|
||||
#
|
||||
# Which units each host runs is in host_vars/<host>/main.yml as
|
||||
# monitored_services, because "what runs here" is a property of the machine.
|
||||
#
|
||||
# Keys are host-qualified because unit names collide - caddy runs on four
|
||||
# machines. Gatus computes sanitize(group)_sanitize(name), so group "services"
|
||||
# and name "vipy/caddy" give services_vipy-caddy.
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the
|
||||
# host play above takes minutes, so every endpoint exists before its first push.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Alerting thresholds, and why they differ by check type.
|
||||
#
|
||||
# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
|
||||
# different amount of wall-clock time per check:
|
||||
#
|
||||
# push/heartbeat endpoints a failure is produced once per heartbeat window
|
||||
# pulled endpoints a failure is produced once per interval
|
||||
#
|
||||
# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
|
||||
# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
|
||||
# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
|
||||
# threshold of 3 on top triples a tolerance that was already chosen.
|
||||
#
|
||||
# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
|
||||
# have no built-in tolerance, so the threshold is where it belongs for them.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
- name: Register the service checks with Gatus
|
||||
hosts: observability
|
||||
become: yes
|
||||
|
||||
tasks:
|
||||
# Two plain steps rather than one clever expression: first collect which
|
||||
# units each host declares, then flatten that into endpoints.
|
||||
- name: Collect the units each host declares
|
||||
ansible.builtin.set_fact:
|
||||
host_units: "{{ host_units | default([]) + [{'host': item, 'units': hostvars[item].monitored_services}] }}"
|
||||
loop: "{{ groups['managed'] | sort }}"
|
||||
when: hostvars[item].monitored_services | default([]) | length > 0
|
||||
|
||||
- name: Build one endpoint per unit
|
||||
ansible.builtin.set_fact:
|
||||
service_endpoints: "{{ service_endpoints | default([]) + [{
|
||||
'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1,
|
||||
'group': 'services',
|
||||
'token': gatus_push_tokens[item.0.host],
|
||||
'heartbeat': '11m'}] }}"
|
||||
loop: "{{ host_units | subelements('units') }}"
|
||||
|
||||
- name: Register the service endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
gatus_endpoint_default_alerts:
|
||||
- type: signal
|
||||
# 1, not 3: the heartbeat window is the tolerance. See the note above.
|
||||
failure-threshold: 1
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
minimum-reminder-interval: 6h
|
||||
gatus_endpoint_name: services
|
||||
gatus_endpoint_external: "{{ service_endpoints }}"
|
||||
|
||||
- name: Monitor systemd services on every host that has them
|
||||
hosts: managed
|
||||
become: yes
|
||||
vars:
|
||||
gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
|
||||
host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}"
|
||||
|
||||
tasks:
|
||||
- name: Is every deployed service running?
|
||||
ansible.builtin.include_role:
|
||||
name: healthcheck
|
||||
vars:
|
||||
healthcheck_name: service-health
|
||||
healthcheck_description: "systemd services on {{ inventory_hostname }}"
|
||||
healthcheck_check: systemd-units
|
||||
healthcheck_units: "{{ monitored_services }}"
|
||||
healthcheck_units_key_prefix: "services_{{ host_key }}"
|
||||
healthcheck_interval: "5min"
|
||||
healthcheck_boot_delay: "2min"
|
||||
# The per-unit results go to keys under this collection; the role's own
|
||||
# single-result push is unused here, so only the base is set.
|
||||
healthcheck_push_base: "{{ gatus_api }}"
|
||||
healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"
|
||||
when: monitored_services | default([]) | length > 0
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
---
|
||||
# Domain expiry, DNS correctness, and public endpoint reachability.
|
||||
#
|
||||
# These are the first checks in the estate that PULL rather than push, and that
|
||||
# is the right way round for them: all three are about how the outside world
|
||||
# sees us, so they must be measured from outside. Gatus polls from the
|
||||
# observability host and needs nothing installed anywhere else - there is no
|
||||
# script, no timer and no token, because nothing is reporting in.
|
||||
#
|
||||
# That also means these have no heartbeat. A heartbeat answers "did the thing
|
||||
# that was supposed to report in do so"; when Gatus does the checking itself,
|
||||
# failure is immediate and self-evident.
|
||||
|
||||
- name: Register the public-facing checks with Gatus
|
||||
hosts: observability
|
||||
become: yes
|
||||
|
||||
vars:
|
||||
# Expected A records, derived from inventory rather than written down again.
|
||||
# The estate's recurring bug is an address recorded in a second place and
|
||||
# then left behind when the machine moved, so the check asserts against
|
||||
# ansible_host - if a box is renumbered, inventory is the one edit.
|
||||
dns_records:
|
||||
- {sub: "{{ subdomains.gatus }}", host: monitoring}
|
||||
- {sub: "{{ subdomains.headscale }}", host: spacey}
|
||||
- {sub: "{{ subdomains.vaultwarden }}", host: vipy}
|
||||
- {sub: "{{ subdomains.forgejo }}", host: vipy}
|
||||
- {sub: "{{ subdomains.lnbits }}", host: vipy}
|
||||
- {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy}
|
||||
- {sub: "{{ subdomains.personal_blog }}", host: vipy}
|
||||
- {sub: "{{ subdomains.memos }}", host: vipy}
|
||||
- {sub: "{{ subdomains.mempool }}", host: vipy}
|
||||
- {sub: "{{ subdomains.datum_gateway }}", host: vipy}
|
||||
|
||||
# A public resolver on purpose: this must test what the internet sees, not
|
||||
# what a local cache or the tailnet's MagicDNS happens to answer.
|
||||
dns_resolver: "1.1.1.1"
|
||||
|
||||
# Expected status per site, checked live before being written down.
|
||||
# 401 is the CORRECT answer for the two behind basic auth - asserting 200
|
||||
# there would go green precisely when the auth broke.
|
||||
public_sites:
|
||||
- {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401}
|
||||
- {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200}
|
||||
- {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200}
|
||||
- {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200}
|
||||
- {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200}
|
||||
- {name: avisame, sub: "{{ subdomains.ntfy_emergency_app }}", path: "/", status: 200}
|
||||
- {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200}
|
||||
- {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200}
|
||||
- {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200}
|
||||
- {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401}
|
||||
|
||||
# Ports published from the edge host by socket_proxy.
|
||||
public_tcp:
|
||||
- {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"}
|
||||
- {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"}
|
||||
- {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"}
|
||||
|
||||
tasks:
|
||||
# ── Domain expiry ────────────────────────────────────────────────────────
|
||||
# Each domain needs a URL SCHEME: Gatus derives the endpoint type from the
|
||||
# prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole
|
||||
# config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so
|
||||
# whatever the apex serves - a real site, or the registrar's parking page -
|
||||
# is irrelevant.
|
||||
#
|
||||
# 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway
|
||||
# because it uses a free whois service that must not be hammered.
|
||||
# 336h = 14 days of runway, because renewal is a manual act at the registrar.
|
||||
- name: Build the domain endpoints
|
||||
ansible.builtin.set_fact:
|
||||
domain_endpoints: "{{ domain_endpoints | default([]) + [{
|
||||
'name': item,
|
||||
'group': 'domain',
|
||||
'url': 'https://' ~ item,
|
||||
'interval': '24h',
|
||||
'conditions': ['[DOMAIN_EXPIRATION] > 336h'],
|
||||
'alerts': [{'type': 'signal', 'failure-threshold': 1,
|
||||
'success-threshold': 1, 'send-on-resolved': true,
|
||||
'minimum-reminder-interval': '168h'}]}] }}"
|
||||
loop: "{{ monitored_domains }}"
|
||||
|
||||
# ── DNS ──────────────────────────────────────────────────────────────────
|
||||
# 6h, not daily: a DNS query is cheap and a wrong record is an outage. The
|
||||
# domain check stays at 24h because it does a WHOIS/RDAP lookup against a
|
||||
# free service. Alert on the first failure - at a 6h interval, waiting for
|
||||
# three would be nearly a day.
|
||||
- name: Build the DNS endpoints
|
||||
ansible.builtin.set_fact:
|
||||
dns_endpoints: "{{ dns_endpoints | default([]) + [{
|
||||
'name': item.sub ~ '.' ~ root_domain,
|
||||
'group': 'dns',
|
||||
'url': dns_resolver,
|
||||
'interval': '6h',
|
||||
'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain},
|
||||
'conditions': ['[DNS_RCODE] == NOERROR',
|
||||
'[BODY] == ' ~ hostvars[item.host].ansible_host],
|
||||
'alerts': [{'type': 'signal', 'failure-threshold': 1,
|
||||
'success-threshold': 1, 'send-on-resolved': true,
|
||||
'minimum-reminder-interval': '24h'}]}] }}"
|
||||
loop: "{{ dns_records }}"
|
||||
|
||||
# ── Public HTTP ──────────────────────────────────────────────────────────
|
||||
# failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has
|
||||
# no heartbeat window, so unlike the push checks the tolerance has to live
|
||||
# in the threshold - and one failed poll of a public site is usually a blip.
|
||||
- name: Build the public HTTP endpoints
|
||||
ansible.builtin.set_fact:
|
||||
http_endpoints: "{{ http_endpoints | default([]) + [{
|
||||
'name': item.name,
|
||||
'group': 'public',
|
||||
'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path,
|
||||
'interval': '5m',
|
||||
'conditions': ['[STATUS] == ' ~ item.status,
|
||||
'[CERTIFICATE_EXPIRATION] > 168h'],
|
||||
'alerts': [{'type': 'signal', 'failure-threshold': 3,
|
||||
'success-threshold': 2, 'send-on-resolved': true,
|
||||
'minimum-reminder-interval': '6h'}]}] }}"
|
||||
loop: "{{ public_sites }}"
|
||||
|
||||
# ── Public TCP ───────────────────────────────────────────────────────────
|
||||
- name: Build the public TCP endpoints
|
||||
ansible.builtin.set_fact:
|
||||
tcp_endpoints: "{{ tcp_endpoints | default([]) + [{
|
||||
'name': item.name,
|
||||
'group': 'public',
|
||||
'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port,
|
||||
'interval': '5m',
|
||||
'conditions': ['[CONNECTED] == true'],
|
||||
'alerts': [{'type': 'signal', 'failure-threshold': 3,
|
||||
'success-threshold': 2, 'send-on-resolved': true,
|
||||
'minimum-reminder-interval': '6h'}]}] }}"
|
||||
loop: "{{ public_tcp }}"
|
||||
|
||||
- name: Register the public-facing endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
gatus_endpoint_name: public
|
||||
gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}"
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
---
|
||||
# The per-service health probes.
|
||||
#
|
||||
# These are NOT the same thing as the systemd checks in infra/401. Those answer
|
||||
# "is the unit running"; these answer "does the service actually respond" - an
|
||||
# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch
|
||||
# from the Mempool backend. A process can be perfectly `active` and useless,
|
||||
# which is precisely the gap these close.
|
||||
#
|
||||
# The checks themselves live in each service's own role, deployed by that
|
||||
# service's playbook. This play only registers where they report, because the
|
||||
# endpoints must exist in Gatus before the first push arrives.
|
||||
#
|
||||
# They used to push to Uptime Kuma. The scripts now POST with a bearer token
|
||||
# instead of GETting ?status=up, and each host uses its own token.
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Alerting thresholds, and why they differ by check type.
|
||||
#
|
||||
# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a
|
||||
# different amount of wall-clock time per check:
|
||||
#
|
||||
# push/heartbeat endpoints a failure is produced once per heartbeat window
|
||||
# pulled endpoints a failure is produced once per interval
|
||||
#
|
||||
# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day
|
||||
# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m
|
||||
# window on a 5-minute push is precisely "one missed push forgiven"; stacking a
|
||||
# threshold of 3 on top triples a tolerance that was already chosen.
|
||||
#
|
||||
# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints
|
||||
# have no built-in tolerance, so the threshold is where it belongs for them.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
- name: Register the per-service probes with Gatus
|
||||
hosts: observability
|
||||
become: yes
|
||||
|
||||
vars:
|
||||
probes:
|
||||
- {name: bitcoin-knots, host: knots_box_local}
|
||||
- {name: datum-gateway, host: knots_box_local}
|
||||
- {name: fulcrum, host: fulcrum_box_local}
|
||||
- {name: phoenixd, host: vipy}
|
||||
- {name: forgejo-runner, host: forgejo_runner_local}
|
||||
- {name: mempool-mariadb, host: mempool_box_local}
|
||||
- {name: mempool-backend, host: mempool_box_local}
|
||||
- {name: mempool-frontend, host: mempool_box_local}
|
||||
|
||||
tasks:
|
||||
- name: Build the probe endpoint list
|
||||
ansible.builtin.set_fact:
|
||||
probe_endpoints: "{{ probe_endpoints | default([]) + [{
|
||||
'name': item.name,
|
||||
'group': 'probe',
|
||||
'token': gatus_push_tokens[item.host],
|
||||
'heartbeat': '11m'}] }}"
|
||||
loop: "{{ probes }}"
|
||||
|
||||
- name: Register the probe endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
gatus_endpoint_default_alerts:
|
||||
- type: signal
|
||||
# 1, not 3: the heartbeat window is the tolerance. See the note above.
|
||||
failure-threshold: 1
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
minimum-reminder-interval: 6h
|
||||
gatus_endpoint_name: probes
|
||||
gatus_endpoint_external: "{{ probe_endpoints }}"
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
---
|
||||
# Remove the Uptime-Kuma-era monitoring that 400/401/402 replaced.
|
||||
#
|
||||
# Deleting the playbooks that installed these is NOT enough: the units are on
|
||||
# the hosts, enabled, and keep firing regardless of what the repo says. Two of
|
||||
# them still push to https://uptime.contrapeso.xyz every 15 minutes. A playbook
|
||||
# that is deleted without a cleanup leaves its output running forever, with
|
||||
# nothing in the repo left to explain it.
|
||||
#
|
||||
# What replaced what, all verified against the deployed scripts before removal:
|
||||
#
|
||||
# disk-usage-monitor -> disk-usage-healthcheck (infra/400)
|
||||
# The old one checked ONLY "/" at 80%. The replacement walks every real
|
||||
# filesystem, excluding tmpfs/devtmpfs/squashfs/overlay, at 85%. Strictly
|
||||
# more coverage, so nothing is lost.
|
||||
#
|
||||
# system-healthcheck -> liveness-healthcheck (infra/400)
|
||||
# The old script computed uptime and pushed. That is exactly a liveness
|
||||
# heartbeat and nothing more.
|
||||
#
|
||||
# nodito-cpu-temp-monitor -> cpu-temp-healthcheck (infra/400)
|
||||
# zfs-health-monitor -> zfs-health-healthcheck (infra/400)
|
||||
# The ZFS check logic was ported verbatim - same five conditions - so only
|
||||
# the reporting transport changed.
|
||||
#
|
||||
# NOT removed, because they are not monitoring:
|
||||
# zfs-monthly-scrub.{timer,service} the actual scrub (infra/nodito/32)
|
||||
# pull-backups, check-backups the backup machinery (playbooks/backups)
|
||||
#
|
||||
# This play is idempotent and kept permanently rather than run once and deleted:
|
||||
# on a host that never had these it does nothing, and it guarantees a rebuilt or
|
||||
# restored machine cannot quietly bring them back.
|
||||
|
||||
- name: Remove the legacy Uptime Kuma monitoring units
|
||||
hosts: managed
|
||||
become: yes
|
||||
|
||||
vars:
|
||||
legacy_units:
|
||||
- disk-usage-monitor
|
||||
- system-healthcheck
|
||||
- nodito-cpu-temp-monitor
|
||||
- zfs-health-monitor
|
||||
legacy_dirs:
|
||||
- /opt/disk-monitoring
|
||||
- /opt/system-healthcheck
|
||||
- /opt/nodito-monitoring
|
||||
- /opt/zfs-monitoring
|
||||
|
||||
tasks:
|
||||
- name: Find which legacy units exist here
|
||||
ansible.builtin.stat:
|
||||
path: "/etc/systemd/system/{{ item.0 }}.{{ item.1 }}"
|
||||
register: legacy_unit_files
|
||||
loop: "{{ legacy_units | product(['timer', 'service']) | list }}"
|
||||
|
||||
# Stop and disable BEFORE deleting the unit file: systemd cannot disable a
|
||||
# unit whose file has already gone, which would leave a dangling symlink in
|
||||
# multi-user.target.wants and a warning on every daemon-reload.
|
||||
- name: Stop and disable the legacy units
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ item.item.0 }}.{{ item.item.1 }}"
|
||||
state: stopped
|
||||
enabled: no
|
||||
loop: "{{ legacy_unit_files.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.0 }}.{{ item.item.1 }}"
|
||||
when: item.stat.exists
|
||||
failed_when: false
|
||||
|
||||
- name: Remove the legacy unit files
|
||||
ansible.builtin.file:
|
||||
path: "/etc/systemd/system/{{ item.item.0 }}.{{ item.item.1 }}"
|
||||
state: absent
|
||||
loop: "{{ legacy_unit_files.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.0 }}.{{ item.item.1 }}"
|
||||
when: item.stat.exists
|
||||
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Remove the legacy monitoring scripts and their logs
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: absent
|
||||
loop: "{{ legacy_dirs }}"
|
||||
|
||||
# An orphan predating all of this: mode 0644, not executable, referenced by
|
||||
# no unit and no cron entry, pushing to a Kuma monitor. Superseded by
|
||||
# ups-status-healthcheck.
|
||||
- name: Remove the orphaned hand-written UPS heartbeat
|
||||
ansible.builtin.file:
|
||||
path: /usr/local/bin/ups-heartbeat.sh
|
||||
state: absent
|
||||
|
||||
- name: Confirm nothing still pushes to Uptime Kuma
|
||||
ansible.builtin.shell: >-
|
||||
grep -rl "uptime.contrapeso.xyz" /etc/systemd/system /usr/local/bin /opt 2>/dev/null || true
|
||||
register: kuma_refs
|
||||
changed_when: false
|
||||
|
||||
- name: Report any remaining references
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ 'clean - nothing references Uptime Kuma'
|
||||
if kuma_refs.stdout | trim | length == 0
|
||||
else 'STILL REFERENCING KUMA: ' ~ kuma_refs.stdout_lines | join(', ') }}
|
||||
343
ansible/infra/410_disk_usage_alerts.yml
Normal file
343
ansible/infra/410_disk_usage_alerts.yml
Normal file
|
|
@ -0,0 +1,343 @@
|
|||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
#
|
||||
# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
|
||||
# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
|
||||
# Kuma configuration" assert stops it before anything is installed or changed.
|
||||
#
|
||||
# It is kept because the CHECK LOGIC is the durable part — what gets measured,
|
||||
# the thresholds, and the systemd timer plumbing. When something replaces Uptime
|
||||
# Kuma, only the push transport needs rewriting; the rest still applies.
|
||||
#
|
||||
# What was being monitored: archive/uptime_kuma/MONITORS.md
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
- name: Deploy Disk Usage Monitoring
|
||||
hosts: managed
|
||||
become: yes
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
- ../services_config.yml
|
||||
- ../infra_secrets.yml
|
||||
|
||||
vars:
|
||||
disk_usage_threshold_percent: 80
|
||||
disk_check_interval_minutes: 15
|
||||
monitored_mount_point: "/"
|
||||
monitoring_script_dir: /opt/disk-monitoring
|
||||
monitoring_script_path: "{{ monitoring_script_dir }}/disk_usage_monitor.sh"
|
||||
log_file: "{{ monitoring_script_dir }}/disk_usage_monitor.log"
|
||||
systemd_service_name: disk-usage-monitor
|
||||
# Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml)
|
||||
uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
|
||||
ntfy_topic: "{{ service_settings.ntfy.topic }}"
|
||||
|
||||
tasks:
|
||||
- name: Validate Uptime Kuma configuration
|
||||
assert:
|
||||
that:
|
||||
- uptime_kuma_api_url is defined
|
||||
- uptime_kuma_api_url != ""
|
||||
- uptime_kuma_username is defined
|
||||
- uptime_kuma_username != ""
|
||||
- uptime_kuma_password is defined
|
||||
- uptime_kuma_password != ""
|
||||
fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
|
||||
|
||||
- name: Get hostname for monitor identification
|
||||
command: hostname
|
||||
register: host_name
|
||||
changed_when: false
|
||||
|
||||
- name: Set monitor name and group based on hostname and mount point
|
||||
set_fact:
|
||||
monitor_name: "disk-usage-{{ host_name.stdout }}-{{ monitored_mount_point | replace('/', 'root') }}"
|
||||
monitor_friendly_name: "Disk Usage: {{ host_name.stdout }} ({{ monitored_mount_point }})"
|
||||
uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
|
||||
|
||||
- name: Create Uptime Kuma monitor setup script
|
||||
copy:
|
||||
dest: /tmp/setup_uptime_kuma_monitor.py
|
||||
content: |
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
from uptime_kuma_api import UptimeKumaApi
|
||||
|
||||
def main():
|
||||
api_url = sys.argv[1]
|
||||
username = sys.argv[2]
|
||||
password = sys.argv[3]
|
||||
group_name = sys.argv[4]
|
||||
monitor_name = sys.argv[5]
|
||||
monitor_description = sys.argv[6]
|
||||
interval = int(sys.argv[7])
|
||||
ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts"
|
||||
|
||||
api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0)
|
||||
api.login(username, password)
|
||||
|
||||
# Get all monitors
|
||||
monitors = api.get_monitors()
|
||||
|
||||
# Get all notifications and find ntfy notification
|
||||
notifications = api.get_notifications()
|
||||
ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
|
||||
notification_id_list = {}
|
||||
if ntfy_notification:
|
||||
notification_id_list[ntfy_notification['id']] = True
|
||||
|
||||
# Find or create group
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
if not group:
|
||||
group_result = api.add_monitor(type='group', name=group_name)
|
||||
# Refresh to get the full group object with id
|
||||
monitors = api.get_monitors()
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
|
||||
# Find or create/update push monitor
|
||||
existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
monitor_data = {
|
||||
'type': 'push',
|
||||
'name': monitor_name,
|
||||
'parent': group['id'],
|
||||
'interval': interval,
|
||||
'upsideDown': True,
|
||||
'description': monitor_description,
|
||||
'notificationIDList': notification_id_list
|
||||
}
|
||||
|
||||
if existing_monitor:
|
||||
monitor = api.edit_monitor(existing_monitor['id'], **monitor_data)
|
||||
# Refresh to get the full monitor object with pushToken
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
else:
|
||||
monitor_result = api.add_monitor(**monitor_data)
|
||||
# Refresh to get the full monitor object with pushToken
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
# Output result as JSON
|
||||
result = {
|
||||
'monitor_id': monitor['id'],
|
||||
'push_token': monitor['pushToken'],
|
||||
'group_name': group_name,
|
||||
'group_id': group['id'],
|
||||
'monitor_name': monitor_name
|
||||
}
|
||||
print(json.dumps(result))
|
||||
|
||||
api.disconnect()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
- name: Run Uptime Kuma monitor setup script
|
||||
command: >
|
||||
{{ ansible_playbook_python }}
|
||||
/tmp/setup_uptime_kuma_monitor.py
|
||||
"{{ uptime_kuma_api_url }}"
|
||||
"{{ uptime_kuma_username }}"
|
||||
"{{ uptime_kuma_password }}"
|
||||
"{{ uptime_kuma_monitor_group }}"
|
||||
"{{ monitor_name }}"
|
||||
"{{ monitor_friendly_name }} - Alerts when usage exceeds {{ disk_usage_threshold_percent }}%"
|
||||
"{{ (disk_check_interval_minutes * 60) + 60 }}"
|
||||
"{{ ntfy_topic }}"
|
||||
register: monitor_setup_result
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
changed_when: false
|
||||
|
||||
- name: Parse monitor setup result
|
||||
set_fact:
|
||||
monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
|
||||
|
||||
- name: Set push URL and monitor ID as facts
|
||||
set_fact:
|
||||
uptime_kuma_disk_usage_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
|
||||
uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
|
||||
|
||||
- name: Install required packages for disk monitoring
|
||||
package:
|
||||
name:
|
||||
- curl
|
||||
state: present
|
||||
|
||||
- name: Create monitoring script directory
|
||||
file:
|
||||
path: "{{ monitoring_script_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create disk usage monitoring script
|
||||
copy:
|
||||
dest: "{{ monitoring_script_path }}"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
|
||||
# Disk Usage Monitoring Script
|
||||
# Monitors disk usage and sends alerts to Uptime Kuma
|
||||
# Mode: "No news is good news" - only sends alerts when disk usage is HIGH
|
||||
|
||||
LOG_FILE="{{ log_file }}"
|
||||
USAGE_THRESHOLD="{{ disk_usage_threshold_percent }}"
|
||||
UPTIME_KUMA_URL="{{ uptime_kuma_disk_usage_push_url }}"
|
||||
MOUNT_POINT="{{ monitored_mount_point }}"
|
||||
|
||||
# Function to log messages
|
||||
log_message() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to get disk usage percentage
|
||||
get_disk_usage() {
|
||||
local mount_point="$1"
|
||||
local usage=""
|
||||
|
||||
# Get disk usage percentage (without % sign)
|
||||
usage=$(df -h "$mount_point" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
|
||||
if [ -z "$usage" ]; then
|
||||
log_message "ERROR: Could not read disk usage for $mount_point"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$usage"
|
||||
}
|
||||
|
||||
# Function to get disk usage details
|
||||
get_disk_details() {
|
||||
local mount_point="$1"
|
||||
df -h "$mount_point" 2>/dev/null | awk 'NR==2 {print "Used: "$3" / Total: "$2" ("$5" full)"}'
|
||||
}
|
||||
|
||||
# Function to send alert to Uptime Kuma when disk usage exceeds threshold
|
||||
# With upside-down mode enabled, sending status=up will trigger an alert
|
||||
send_uptime_kuma_alert() {
|
||||
local usage="$1"
|
||||
local details="$2"
|
||||
local message="DISK FULL WARNING: ${MOUNT_POINT} is ${usage}% full (Threshold: ${USAGE_THRESHOLD}%) - ${details}"
|
||||
|
||||
log_message "ALERT: $message"
|
||||
|
||||
# Send push notification to Uptime Kuma with status=up
|
||||
# In upside-down mode, status=up is treated as down/alert
|
||||
response=$(curl -s -w "\n%{http_code}" -G \
|
||||
--data-urlencode "status=up" \
|
||||
--data-urlencode "msg=$message" \
|
||||
"$UPTIME_KUMA_URL" 2>&1)
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)"
|
||||
else
|
||||
log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main monitoring logic
|
||||
main() {
|
||||
log_message "Starting disk usage check for $MOUNT_POINT"
|
||||
|
||||
# Get current disk usage
|
||||
current_usage=$(get_disk_usage "$MOUNT_POINT")
|
||||
|
||||
if [ $? -ne 0 ] || [ -z "$current_usage" ]; then
|
||||
log_message "ERROR: Could not read disk usage"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get disk details
|
||||
disk_details=$(get_disk_details "$MOUNT_POINT")
|
||||
|
||||
log_message "Current disk usage: ${current_usage}% - $disk_details"
|
||||
|
||||
# Check if usage exceeds threshold
|
||||
if [ "$current_usage" -gt "$USAGE_THRESHOLD" ]; then
|
||||
log_message "WARNING: Disk usage ${current_usage}% exceeds threshold ${USAGE_THRESHOLD}%"
|
||||
send_uptime_kuma_alert "$current_usage" "$disk_details"
|
||||
else
|
||||
log_message "Disk usage is within normal range - no alert needed (no news is good news)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create systemd service for disk usage monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ systemd_service_name }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Disk Usage Monitor
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ monitoring_script_path }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd timer for disk usage monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ systemd_service_name }}.timer"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Run Disk Usage Monitor every {{ disk_check_interval_minutes }} minute(s)
|
||||
Requires={{ systemd_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec={{ disk_check_interval_minutes }}min
|
||||
OnUnitActiveSec={{ disk_check_interval_minutes }}min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start disk usage monitoring timer
|
||||
systemd:
|
||||
name: "{{ systemd_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Test disk usage monitoring script
|
||||
command: "{{ monitoring_script_path }}"
|
||||
register: script_test
|
||||
changed_when: false
|
||||
|
||||
- name: Verify script execution
|
||||
assert:
|
||||
that:
|
||||
- script_test.rc == 0
|
||||
fail_msg: "Disk usage monitoring script failed to execute properly"
|
||||
|
||||
- name: Clean up temporary Uptime Kuma setup script
|
||||
file:
|
||||
path: /tmp/setup_uptime_kuma_monitor.py
|
||||
state: absent
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
325
ansible/infra/420_system_healthcheck.yml
Normal file
325
ansible/infra/420_system_healthcheck.yml
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
#
|
||||
# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
|
||||
# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
|
||||
# Kuma configuration" assert stops it before anything is installed or changed.
|
||||
#
|
||||
# It is kept because the CHECK LOGIC is the durable part — what gets measured,
|
||||
# the thresholds, and the systemd timer plumbing. When something replaces Uptime
|
||||
# Kuma, only the push transport needs rewriting; the rest still applies.
|
||||
#
|
||||
# What was being monitored: archive/uptime_kuma/MONITORS.md
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
- name: Deploy System Healthcheck Monitoring
|
||||
hosts: managed
|
||||
become: yes
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
- ../services_config.yml
|
||||
- ../infra_secrets.yml
|
||||
|
||||
vars:
|
||||
healthcheck_interval_seconds: 60 # Send healthcheck every 60 seconds (1 minute)
|
||||
healthcheck_timeout_seconds: 90 # Uptime Kuma should alert if no ping received within 90s
|
||||
healthcheck_retries: 1 # Number of retries before alerting
|
||||
monitoring_script_dir: /opt/system-healthcheck
|
||||
monitoring_script_path: "{{ monitoring_script_dir }}/system_healthcheck.sh"
|
||||
log_file: "{{ monitoring_script_dir }}/system_healthcheck.log"
|
||||
systemd_service_name: system-healthcheck
|
||||
# Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml)
|
||||
uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
|
||||
ntfy_topic: "{{ service_settings.ntfy.topic }}"
|
||||
|
||||
tasks:
|
||||
- name: Validate Uptime Kuma configuration
|
||||
assert:
|
||||
that:
|
||||
- uptime_kuma_api_url is defined
|
||||
- uptime_kuma_api_url != ""
|
||||
- uptime_kuma_username is defined
|
||||
- uptime_kuma_username != ""
|
||||
- uptime_kuma_password is defined
|
||||
- uptime_kuma_password != ""
|
||||
fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
|
||||
|
||||
- name: Get hostname for monitor identification
|
||||
command: hostname
|
||||
register: host_name
|
||||
changed_when: false
|
||||
|
||||
- name: Set monitor name and group based on hostname
|
||||
set_fact:
|
||||
monitor_name: "system-healthcheck-{{ host_name.stdout }}"
|
||||
monitor_friendly_name: "System Healthcheck: {{ host_name.stdout }}"
|
||||
uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
|
||||
|
||||
- name: Create Uptime Kuma monitor setup script
|
||||
copy:
|
||||
dest: /tmp/setup_uptime_kuma_healthcheck_monitor.py
|
||||
content: |
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
from uptime_kuma_api import UptimeKumaApi
|
||||
|
||||
def main():
|
||||
api_url = sys.argv[1]
|
||||
username = sys.argv[2]
|
||||
password = sys.argv[3]
|
||||
group_name = sys.argv[4]
|
||||
monitor_name = sys.argv[5]
|
||||
monitor_description = sys.argv[6]
|
||||
interval = int(sys.argv[7])
|
||||
retries = int(sys.argv[8])
|
||||
ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts"
|
||||
|
||||
api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0)
|
||||
api.login(username, password)
|
||||
|
||||
# Get all monitors
|
||||
monitors = api.get_monitors()
|
||||
|
||||
# Get all notifications and find ntfy notification
|
||||
notifications = api.get_notifications()
|
||||
ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
|
||||
notification_id_list = {}
|
||||
if ntfy_notification:
|
||||
notification_id_list[ntfy_notification['id']] = True
|
||||
|
||||
# Find or create group
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
if not group:
|
||||
group_result = api.add_monitor(type='group', name=group_name)
|
||||
# Refresh to get the full group object with id
|
||||
monitors = api.get_monitors()
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
|
||||
# Find or create/update push monitor
|
||||
existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
monitor_data = {
|
||||
'type': 'push',
|
||||
'name': monitor_name,
|
||||
'parent': group['id'],
|
||||
'interval': interval,
|
||||
'upsideDown': False, # Normal mode: receiving pings = healthy
|
||||
'maxretries': retries,
|
||||
'description': monitor_description,
|
||||
'notificationIDList': notification_id_list
|
||||
}
|
||||
|
||||
if existing_monitor:
|
||||
monitor = api.edit_monitor(existing_monitor['id'], **monitor_data)
|
||||
# Refresh to get the full monitor object with pushToken
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
else:
|
||||
monitor_result = api.add_monitor(**monitor_data)
|
||||
# Refresh to get the full monitor object with pushToken
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
# Output result as JSON
|
||||
result = {
|
||||
'monitor_id': monitor['id'],
|
||||
'push_token': monitor['pushToken'],
|
||||
'group_name': group_name,
|
||||
'group_id': group['id'],
|
||||
'monitor_name': monitor_name
|
||||
}
|
||||
print(json.dumps(result))
|
||||
|
||||
api.disconnect()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
- name: Run Uptime Kuma monitor setup script
|
||||
command: >
|
||||
{{ ansible_playbook_python }}
|
||||
/tmp/setup_uptime_kuma_healthcheck_monitor.py
|
||||
"{{ uptime_kuma_api_url }}"
|
||||
"{{ uptime_kuma_username }}"
|
||||
"{{ uptime_kuma_password }}"
|
||||
"{{ uptime_kuma_monitor_group }}"
|
||||
"{{ monitor_name }}"
|
||||
"{{ monitor_friendly_name }} - Regular healthcheck ping every {{ healthcheck_interval_seconds }}s"
|
||||
"{{ healthcheck_timeout_seconds }}"
|
||||
"{{ healthcheck_retries }}"
|
||||
"{{ ntfy_topic }}"
|
||||
register: monitor_setup_result
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
changed_when: false
|
||||
|
||||
- name: Parse monitor setup result
|
||||
set_fact:
|
||||
monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
|
||||
|
||||
- name: Set push URL and monitor ID as facts
|
||||
set_fact:
|
||||
uptime_kuma_healthcheck_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
|
||||
uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
|
||||
|
||||
- name: Install required packages for healthcheck monitoring
|
||||
package:
|
||||
name:
|
||||
- curl
|
||||
state: present
|
||||
|
||||
- name: Create monitoring script directory
|
||||
file:
|
||||
path: "{{ monitoring_script_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create system healthcheck script
|
||||
copy:
|
||||
dest: "{{ monitoring_script_path }}"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
|
||||
# System Healthcheck Script
|
||||
# Sends regular heartbeat pings to Uptime Kuma
|
||||
# This ensures the system is running and able to communicate
|
||||
|
||||
LOG_FILE="{{ log_file }}"
|
||||
UPTIME_KUMA_URL="{{ uptime_kuma_healthcheck_push_url }}"
|
||||
HOSTNAME=$(hostname)
|
||||
|
||||
# Function to log messages
|
||||
log_message() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to send healthcheck ping to Uptime Kuma
|
||||
send_healthcheck() {
|
||||
local uptime_seconds=$(awk '{print int($1)}' /proc/uptime)
|
||||
local uptime_days=$((uptime_seconds / 86400))
|
||||
local uptime_hours=$(((uptime_seconds % 86400) / 3600))
|
||||
local uptime_minutes=$(((uptime_seconds % 3600) / 60))
|
||||
|
||||
local message="System healthy - Uptime: ${uptime_days}d ${uptime_hours}h ${uptime_minutes}m"
|
||||
|
||||
log_message "Sending healthcheck ping: $message"
|
||||
|
||||
# Send push notification to Uptime Kuma with status=up
|
||||
encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g')
|
||||
response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
log_message "Healthcheck ping sent successfully (HTTP $http_code)"
|
||||
else
|
||||
log_message "ERROR: Failed to send healthcheck ping (HTTP $http_code)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main healthcheck logic
|
||||
main() {
|
||||
log_message "Starting system healthcheck for $HOSTNAME"
|
||||
|
||||
# Send healthcheck ping
|
||||
if send_healthcheck; then
|
||||
log_message "Healthcheck completed successfully"
|
||||
else
|
||||
log_message "ERROR: Healthcheck failed"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create systemd service for system healthcheck
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ systemd_service_name }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=System Healthcheck Monitor
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ monitoring_script_path }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd timer for system healthcheck
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ systemd_service_name }}.timer"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Run System Healthcheck every minute
|
||||
Requires={{ systemd_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30sec
|
||||
OnUnitActiveSec={{ healthcheck_interval_seconds }}sec
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start system healthcheck timer
|
||||
systemd:
|
||||
name: "{{ systemd_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Test system healthcheck script
|
||||
command: "{{ monitoring_script_path }}"
|
||||
register: script_test
|
||||
changed_when: false
|
||||
|
||||
- name: Verify script execution
|
||||
assert:
|
||||
that:
|
||||
- script_test.rc == 0
|
||||
fail_msg: "System healthcheck script failed to execute properly"
|
||||
|
||||
- name: Display monitor information
|
||||
debug:
|
||||
msg: |
|
||||
✓ System healthcheck monitoring deployed successfully!
|
||||
|
||||
Monitor Name: {{ monitor_friendly_name }}
|
||||
Monitor Group: {{ uptime_kuma_monitor_group }}
|
||||
Healthcheck Interval: Every {{ healthcheck_interval_seconds }} seconds (1 minute)
|
||||
Timeout: {{ healthcheck_timeout_seconds }} seconds (90s)
|
||||
Retries: {{ healthcheck_retries }}
|
||||
|
||||
The system will send a heartbeat ping every minute.
|
||||
Uptime Kuma will alert if no ping is received within 90 seconds (with 1 retry).
|
||||
|
||||
- name: Clean up temporary Uptime Kuma setup script
|
||||
file:
|
||||
path: /tmp/setup_uptime_kuma_healthcheck_monitor.py
|
||||
state: absent
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
329
ansible/infra/430_cpu_temp_alerts.yml
Normal file
329
ansible/infra/430_cpu_temp_alerts.yml
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
#
|
||||
# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
|
||||
# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
|
||||
# Kuma configuration" assert stops it before anything is installed or changed.
|
||||
#
|
||||
# It is kept because the CHECK LOGIC is the durable part — what gets measured,
|
||||
# the thresholds, and the systemd timer plumbing. When something replaces Uptime
|
||||
# Kuma, only the push transport needs rewriting; the rest still applies.
|
||||
#
|
||||
# What was being monitored: archive/uptime_kuma/MONITORS.md
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
- name: Deploy CPU Temperature Monitoring
|
||||
hosts: hypervisor
|
||||
become: yes
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
- ../services_config.yml
|
||||
- ../infra_secrets.yml
|
||||
|
||||
vars:
|
||||
temp_threshold_celsius: 80
|
||||
temp_check_interval_minutes: 1
|
||||
monitoring_script_dir: /opt/nodito-monitoring
|
||||
monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh"
|
||||
log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log"
|
||||
systemd_service_name: nodito-cpu-temp-monitor
|
||||
uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
|
||||
ntfy_topic: "{{ service_settings.ntfy.topic }}"
|
||||
|
||||
tasks:
|
||||
- name: Validate Uptime Kuma configuration
|
||||
assert:
|
||||
that:
|
||||
- uptime_kuma_api_url is defined
|
||||
- uptime_kuma_api_url != ""
|
||||
- uptime_kuma_username is defined
|
||||
- uptime_kuma_username != ""
|
||||
- uptime_kuma_password is defined
|
||||
- uptime_kuma_password != ""
|
||||
fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
|
||||
|
||||
- name: Get hostname for monitor identification
|
||||
command: hostname
|
||||
register: host_name
|
||||
changed_when: false
|
||||
|
||||
- name: Set monitor name and group based on hostname
|
||||
set_fact:
|
||||
monitor_name: "cpu-temp-{{ host_name.stdout }}"
|
||||
monitor_friendly_name: "CPU Temperature: {{ host_name.stdout }}"
|
||||
uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
|
||||
|
||||
- name: Create Uptime Kuma CPU temperature monitor setup script
|
||||
copy:
|
||||
dest: /tmp/setup_uptime_kuma_cpu_temp_monitor.py
|
||||
content: |
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
from uptime_kuma_api import UptimeKumaApi
|
||||
|
||||
def main():
|
||||
api_url = sys.argv[1]
|
||||
username = sys.argv[2]
|
||||
password = sys.argv[3]
|
||||
group_name = sys.argv[4]
|
||||
monitor_name = sys.argv[5]
|
||||
monitor_description = sys.argv[6]
|
||||
interval = int(sys.argv[7])
|
||||
ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts"
|
||||
|
||||
api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0)
|
||||
api.login(username, password)
|
||||
|
||||
monitors = api.get_monitors()
|
||||
notifications = api.get_notifications()
|
||||
|
||||
ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
|
||||
notification_id_list = {}
|
||||
if ntfy_notification:
|
||||
notification_id_list[ntfy_notification['id']] = True
|
||||
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
if not group:
|
||||
api.add_monitor(type='group', name=group_name)
|
||||
monitors = api.get_monitors()
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
|
||||
existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
monitor_data = {
|
||||
'type': 'push',
|
||||
'name': monitor_name,
|
||||
'parent': group['id'],
|
||||
'interval': interval,
|
||||
'upsideDown': True,
|
||||
'description': monitor_description,
|
||||
'notificationIDList': notification_id_list
|
||||
}
|
||||
|
||||
if existing_monitor:
|
||||
api.edit_monitor(existing_monitor['id'], **monitor_data)
|
||||
else:
|
||||
api.add_monitor(**monitor_data)
|
||||
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
result = {
|
||||
'monitor_id': monitor['id'],
|
||||
'push_token': monitor['pushToken'],
|
||||
'group_name': group_name,
|
||||
'group_id': group['id'],
|
||||
'monitor_name': monitor_name
|
||||
}
|
||||
print(json.dumps(result))
|
||||
|
||||
api.disconnect()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
- name: Run Uptime Kuma monitor setup script
|
||||
command: >
|
||||
{{ ansible_playbook_python }}
|
||||
/tmp/setup_uptime_kuma_cpu_temp_monitor.py
|
||||
"{{ uptime_kuma_api_url }}"
|
||||
"{{ uptime_kuma_username }}"
|
||||
"{{ uptime_kuma_password }}"
|
||||
"{{ uptime_kuma_monitor_group }}"
|
||||
"{{ monitor_name }}"
|
||||
"{{ monitor_friendly_name }} - Alerts when temperature exceeds {{ temp_threshold_celsius }}°C"
|
||||
"{{ (temp_check_interval_minutes * 60) + 60 }}"
|
||||
"{{ ntfy_topic }}"
|
||||
register: monitor_setup_result
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
changed_when: false
|
||||
|
||||
- name: Parse monitor setup result
|
||||
set_fact:
|
||||
monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
|
||||
|
||||
- name: Set push URL and monitor ID as facts
|
||||
set_fact:
|
||||
uptime_kuma_cpu_temp_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
|
||||
uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
|
||||
|
||||
- name: Install required packages for temperature monitoring
|
||||
package:
|
||||
name:
|
||||
- lm-sensors
|
||||
- curl
|
||||
- jq
|
||||
- bc
|
||||
state: present
|
||||
|
||||
- name: Create monitoring script directory
|
||||
file:
|
||||
path: "{{ monitoring_script_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create CPU temperature monitoring script
|
||||
copy:
|
||||
dest: "{{ monitoring_script_path }}"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
|
||||
# CPU Temperature Monitoring Script
|
||||
# Monitors CPU temperature and sends alerts to Uptime Kuma
|
||||
|
||||
LOG_FILE="{{ log_file }}"
|
||||
TEMP_THRESHOLD="{{ temp_threshold_celsius }}"
|
||||
UPTIME_KUMA_URL="{{ uptime_kuma_cpu_temp_push_url }}"
|
||||
|
||||
log_message() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
get_cpu_temp() {
|
||||
local temp=""
|
||||
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
temp=$(sensors 2>/dev/null | grep -E "Core 0|Package id 0|Tdie|Tctl" | head -1 | grep -oE '[0-9]+\.[0-9]+°C' | grep -oE '[0-9]+\.[0-9]+')
|
||||
fi
|
||||
|
||||
if [ -z "$temp" ] && [ -f /sys/class/thermal/thermal_zone0/temp ]; then
|
||||
temp=$(cat /sys/class/thermal/thermal_zone0/temp)
|
||||
temp=$(echo "scale=1; $temp/1000" | bc -l 2>/dev/null || echo "$temp")
|
||||
fi
|
||||
|
||||
if [ -z "$temp" ] && command -v acpi >/dev/null 2>&1; then
|
||||
temp=$(acpi -t 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1)
|
||||
fi
|
||||
|
||||
echo "$temp"
|
||||
}
|
||||
|
||||
send_uptime_kuma_alert() {
|
||||
local temp="$1"
|
||||
local message="CPU Temperature Alert: ${temp}°C (Threshold: ${TEMP_THRESHOLD}°C)"
|
||||
|
||||
log_message "ALERT: $message"
|
||||
|
||||
encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/°/%C2%B0/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g')
|
||||
response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)"
|
||||
else
|
||||
log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
log_message "Starting CPU temperature check"
|
||||
|
||||
current_temp=$(get_cpu_temp)
|
||||
|
||||
if [ -z "$current_temp" ]; then
|
||||
log_message "ERROR: Could not read CPU temperature"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_message "Current CPU temperature: ${current_temp}°C"
|
||||
|
||||
if (( $(echo "$current_temp > $TEMP_THRESHOLD" | bc -l) )); then
|
||||
log_message "WARNING: CPU temperature ${current_temp}°C exceeds threshold ${TEMP_THRESHOLD}°C"
|
||||
send_uptime_kuma_alert "$current_temp"
|
||||
else
|
||||
log_message "CPU temperature is within normal range"
|
||||
fi
|
||||
}
|
||||
|
||||
main
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create systemd service for CPU temperature monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ systemd_service_name }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=CPU Temperature Monitor
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ monitoring_script_path }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd timer for CPU temperature monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ systemd_service_name }}.timer"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Run CPU Temperature Monitor every {{ temp_check_interval_minutes }} minute(s)
|
||||
Requires={{ systemd_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec={{ temp_check_interval_minutes }}min
|
||||
OnUnitActiveSec={{ temp_check_interval_minutes }}min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start CPU temperature monitoring timer
|
||||
systemd:
|
||||
name: "{{ systemd_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Test CPU temperature monitoring script
|
||||
command: "{{ monitoring_script_path }}"
|
||||
register: script_test
|
||||
changed_when: false
|
||||
|
||||
- name: Verify script execution
|
||||
assert:
|
||||
that:
|
||||
- script_test.rc == 0
|
||||
fail_msg: "CPU temperature monitoring script failed to execute properly"
|
||||
|
||||
- name: Display monitoring configuration
|
||||
debug:
|
||||
msg:
|
||||
- "CPU Temperature Monitoring configured successfully"
|
||||
- "Temperature threshold: {{ temp_threshold_celsius }}°C"
|
||||
- "Check interval: {{ temp_check_interval_minutes }} minute(s)"
|
||||
- "Monitor Name: {{ monitor_friendly_name }}"
|
||||
- "Monitor Group: {{ uptime_kuma_monitor_group }}"
|
||||
- "Uptime Kuma Push URL: {{ uptime_kuma_cpu_temp_push_url }}"
|
||||
- "Monitoring script: {{ monitoring_script_path }}"
|
||||
- "Systemd Service: {{ systemd_service_name }}.service"
|
||||
- "Systemd Timer: {{ systemd_service_name }}.timer"
|
||||
|
||||
- name: Clean up temporary Uptime Kuma setup script
|
||||
file:
|
||||
path: /tmp/setup_uptime_kuma_cpu_temp_monitor.py
|
||||
state: absent
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
- name: Install rsync
|
||||
hosts: managed
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
- name: Join machine to headscale mesh network
|
||||
hosts: managed
|
||||
become: yes
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
- ../services_config.yml
|
||||
vars:
|
||||
headscale_host_name: "spacey"
|
||||
headscale_subdomain: "{{ subdomains.headscale }}"
|
||||
headscale_domain: "https://{{ headscale_subdomain }}.{{ root_domain }}"
|
||||
headscale_namespace: "{{ service_settings.headscale.namespace }}"
|
||||
|
||||
tasks:
|
||||
- name: Set facts for headscale server connection
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
- name: Bootstrap Nodito SSH Key Access
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
|
||||
tasks:
|
||||
- name: Install sudo package
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
- name: Switch Proxmox VE from Enterprise to Community Repositories
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
|
||||
tasks:
|
||||
- name: Check for deb822 sources format
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
- name: Setup ZFS RAID 1 Pool for Proxmox Storage
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../infra_vars.yml
|
||||
- nodito_vars.yml
|
||||
|
||||
tasks:
|
||||
- name: Verify Proxmox VE is running
|
||||
|
|
@ -136,18 +139,17 @@
|
|||
Config file exists: {{ storage_cfg_file.stat.exists }}
|
||||
Storage check result: {{ storage_exists_check.rc }}
|
||||
Pool exists: {{ zfs_pool_exists.rc == 0 }}
|
||||
Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }}
|
||||
Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }}
|
||||
|
||||
# Registration is add-only on purpose. There used to be a "Remove existing
|
||||
# storage if it exists" task here that ran `pvesm remove` whenever the
|
||||
# storage WAS present, paired with an add that only ran when it was ABSENT.
|
||||
# The two conditions are mutually exclusive, so a real run against a
|
||||
# correctly-configured hypervisor removed the storage entry backing every VM
|
||||
# and never put it back. It would also have dropped `mountpoint /var/lib/vz`,
|
||||
# which the live entry has and which `pvesm add` below does not set.
|
||||
#
|
||||
# If the storage entry ever needs its options changed, edit
|
||||
# /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here.
|
||||
- name: Remove existing storage if it exists
|
||||
command: pvesm remove {{ zfs_pool_name }}
|
||||
register: pvesm_remove_result
|
||||
failed_when: false
|
||||
when:
|
||||
- zfs_pool_exists.rc == 0
|
||||
- storage_exists_check.rc == 0
|
||||
|
||||
- name: Add ZFS pool storage to Proxmox using pvesm
|
||||
command: >
|
||||
pvesm add zfspool {{ zfs_pool_name }}
|
||||
|
|
@ -169,56 +171,511 @@
|
|||
msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state"
|
||||
when: "'ONLINE' not in final_zfs_status.stdout"
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# The monthly scrub.
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
#
|
||||
# The ZFS HEALTH CHECK that used to share this play is gone: it is now the
|
||||
# zfs-health check in infra/400_host_monitoring.yml, which carries the same five
|
||||
# conditions - pool state, device states, resilver in progress, read/write/
|
||||
# checksum errors, and errors from the last scan - but reports to Gatus like
|
||||
# every other host check instead of owning its own push plumbing.
|
||||
# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
|
||||
# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
|
||||
# Kuma configuration" assert stops it before anything is installed or changed.
|
||||
#
|
||||
# The scrub itself stays here, because it is not monitoring: it is the
|
||||
# maintenance that gives the health check something true to report. A pool that
|
||||
# is never scrubbed has no idea whether it is healthy.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
- name: Schedule the monthly ZFS scrub
|
||||
# It is kept because the CHECK LOGIC is the durable part — what gets measured,
|
||||
# the thresholds, and the systemd timer plumbing. When something replaces Uptime
|
||||
# Kuma, only the push transport needs rewriting; the rest still applies.
|
||||
#
|
||||
# What was being monitored: archive/uptime_kuma/MONITORS.md
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../../infra_vars.yml
|
||||
- ../../services_config.yml
|
||||
- ../../infra_secrets.yml
|
||||
- nodito_vars.yml
|
||||
|
||||
vars:
|
||||
zfs_check_interval_seconds: 86400 # 24 hours
|
||||
zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer)
|
||||
zfs_check_retries: 1
|
||||
zfs_monitoring_script_dir: /opt/zfs-monitoring
|
||||
zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh"
|
||||
zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log"
|
||||
zfs_systemd_health_service_name: zfs-health-monitor
|
||||
zfs_systemd_scrub_service_name: zfs-monthly-scrub
|
||||
uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
|
||||
ntfy_topic: "{{ service_settings.ntfy.topic }}"
|
||||
|
||||
tasks:
|
||||
- name: Validate Uptime Kuma configuration
|
||||
assert:
|
||||
that:
|
||||
- uptime_kuma_api_url is defined
|
||||
- uptime_kuma_api_url != ""
|
||||
- uptime_kuma_username is defined
|
||||
- uptime_kuma_username != ""
|
||||
- uptime_kuma_password is defined
|
||||
- uptime_kuma_password != ""
|
||||
fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
|
||||
|
||||
- name: Get hostname for monitor identification
|
||||
command: hostname
|
||||
register: host_name
|
||||
changed_when: false
|
||||
|
||||
- name: Set monitor name and group based on hostname
|
||||
set_fact:
|
||||
monitor_name: "zfs-health-{{ host_name.stdout }}"
|
||||
monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}"
|
||||
uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
|
||||
|
||||
- name: Create Uptime Kuma ZFS health monitor setup script
|
||||
copy:
|
||||
dest: /tmp/setup_uptime_kuma_zfs_monitor.py
|
||||
content: |
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
from uptime_kuma_api import UptimeKumaApi
|
||||
|
||||
def main():
|
||||
api_url = sys.argv[1]
|
||||
username = sys.argv[2]
|
||||
password = sys.argv[3]
|
||||
group_name = sys.argv[4]
|
||||
monitor_name = sys.argv[5]
|
||||
monitor_description = sys.argv[6]
|
||||
interval = int(sys.argv[7])
|
||||
retries = int(sys.argv[8])
|
||||
ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts"
|
||||
|
||||
api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0)
|
||||
api.login(username, password)
|
||||
|
||||
# Get all monitors
|
||||
monitors = api.get_monitors()
|
||||
|
||||
# Get all notifications and find ntfy notification
|
||||
notifications = api.get_notifications()
|
||||
ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
|
||||
notification_id_list = {}
|
||||
if ntfy_notification:
|
||||
notification_id_list[ntfy_notification['id']] = True
|
||||
|
||||
# Find or create group
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
if not group:
|
||||
group_result = api.add_monitor(type='group', name=group_name)
|
||||
# Refresh to get the full group object with id
|
||||
monitors = api.get_monitors()
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
|
||||
# Find or create/update push monitor
|
||||
existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
monitor_data = {
|
||||
'type': 'push',
|
||||
'name': monitor_name,
|
||||
'parent': group['id'],
|
||||
'interval': interval,
|
||||
'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy
|
||||
'maxretries': retries,
|
||||
'description': monitor_description,
|
||||
'notificationIDList': notification_id_list
|
||||
}
|
||||
|
||||
if existing_monitor:
|
||||
monitor = api.edit_monitor(existing_monitor['id'], **monitor_data)
|
||||
# Refresh to get the full monitor object with pushToken
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
else:
|
||||
monitor_result = api.add_monitor(**monitor_data)
|
||||
# Refresh to get the full monitor object with pushToken
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
# Output result as JSON
|
||||
result = {
|
||||
'monitor_id': monitor['id'],
|
||||
'push_token': monitor['pushToken'],
|
||||
'group_name': group_name,
|
||||
'group_id': group['id'],
|
||||
'monitor_name': monitor_name
|
||||
}
|
||||
print(json.dumps(result))
|
||||
|
||||
api.disconnect()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
- name: Run Uptime Kuma ZFS monitor setup script
|
||||
command: >
|
||||
{{ ansible_playbook_python }}
|
||||
/tmp/setup_uptime_kuma_zfs_monitor.py
|
||||
"{{ uptime_kuma_api_url }}"
|
||||
"{{ uptime_kuma_username }}"
|
||||
"{{ uptime_kuma_password }}"
|
||||
"{{ uptime_kuma_monitor_group }}"
|
||||
"{{ monitor_name }}"
|
||||
"{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}"
|
||||
"{{ zfs_check_timeout_seconds }}"
|
||||
"{{ zfs_check_retries }}"
|
||||
"{{ ntfy_topic }}"
|
||||
register: monitor_setup_result
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
changed_when: false
|
||||
|
||||
- name: Parse monitor setup result
|
||||
set_fact:
|
||||
monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
|
||||
|
||||
- name: Set push URL and monitor ID as facts
|
||||
set_fact:
|
||||
uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
|
||||
uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}"
|
||||
|
||||
- name: Install required packages for ZFS monitoring
|
||||
package:
|
||||
name:
|
||||
- curl
|
||||
- jq
|
||||
state: present
|
||||
|
||||
- name: Create monitoring script directory
|
||||
file:
|
||||
path: "{{ zfs_monitoring_script_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create ZFS health monitoring script
|
||||
copy:
|
||||
dest: "{{ zfs_monitoring_script_path }}"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
|
||||
# ZFS Pool Health Monitoring Script
|
||||
# Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy
|
||||
# If any issues detected, does NOT send heartbeat (triggers timeout alert)
|
||||
|
||||
LOG_FILE="{{ zfs_log_file }}"
|
||||
UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}"
|
||||
POOL_NAME="{{ zfs_pool_name }}"
|
||||
HOSTNAME=$(hostname)
|
||||
|
||||
# Function to log messages
|
||||
log_message() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to check pool health using JSON output
|
||||
check_pool_health() {
|
||||
local pool="$1"
|
||||
local issues_found=0
|
||||
|
||||
# Get pool status as JSON
|
||||
local pool_json
|
||||
pool_json=$(zpool status -j "$pool" 2>&1)
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log_message "ERROR: Failed to get pool status for $pool"
|
||||
log_message " -> $pool_json"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check 1: Pool state must be ONLINE
|
||||
local pool_state
|
||||
pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state')
|
||||
|
||||
if [ "$pool_state" != "ONLINE" ]; then
|
||||
log_message "ISSUE: Pool state is $pool_state (expected ONLINE)"
|
||||
issues_found=1
|
||||
else
|
||||
log_message "OK: Pool state is ONLINE"
|
||||
fi
|
||||
|
||||
# Check 2: Check all vdevs and devices for non-ONLINE states
|
||||
local bad_states
|
||||
bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" '
|
||||
.pools[$pool].vdevs[] |
|
||||
.. | objects |
|
||||
select(.state? and .state != "ONLINE") |
|
||||
"\(.name // "unknown"): \(.state)"
|
||||
' 2>/dev/null)
|
||||
|
||||
if [ -n "$bad_states" ]; then
|
||||
log_message "ISSUE: Found devices not in ONLINE state:"
|
||||
echo "$bad_states" | while read -r line; do
|
||||
log_message " -> $line"
|
||||
done
|
||||
issues_found=1
|
||||
else
|
||||
log_message "OK: All devices are ONLINE"
|
||||
fi
|
||||
|
||||
# Check 3: Check for resilvering in progress
|
||||
local scan_function scan_state
|
||||
scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"')
|
||||
scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"')
|
||||
|
||||
if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then
|
||||
local resilver_progress
|
||||
resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"')
|
||||
log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed"
|
||||
issues_found=1
|
||||
fi
|
||||
|
||||
# Check 4: Check for read/write/checksum errors on all devices
|
||||
# Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison
|
||||
local devices_with_errors
|
||||
devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '
|
||||
.pools[$pool].vdevs[] |
|
||||
.. | objects |
|
||||
select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) |
|
||||
"\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)"
|
||||
' 2>/dev/null)
|
||||
|
||||
if [ -n "$devices_with_errors" ]; then
|
||||
log_message "ISSUE: Found devices with I/O errors:"
|
||||
echo "$devices_with_errors" | while read -r line; do
|
||||
log_message " -> $line"
|
||||
done
|
||||
issues_found=1
|
||||
else
|
||||
log_message "OK: No read/write/checksum errors detected"
|
||||
fi
|
||||
|
||||
# Check 5: Check for scan errors (from last scrub/resilver)
|
||||
local scan_errors
|
||||
scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"')
|
||||
|
||||
if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then
|
||||
log_message "ISSUE: Last scan reported $scan_errors errors"
|
||||
issues_found=1
|
||||
else
|
||||
log_message "OK: No scan errors"
|
||||
fi
|
||||
|
||||
return $issues_found
|
||||
}
|
||||
|
||||
# Function to get last scrub info for status message
|
||||
get_scrub_info() {
|
||||
local pool="$1"
|
||||
local pool_json
|
||||
pool_json=$(zpool status -j "$pool" 2>/dev/null)
|
||||
|
||||
local scan_func scan_state scan_start
|
||||
scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"')
|
||||
scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"')
|
||||
scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""')
|
||||
|
||||
if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then
|
||||
echo "scrub in progress (started $scan_start)"
|
||||
elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then
|
||||
echo "last scrub: $scan_start"
|
||||
else
|
||||
echo "no scrub history"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to send heartbeat to Uptime Kuma
|
||||
send_heartbeat() {
|
||||
local message="$1"
|
||||
|
||||
log_message "Sending heartbeat to Uptime Kuma: $message"
|
||||
|
||||
# URL encode the message
|
||||
local encoded_message
|
||||
encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g')
|
||||
|
||||
local response http_code
|
||||
response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
log_message "Heartbeat sent successfully (HTTP $http_code)"
|
||||
return 0
|
||||
else
|
||||
log_message "ERROR: Failed to send heartbeat (HTTP $http_code)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main health check logic
|
||||
main() {
|
||||
log_message "=========================================="
|
||||
log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME"
|
||||
|
||||
# Run all health checks
|
||||
if check_pool_health "$POOL_NAME"; then
|
||||
# All checks passed - send heartbeat
|
||||
local scrub_info
|
||||
scrub_info=$(get_scrub_info "$POOL_NAME")
|
||||
|
||||
local message="Pool $POOL_NAME healthy ($scrub_info)"
|
||||
send_heartbeat "$message"
|
||||
|
||||
log_message "Health check completed: ALL OK"
|
||||
exit 0
|
||||
else
|
||||
# Issues found - do NOT send heartbeat (will trigger timeout alert)
|
||||
log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat"
|
||||
log_message "Uptime Kuma will alert after timeout due to missing heartbeat"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create systemd service for ZFS health monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=ZFS Pool Health Monitor
|
||||
After=zfs.target network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ zfs_monitoring_script_path }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd timer for daily ZFS health monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Run ZFS Pool Health Monitor daily
|
||||
Requires={{ zfs_systemd_health_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec={{ zfs_check_interval_seconds }}sec
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd service for ZFS monthly scrub
|
||||
template:
|
||||
src: templates/zfs-monthly-scrub.service.j2
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=ZFS Monthly Scrub for {{ zfs_pool_name }}
|
||||
After=zfs.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/sbin/zpool scrub {{ zfs_pool_name }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd timer for monthly ZFS scrub
|
||||
template:
|
||||
src: templates/zfs-monthly-scrub.timer.j2
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Run ZFS Scrub on last day of every month at 4:00 AM
|
||||
Requires={{ zfs_systemd_scrub_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*~01 04:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Enable and start the monthly scrub timer
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start ZFS health monitoring timer
|
||||
systemd:
|
||||
name: "{{ zfs_systemd_health_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Enable and start ZFS monthly scrub timer
|
||||
systemd:
|
||||
name: "{{ zfs_systemd_scrub_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Report the scrub schedule
|
||||
- name: Test ZFS health monitoring script
|
||||
command: "{{ zfs_monitoring_script_path }}"
|
||||
register: script_test
|
||||
changed_when: false
|
||||
|
||||
- name: Verify script execution
|
||||
assert:
|
||||
that:
|
||||
- script_test.rc == 0
|
||||
fail_msg: "ZFS health monitoring script failed - check pool health"
|
||||
|
||||
- name: Display monitoring configuration
|
||||
debug:
|
||||
msg: >-
|
||||
Monthly scrub of {{ zfs_pool_name }}:
|
||||
last day of each month at 04:00.
|
||||
Health is reported separately by the zfs-health check
|
||||
(infra/400_host_monitoring.yml).
|
||||
msg: |
|
||||
✓ ZFS Pool Health Monitoring deployed successfully!
|
||||
|
||||
Monitor Name: {{ monitor_friendly_name }}
|
||||
Monitor Group: {{ uptime_kuma_monitor_group }}
|
||||
Pool Name: {{ zfs_pool_name }}
|
||||
|
||||
Health Check:
|
||||
- Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours)
|
||||
- Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours)
|
||||
- Script: {{ zfs_monitoring_script_path }}
|
||||
- Log: {{ zfs_log_file }}
|
||||
- Service: {{ zfs_systemd_health_service_name }}.service
|
||||
- Timer: {{ zfs_systemd_health_service_name }}.timer
|
||||
|
||||
Monthly Scrub:
|
||||
- Schedule: Last day of month at 4:00 AM
|
||||
- Service: {{ zfs_systemd_scrub_service_name }}.service
|
||||
- Timer: {{ zfs_systemd_scrub_service_name }}.timer
|
||||
|
||||
Conditions monitored:
|
||||
- Pool state (must be ONLINE)
|
||||
- Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL)
|
||||
- Resilver status (alerts if resilvering)
|
||||
- Read/Write/Checksum errors
|
||||
- Scrub errors
|
||||
|
||||
- name: Clean up temporary Uptime Kuma setup script
|
||||
file:
|
||||
path: /tmp/setup_uptime_kuma_zfs_monitor.py
|
||||
state: absent
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
- name: Create Proxmox template from Debian cloud image (no VM clone)
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../../infra_vars.yml
|
||||
- nodito_vars.yml
|
||||
|
||||
vars:
|
||||
# Defaults (override via vars_files or --extra-vars as needed)
|
||||
|
|
|
|||
|
|
@ -1,33 +1,12 @@
|
|||
- name: Setup NUT (Network UPS Tools) for CyberPower UPS
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../../infra_vars.yml
|
||||
- nodito_vars.yml
|
||||
- nodito_secrets.yml
|
||||
|
||||
tasks:
|
||||
# ------------------------------------------------------------------
|
||||
# Safety catch
|
||||
#
|
||||
# /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by
|
||||
# hand in January 2026 and carry a working password. host_vars/nodito/vault.yml
|
||||
# (formerly infra/nodito/nodito_secrets.yml) still holds the literal string
|
||||
# CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that
|
||||
# working pair with a placeholder and restart NUT - leaving the hypervisor's
|
||||
# UPS unmonitored and unable to trigger a clean shutdown on mains loss.
|
||||
#
|
||||
# Until the real password is put in the vault, stop here.
|
||||
# ansible-vault edit host_vars/nodito/vault.yml
|
||||
# ------------------------------------------------------------------
|
||||
- name: Refuse to run with a placeholder UPS password
|
||||
assert:
|
||||
that:
|
||||
- ups_password is defined
|
||||
- ups_password | length > 0
|
||||
- ups_password != "CHANGE_ME_TO_SECURE_PASSWORD"
|
||||
fail_msg: >-
|
||||
ups_password is unset or still the placeholder. Applying this play would
|
||||
overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on
|
||||
nodito and restart NUT. Put the real password in the vault first:
|
||||
ansible-vault edit host_vars/nodito/vault.yml
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Installation
|
||||
# ------------------------------------------------------------------
|
||||
|
|
@ -96,45 +75,90 @@
|
|||
# Configuration files
|
||||
# ------------------------------------------------------------------
|
||||
- name: Configure NUT mode (standalone)
|
||||
template:
|
||||
copy:
|
||||
dest: /etc/nut/nut.conf
|
||||
src: templates/nut.conf.j2
|
||||
content: |
|
||||
# Managed by Ansible
|
||||
MODE=standalone
|
||||
owner: root
|
||||
group: nut
|
||||
mode: "0640"
|
||||
notify: Restart NUT services
|
||||
|
||||
- name: Configure UPS device
|
||||
template:
|
||||
copy:
|
||||
dest: /etc/nut/ups.conf
|
||||
src: templates/ups.conf.j2
|
||||
content: |
|
||||
# Managed by Ansible
|
||||
[{{ ups_name }}]
|
||||
driver = {{ ups_driver }}
|
||||
port = {{ ups_port }}
|
||||
desc = "{{ ups_desc }}"
|
||||
offdelay = {{ ups_offdelay }}
|
||||
ondelay = {{ ups_ondelay }}
|
||||
owner: root
|
||||
group: nut
|
||||
mode: "0640"
|
||||
notify: Restart NUT services
|
||||
|
||||
- name: Configure upsd to listen on localhost
|
||||
template:
|
||||
copy:
|
||||
dest: /etc/nut/upsd.conf
|
||||
src: templates/upsd.conf.j2
|
||||
content: |
|
||||
# Managed by Ansible
|
||||
LISTEN 127.0.0.1 3493
|
||||
owner: root
|
||||
group: nut
|
||||
mode: "0640"
|
||||
notify: Restart NUT services
|
||||
|
||||
- name: Configure upsd users
|
||||
template:
|
||||
copy:
|
||||
dest: /etc/nut/upsd.users
|
||||
src: templates/upsd.users.j2
|
||||
content: |
|
||||
# Managed by Ansible
|
||||
[{{ ups_user }}]
|
||||
password = {{ ups_password }}
|
||||
upsmon master
|
||||
owner: root
|
||||
group: nut
|
||||
mode: "0640"
|
||||
notify: Restart NUT services
|
||||
|
||||
- name: Configure upsmon
|
||||
template:
|
||||
copy:
|
||||
dest: /etc/nut/upsmon.conf
|
||||
src: templates/upsmon.conf.j2
|
||||
content: |
|
||||
# Managed by Ansible
|
||||
MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master
|
||||
|
||||
MINSUPPLIES 1
|
||||
SHUTDOWNCMD "/sbin/shutdown -h +0"
|
||||
POLLFREQ 5
|
||||
POLLFREQALERT 5
|
||||
HOSTSYNC 15
|
||||
DEADTIME 15
|
||||
POWERDOWNFLAG /etc/killpower
|
||||
|
||||
# Notifications
|
||||
NOTIFYMSG ONLINE "UPS %s on line power"
|
||||
NOTIFYMSG ONBATT "UPS %s on battery"
|
||||
NOTIFYMSG LOWBATT "UPS %s battery is low"
|
||||
NOTIFYMSG FSD "UPS %s: forced shutdown in progress"
|
||||
NOTIFYMSG COMMOK "Communications with UPS %s established"
|
||||
NOTIFYMSG COMMBAD "Communications with UPS %s lost"
|
||||
NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding"
|
||||
NOTIFYMSG REPLBATT "UPS %s battery needs replacing"
|
||||
|
||||
# Log all events to syslog
|
||||
NOTIFYFLAG ONLINE SYSLOG
|
||||
NOTIFYFLAG ONBATT SYSLOG
|
||||
NOTIFYFLAG LOWBATT SYSLOG
|
||||
NOTIFYFLAG FSD SYSLOG
|
||||
NOTIFYFLAG COMMOK SYSLOG
|
||||
NOTIFYFLAG COMMBAD SYSLOG
|
||||
NOTIFYFLAG SHUTDOWN SYSLOG
|
||||
NOTIFYFLAG REPLBATT SYSLOG
|
||||
owner: root
|
||||
group: nut
|
||||
mode: "0640"
|
||||
|
|
@ -225,11 +249,334 @@
|
|||
- nut-server
|
||||
- nut-monitor
|
||||
|
||||
# The UPS heartbeat play that used to live here is gone. What it deployed -
|
||||
# /opt/ups-monitoring plus a ups-heartbeat timer - is now the ups-status check
|
||||
# in infra/400_host_monitoring.yml, which reports to Gatus like every other
|
||||
# host check instead of carrying its own push plumbing.
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
#
|
||||
# This playbook is now purely NUT setup: the driver, upsd, upsmon and the
|
||||
# shutdown behaviour. Monitoring whether the UPS is on mains is a separate
|
||||
# concern and belongs with the other host checks.
|
||||
# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username
|
||||
# and uptime_kuma_password were removed from the vault, so the "Validate Uptime
|
||||
# Kuma configuration" assert stops it before anything is installed or changed.
|
||||
#
|
||||
# It is kept because the CHECK LOGIC is the durable part — what gets measured,
|
||||
# the thresholds, and the systemd timer plumbing. When something replaces Uptime
|
||||
# Kuma, only the push transport needs rewriting; the rest still applies.
|
||||
#
|
||||
# What was being monitored: archive/uptime_kuma/MONITORS.md
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
- name: Setup UPS Heartbeat Monitoring with Uptime Kuma
|
||||
hosts: hypervisor
|
||||
become: true
|
||||
vars_files:
|
||||
- ../../infra_vars.yml
|
||||
- ../../services_config.yml
|
||||
- ../../infra_secrets.yml
|
||||
- nodito_vars.yml
|
||||
- nodito_secrets.yml
|
||||
|
||||
vars:
|
||||
ups_heartbeat_interval_seconds: 60
|
||||
ups_heartbeat_timeout_seconds: 120
|
||||
ups_heartbeat_retries: 1
|
||||
ups_monitoring_script_dir: /opt/ups-monitoring
|
||||
ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh"
|
||||
ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log"
|
||||
ups_systemd_service_name: ups-heartbeat
|
||||
uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}"
|
||||
ntfy_topic: "{{ service_settings.ntfy.topic }}"
|
||||
|
||||
tasks:
|
||||
- name: Validate Uptime Kuma configuration
|
||||
assert:
|
||||
that:
|
||||
- uptime_kuma_api_url is defined
|
||||
- uptime_kuma_api_url != ""
|
||||
- uptime_kuma_username is defined
|
||||
- uptime_kuma_username != ""
|
||||
- uptime_kuma_password is defined
|
||||
- uptime_kuma_password != ""
|
||||
fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set"
|
||||
|
||||
- name: Get hostname for monitor identification
|
||||
command: hostname
|
||||
register: host_name
|
||||
changed_when: false
|
||||
|
||||
- name: Set monitor name and group based on hostname
|
||||
set_fact:
|
||||
monitor_name: "ups-{{ host_name.stdout }}"
|
||||
monitor_friendly_name: "UPS Status: {{ host_name.stdout }}"
|
||||
uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra"
|
||||
|
||||
- name: Create Uptime Kuma UPS monitor setup script
|
||||
copy:
|
||||
dest: /tmp/setup_uptime_kuma_ups_monitor.py
|
||||
content: |
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
from uptime_kuma_api import UptimeKumaApi
|
||||
|
||||
def main():
|
||||
api_url = sys.argv[1]
|
||||
username = sys.argv[2]
|
||||
password = sys.argv[3]
|
||||
group_name = sys.argv[4]
|
||||
monitor_name = sys.argv[5]
|
||||
monitor_description = sys.argv[6]
|
||||
interval = int(sys.argv[7])
|
||||
retries = int(sys.argv[8])
|
||||
ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts"
|
||||
|
||||
api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0)
|
||||
api.login(username, password)
|
||||
|
||||
monitors = api.get_monitors()
|
||||
notifications = api.get_notifications()
|
||||
|
||||
ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None)
|
||||
notification_id_list = {}
|
||||
if ntfy_notification:
|
||||
notification_id_list[ntfy_notification['id']] = True
|
||||
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
if not group:
|
||||
api.add_monitor(type='group', name=group_name)
|
||||
monitors = api.get_monitors()
|
||||
group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None)
|
||||
|
||||
existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
monitor_data = {
|
||||
'type': 'push',
|
||||
'name': monitor_name,
|
||||
'parent': group['id'],
|
||||
'interval': interval,
|
||||
'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy
|
||||
'maxretries': retries,
|
||||
'description': monitor_description,
|
||||
'notificationIDList': notification_id_list
|
||||
}
|
||||
|
||||
if existing_monitor:
|
||||
api.edit_monitor(existing_monitor['id'], **monitor_data)
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
else:
|
||||
api.add_monitor(**monitor_data)
|
||||
monitors = api.get_monitors()
|
||||
monitor = next((m for m in monitors if m.get('name') == monitor_name), None)
|
||||
|
||||
result = {
|
||||
'monitor_id': monitor['id'],
|
||||
'push_token': monitor['pushToken'],
|
||||
'group_name': group_name,
|
||||
'group_id': group['id'],
|
||||
'monitor_name': monitor_name
|
||||
}
|
||||
print(json.dumps(result))
|
||||
|
||||
api.disconnect()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
||||
- name: Run Uptime Kuma UPS monitor setup script
|
||||
command: >
|
||||
{{ ansible_playbook_python }}
|
||||
/tmp/setup_uptime_kuma_ups_monitor.py
|
||||
"{{ uptime_kuma_api_url }}"
|
||||
"{{ uptime_kuma_username }}"
|
||||
"{{ uptime_kuma_password }}"
|
||||
"{{ uptime_kuma_monitor_group }}"
|
||||
"{{ monitor_name }}"
|
||||
"{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication"
|
||||
"{{ ups_heartbeat_timeout_seconds }}"
|
||||
"{{ ups_heartbeat_retries }}"
|
||||
"{{ ntfy_topic }}"
|
||||
register: monitor_setup_result
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
changed_when: false
|
||||
|
||||
- name: Parse monitor setup result
|
||||
set_fact:
|
||||
monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}"
|
||||
|
||||
- name: Set push URL as fact
|
||||
set_fact:
|
||||
uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}"
|
||||
|
||||
- name: Install required packages for UPS monitoring
|
||||
package:
|
||||
name:
|
||||
- curl
|
||||
state: present
|
||||
|
||||
- name: Create monitoring script directory
|
||||
file:
|
||||
path: "{{ ups_monitoring_script_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create UPS heartbeat monitoring script
|
||||
copy:
|
||||
dest: "{{ ups_monitoring_script_path }}"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
|
||||
# UPS Heartbeat Monitoring Script
|
||||
# Sends heartbeat to Uptime Kuma only when UPS is on mains power
|
||||
# When on battery or communication lost, no heartbeat is sent (triggers timeout alert)
|
||||
|
||||
LOG_FILE="{{ ups_log_file }}"
|
||||
UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}"
|
||||
UPS_NAME="{{ ups_name }}"
|
||||
|
||||
log_message() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
send_heartbeat() {
|
||||
local message="$1"
|
||||
|
||||
local encoded_message
|
||||
encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g')
|
||||
|
||||
local response http_code
|
||||
response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1)
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
log_message "Heartbeat sent: $message (HTTP $http_code)"
|
||||
return 0
|
||||
else
|
||||
log_message "ERROR: Failed to send heartbeat (HTTP $http_code)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local status charge runtime load
|
||||
|
||||
status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null)
|
||||
|
||||
if [ -z "$status" ]; then
|
||||
log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null)
|
||||
runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null)
|
||||
load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null)
|
||||
|
||||
if [[ "$status" == *"OL"* ]]; then
|
||||
local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)"
|
||||
send_heartbeat "$message"
|
||||
exit 0
|
||||
else
|
||||
log_message "UPS not on mains power (status=$status) - NOT sending heartbeat"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create systemd service for UPS heartbeat
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=UPS Heartbeat Monitor
|
||||
After=network.target nut-monitor.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ ups_monitoring_script_path }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Create systemd timer for UPS heartbeat
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds
|
||||
Requires={{ ups_systemd_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start UPS heartbeat timer
|
||||
systemd:
|
||||
name: "{{ ups_systemd_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Test UPS heartbeat script
|
||||
command: "{{ ups_monitoring_script_path }}"
|
||||
register: script_test
|
||||
changed_when: false
|
||||
|
||||
- name: Verify script execution
|
||||
assert:
|
||||
that:
|
||||
- script_test.rc == 0
|
||||
fail_msg: "UPS heartbeat script failed - check UPS status and communication"
|
||||
|
||||
- name: Display monitoring configuration
|
||||
debug:
|
||||
msg:
|
||||
- "UPS Monitoring configured successfully"
|
||||
- ""
|
||||
- "NUT Configuration:"
|
||||
- " UPS Name: {{ ups_name }}"
|
||||
- " UPS Description: {{ ups_desc }}"
|
||||
- " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)"
|
||||
- " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)"
|
||||
- ""
|
||||
- "Uptime Kuma Monitoring:"
|
||||
- " Monitor Name: {{ monitor_friendly_name }}"
|
||||
- " Monitor Group: {{ uptime_kuma_monitor_group }}"
|
||||
- " Push URL: {{ uptime_kuma_ups_push_url }}"
|
||||
- " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s"
|
||||
- " Timeout: {{ ups_heartbeat_timeout_seconds }}s"
|
||||
- ""
|
||||
- "Scripts and Services:"
|
||||
- " Script: {{ ups_monitoring_script_path }}"
|
||||
- " Log: {{ ups_log_file }}"
|
||||
- " Service: {{ ups_systemd_service_name }}.service"
|
||||
- " Timer: {{ ups_systemd_service_name }}.timer"
|
||||
|
||||
- name: Clean up temporary Uptime Kuma setup script
|
||||
file:
|
||||
path: /tmp/setup_uptime_kuma_ups_monitor.py
|
||||
state: absent
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
|
|
|
|||
12
ansible/infra/nodito/nodito_secrets.yml
Normal file
12
ansible/infra/nodito/nodito_secrets.yml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
$ANSIBLE_VAULT;1.1;AES256
|
||||
39386134336137333139343861353965353665626161303634323563386563356364373265616163
|
||||
3237623438636161323366313363356335613566303338630a663732316663333136653737663935
|
||||
34373466316538663962636134303530323837316563336136653636643430363566376363323666
|
||||
3531613636393331310a306633613030343138656237663061653438336562323135633732356563
|
||||
33396136656564353465636361613161326465303166373131343562363338303966666362643532
|
||||
33383561366539393930633766363533313363653733393263376634316362643863376635366638
|
||||
36396333633231616662323465323932656565396138313264613832346261616231333265636439
|
||||
61666334613662373839613833613663333436373365376534643662656335316536303739616437
|
||||
65333635613330353536353162646234323266316338643435653864666364313734386665303830
|
||||
34383961633734646134373866623330663038366130306265656466653562643764346162313333
|
||||
353664363335303433393330306332666336
|
||||
28
ansible/infra/nodito/nodito_vars.yml
Normal file
28
ansible/infra/nodito/nodito_vars.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# Nodito CPU Temperature Monitoring Configuration
|
||||
|
||||
# Temperature Monitoring Configuration
|
||||
temp_threshold_celsius: 80
|
||||
temp_check_interval_minutes: 1
|
||||
|
||||
# Script Configuration
|
||||
monitoring_script_dir: /opt/nodito-monitoring
|
||||
monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh"
|
||||
log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log"
|
||||
|
||||
# System Configuration
|
||||
systemd_service_name: nodito-cpu-temp-monitor
|
||||
|
||||
# ZFS Pool Configuration
|
||||
zfs_pool_name: "proxmox-tank-1"
|
||||
zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror
|
||||
zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror
|
||||
zfs_pool_mountpoint: "/var/lib/vz"
|
||||
|
||||
# UPS Configuration (CyberPower CP900EPFCLCD via USB)
|
||||
ups_name: cyberpower
|
||||
ups_desc: "CyberPower CP900EPFCLCD"
|
||||
ups_driver: usbhid-ups
|
||||
ups_port: auto
|
||||
ups_user: counterweight
|
||||
ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power
|
||||
ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
# Managed by Ansible
|
||||
MODE=standalone
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# Managed by Ansible
|
||||
maxretry = 3
|
||||
|
||||
[{{ ups_name }}]
|
||||
driver = {{ ups_driver }}
|
||||
port = {{ ups_port }}
|
||||
desc = "{{ ups_desc }}"
|
||||
offdelay = {{ ups_offdelay }}
|
||||
ondelay = {{ ups_ondelay }}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
# Managed by Ansible
|
||||
LISTEN 127.0.0.1 3493
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
# Managed by Ansible
|
||||
[{{ ups_user }}]
|
||||
password = {{ ups_password }}
|
||||
upsmon master
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# Managed by Ansible
|
||||
MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master
|
||||
|
||||
MINSUPPLIES 1
|
||||
SHUTDOWNCMD "/sbin/shutdown -h +0"
|
||||
POLLFREQ 5
|
||||
POLLFREQALERT 5
|
||||
HOSTSYNC 15
|
||||
DEADTIME 15
|
||||
POWERDOWNFLAG "/etc/killpower"
|
||||
OFFDURATION 30
|
||||
RBWARNTIME 43200
|
||||
NOCOMMWARNTIME 300
|
||||
FINALDELAY 5
|
||||
|
||||
# Notifications
|
||||
NOTIFYMSG ONLINE "UPS %s on line power"
|
||||
NOTIFYMSG ONBATT "UPS %s on battery"
|
||||
NOTIFYMSG LOWBATT "UPS %s battery is low"
|
||||
NOTIFYMSG FSD "UPS %s: forced shutdown in progress"
|
||||
NOTIFYMSG COMMOK "Communications with UPS %s established"
|
||||
NOTIFYMSG COMMBAD "Communications with UPS %s lost"
|
||||
NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding"
|
||||
NOTIFYMSG REPLBATT "UPS %s battery needs replacing"
|
||||
|
||||
# Log all events to syslog
|
||||
NOTIFYFLAG ONLINE SYSLOG
|
||||
NOTIFYFLAG ONBATT SYSLOG
|
||||
NOTIFYFLAG LOWBATT SYSLOG
|
||||
NOTIFYFLAG FSD SYSLOG
|
||||
NOTIFYFLAG COMMOK SYSLOG
|
||||
NOTIFYFLAG COMMBAD SYSLOG
|
||||
NOTIFYFLAG SHUTDOWN SYSLOG
|
||||
NOTIFYFLAG REPLBATT SYSLOG
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
[Unit]
|
||||
Description=ZFS Monthly Scrub for {{ zfs_pool_name }}
|
||||
After=zfs.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/sbin/zpool scrub {{ zfs_pool_name }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[Unit]
|
||||
Description=Run ZFS Scrub on last day of every month at 4:00 AM
|
||||
Requires={{ zfs_systemd_scrub_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*~01 04:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
60
ansible/infra_secrets.yml
Normal file
60
ansible/infra_secrets.yml
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
$ANSIBLE_VAULT;1.1;AES256
|
||||
34616531626537376463326332306434383163393761363536633133363161373631376437653234
|
||||
3831656536313531343433376264643261396634633965370a303931336236323766313065636535
|
||||
31343338616364386132623232373266663665353563666535383232666262663062323864303932
|
||||
3439373734316639650a633732333064633335383833663261326464336538636666323063646331
|
||||
66653464636534663935616461636132613162303530663237376661343338396133323431623561
|
||||
30303563303465633231306339326561313533333536376433393130376634303932643339656336
|
||||
30616130343733626162366561393939393766623138616537393032376665396139333561623830
|
||||
33623130336235633034666561383165616439316334323664623661343438663733373833306335
|
||||
30396165313731636435636263353133326135346236323030653734353831626238666663656364
|
||||
62386164363361373464383730333733626333333736306535336635613634646535383237326332
|
||||
37636532663933663739303634333235343137333363316430643263326231613635643633636437
|
||||
30336462633233396430633939616661643261666136656363363461343230653738613166346434
|
||||
63376363666462346539343063343562623962303130616535306439653134633630353963373337
|
||||
64313239353162363862643432666438383330636638333464323731643163643236313535313030
|
||||
65656231383738356537326661663163376634613031396436646633376139313237343534653065
|
||||
64323532346133313265346334353530393633396339316330366536646565353836396662303866
|
||||
65373030393463306664623235626437613965303837313365373632643935656630303936373837
|
||||
64663539623764656636613263346638376665626262663430333231633735653563643864343835
|
||||
62303637383332653366383066326536316336306539623230353066343739316430356437313365
|
||||
33323031613130643832636133636263303565623330306135333762363036633737343933623437
|
||||
33633436363062643436376632336235666330656265316333323361316430656631383734343238
|
||||
65616366666536363866343361666563643336306532666332643230656264303565353032393932
|
||||
36613738326239376161383064646634623639356439663965323237303361343838373737636235
|
||||
39656465333063633136643463356564636537326339653033633063366265636138656466646264
|
||||
32643961343863656464353439346531633036613138353333353631626665396239326465303632
|
||||
39336664333038636435366333623336353963323535316436646136313739373537376537353735
|
||||
66333265633232346536313933333836643439656666636236626266323333383935383565616139
|
||||
38626133356364613562393664643563346539366364636332613162663337356232333735393431
|
||||
30343635393462356431646336636663333735343164616261343836363366616162613563613066
|
||||
66333034613234626431356536323832393466363135653231336238356334353638613838626430
|
||||
63636630313533363563376161663637353362383431663130646666646433353864373736613662
|
||||
62343666633862643839666532656132653939663437373164393536653739376265303130353636
|
||||
62616538326330333136626238306133363634326365366637363961333635356133613564373131
|
||||
64626139656633663065353862343135393233333231386166346532656534623361623061336434
|
||||
37616565626134333863376532393861376132323434656433613731613834633963386635353336
|
||||
39653263346637303836663763653264396139383363616461393333333362616332663863363834
|
||||
61613832326466646133626337663364333566326633313664613236346135653332373035313034
|
||||
62356530313065393737393634313166313039616635363365633031336434396463326462316236
|
||||
62353838636533303664663132353838636632353733316639643964666139383539333761663531
|
||||
37366530373165653063303032383461326535613336343635626538373635356266396666346133
|
||||
61383434323565306438356634646338666232363238393932353365363461376130323363363236
|
||||
34643862653937353461383265663933646264373365623465313666643662633334366437646434
|
||||
34343332613832366235663563353435323738356339383338333361383561336366376238393237
|
||||
38656265663164376666616366623062393530366261623361383365376235643035353838313235
|
||||
32396430376139313430316439333539343266393965353030636638656661316137313437663832
|
||||
64366661323865383262323036613230316233666566386133363633303931323464663238323861
|
||||
62333132346165353830653062353763306232366563323430383933653163373562326536393362
|
||||
66353931376262383931613931303034356537363137366235643430323465623162363031373034
|
||||
64653563666533363062333831336463376530306639616631303266386136666233653638653732
|
||||
31313537393838333730303235363661326333646531653935633935373362303732653565656438
|
||||
36626330623063373832333739643666396135653838643164376264393863646433353036643731
|
||||
34623636623366353466333237373265323936643134613230326565303663306431373634356334
|
||||
37613732393262626135306632623336613364323632316636623037656538313236666562633438
|
||||
37306361333130376538366232393637646263656239303431623963626265386634313735373162
|
||||
64393733303132383464386330626135336237303563326238633437346164333738646431333730
|
||||
62616533363638643261306565373835303832313539656564386132393863373039653938616261
|
||||
36343031356361336563316230343433633033353130373933313361326164633765316433616162
|
||||
62356338393634653330623933313433343630386264396337373532376335316362333164363963
|
||||
363663666365336564663232363462613162
|
||||
38
ansible/infra_secrets.yml.example
Normal file
38
ansible/infra_secrets.yml.example
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# Uptime Kuma login credentials
|
||||
# Used by the disk monitoring playbook to create monitors automatically
|
||||
|
||||
|
||||
# ntfy credentials
|
||||
# Used for notification channel setup in Uptime Kuma
|
||||
|
||||
ntfy_username: "your_ntfy_username"
|
||||
ntfy_password: "your_ntfy_password"
|
||||
|
||||
# headscale-ui credentials
|
||||
# Used for HTTP basic authentication via Caddy
|
||||
# Provide either:
|
||||
# - headscale_ui_password: plain text password (will be hashed automatically)
|
||||
# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate)
|
||||
|
||||
headscale_ui_username: "admin"
|
||||
headscale_ui_password: "your_secure_password_here"
|
||||
# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password
|
||||
|
||||
bitcoin_rpc_user: "bitcoinrpc"
|
||||
bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD"
|
||||
|
||||
# Mempool MariaDB credentials
|
||||
# Used by: services/mempool/deploy_mempool_playbook.yml
|
||||
mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD"
|
||||
|
||||
# Forgejo Runner registration token
|
||||
# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml
|
||||
# See: services/forgejo-runner/SETUP.md for how to obtain this token
|
||||
forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE"
|
||||
|
||||
# DATUM Gateway secrets
|
||||
# Used by: services/datum-gateway/deploy_datum_gateway_playbook.yml
|
||||
datum_mining_address: "YOUR_BITCOIN_ADDRESS_FOR_BLOCK_REWARDS"
|
||||
datum_gateway_admin_password: "CHANGE_ME_TO_SECURE_PASSWORD"
|
||||
datum_dashboard_username: "admin"
|
||||
datum_dashboard_password_hash: "$2a$14$..." # Generate with: caddy hash-password
|
||||
9
ansible/infra_vars.yml
Normal file
9
ansible/infra_vars.yml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
new_user: counterweight
|
||||
ssh_port: 22
|
||||
allow_ssh_from: "any"
|
||||
root_domain: contrapeso.xyz
|
||||
|
||||
# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the
|
||||
# playbooks are kept deliberately — the check logic is meant to be rewired to
|
||||
# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/.
|
||||
uptime_kuma_enabled: false
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
[vps]
|
||||
vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
monitoring ansible_host=64.226.70.190 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
|
||||
[nodito_host]
|
||||
nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
|
|
@ -15,10 +15,11 @@ memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counter
|
|||
forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
nonkeiwaisi_local ansible_host=nonkeiwaisi-box lan_ip=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
|
||||
# Local connection to laptop: this assumes you're running ansible commands from your personal laptop
|
||||
[lapy]
|
||||
localhost ansible_connection=local ansible_user=counterweight
|
||||
localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C
|
||||
|
||||
[arbret]
|
||||
prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua
|
||||
|
|
@ -26,12 +27,8 @@ prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22
|
|||
[edge]
|
||||
vipy
|
||||
|
||||
# The group is `observability`, NOT `monitoring` — there is a HOST named
|
||||
# `monitoring` on line 5, and a group with the same name makes `hosts: monitoring`
|
||||
# ambiguous. Ansible resolved it to the host and warned:
|
||||
# [WARNING]: Found both group and host with same name: monitoring
|
||||
[observability]
|
||||
monitoring
|
||||
[monitoring]
|
||||
watchtower
|
||||
|
||||
[vpn_control]
|
||||
spacey
|
||||
|
|
@ -67,7 +64,7 @@ nodito_vms
|
|||
# Hosts that run Caddy and therefore have /etc/caddy/sites-enabled.
|
||||
[caddy:children]
|
||||
edge
|
||||
observability
|
||||
monitoring
|
||||
vpn_control
|
||||
|
||||
[backup_store]
|
||||
|
|
|
|||
|
|
@ -7,10 +7,6 @@
|
|||
ansible.builtin.include_role:
|
||||
name: backup_store
|
||||
vars:
|
||||
# check-backups.sh reports one result per source plus one for the store
|
||||
# itself, so it needs the collection URL and appends each key.
|
||||
backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints"
|
||||
backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}"
|
||||
backup_store_sources:
|
||||
- name: arbret
|
||||
source: "arbret@prd-arbret:/opt/arbret/backups/"
|
||||
|
|
@ -30,78 +26,3 @@
|
|||
- name: forgejo
|
||||
source: "backup-pull@prd-vipy:/opt/backups/forgejo/"
|
||||
retention_days: 14
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Register the backup checks with Gatus.
|
||||
#
|
||||
# Two groups on purpose, because they answer different questions and fail for
|
||||
# different reasons:
|
||||
#
|
||||
# backup-dump did the SOURCE produce an artefact? Pushed by each dump right
|
||||
# after it runs, so a broken dump is visible within minutes.
|
||||
# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is
|
||||
# retention pruning? Pushed by check-backups.sh at 05:30.
|
||||
#
|
||||
# The store alone could catch almost everything, because the artefact filename
|
||||
# carries the source's dump timestamp - a source whose timer died still pulls
|
||||
# "ok" forever, but the timestamp gives it away. What the source side adds is
|
||||
# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it
|
||||
# cannot tell you whether the dump broke or the pull did.
|
||||
#
|
||||
# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed`
|
||||
# deliberately excludes, so nothing of ours runs there. It is store-checked only.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
- name: Register the backup checks with Gatus
|
||||
hosts: observability
|
||||
become: yes
|
||||
vars:
|
||||
# Sources we deploy the dump for, and the host each one runs on.
|
||||
dump_sources:
|
||||
- {name: headscale, host: spacey}
|
||||
- {name: memos, host: memos_box_local}
|
||||
- {name: vaultwarden, host: vipy}
|
||||
- {name: lnbits, host: vipy}
|
||||
- {name: forgejo, host: vipy}
|
||||
store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo]
|
||||
|
||||
tasks:
|
||||
# 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter
|
||||
# than a day plus slack. The store-side check catches the same fault within
|
||||
# 6h by reading the artefact's dump timestamp out of the filename, so this is
|
||||
# the slow backstop rather than the primary signal.
|
||||
- name: Build the dump endpoint list
|
||||
ansible.builtin.set_fact:
|
||||
dump_endpoints: "{{ dump_endpoints | default([]) + [{
|
||||
'name': item.name,
|
||||
'group': 'backup-dump',
|
||||
'token': gatus_push_tokens[item.host],
|
||||
'heartbeat': '26h'}] }}"
|
||||
loop: "{{ dump_sources }}"
|
||||
|
||||
- name: Build the store endpoint list
|
||||
ansible.builtin.set_fact:
|
||||
store_endpoints: "{{ store_endpoints | default([]) + [{
|
||||
'name': item,
|
||||
'group': 'backup-store',
|
||||
'token': gatus_push_tokens['small_backups_local'],
|
||||
'heartbeat': '7h'}] }}"
|
||||
loop: "{{ store_sources }}"
|
||||
|
||||
- name: Register the backup endpoints
|
||||
ansible.builtin.include_role:
|
||||
name: gatus_endpoint
|
||||
vars:
|
||||
# Push endpoints: the heartbeat window is the tolerance, so alert on
|
||||
# the first failure rather than waiting for three 7h windows to pass.
|
||||
gatus_endpoint_default_alerts:
|
||||
- type: signal
|
||||
failure-threshold: 1
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
minimum-reminder-interval: 12h
|
||||
gatus_endpoint_name: backups
|
||||
gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{
|
||||
'name': 'pull job',
|
||||
'group': 'backup-store',
|
||||
'token': gatus_push_tokens['small_backups_local'],
|
||||
'heartbeat': '7h'}] }}"
|
||||
|
|
|
|||
|
|
@ -29,16 +29,3 @@ backup_source_retention_days: 7
|
|||
|
||||
# Schedule. The box pulls at 04:00, so dumps must land before that.
|
||||
backup_source_on_calendar: "*-*-* 02:00:00"
|
||||
|
||||
# ── Reporting ────────────────────────────────────────────────────────────────
|
||||
# Where to report that this dump ran and produced a plausible artefact.
|
||||
# Gatus external endpoint:
|
||||
# POST {url}?success=true|false&error=...
|
||||
# Authorization: Bearer {token}
|
||||
# Empty is valid and is not an error: the unit's exit code is still the answer,
|
||||
# and the STORE will independently notice a stale dump within ~26h because the
|
||||
# artefact filename carries this dump's timestamp. Reporting here only buys
|
||||
# earlier detection and tells you it was the DUMP that broke rather than the
|
||||
# pull.
|
||||
backup_source_push_url: ""
|
||||
backup_source_push_token: ""
|
||||
|
|
|
|||
|
|
@ -16,27 +16,12 @@
|
|||
|
||||
# Declared here rather than assumed. Stage 1 installed it by hand; this is what
|
||||
# makes a rebuilt host get it too.
|
||||
# Cache refresh is best-effort on purpose. An unrelated third-party repo with a
|
||||
# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource
|
||||
# key) makes `apt-get update` return warnings, which the apt module treats as a
|
||||
# hard failure — and that must not stop backups being configured. Installing the
|
||||
# package is NOT best-effort: if age is genuinely unavailable, the next task fails.
|
||||
- name: Refresh the apt cache (best effort)
|
||||
ansible.builtin.apt:
|
||||
update_cache: yes
|
||||
cache_valid_time: 3600
|
||||
failed_when: false
|
||||
changed_when: false
|
||||
|
||||
- name: Ensure age is installed
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- age
|
||||
# curl is needed only when backup_source_push_url is set, but installing it
|
||||
# unconditionally keeps the task idempotent and it is present on every
|
||||
# Debian host here anyway.
|
||||
- curl
|
||||
name: age
|
||||
state: present
|
||||
update_cache: yes
|
||||
cache_valid_time: 3600
|
||||
|
||||
# The pull account: unprivileged, no sudo, exists only so small-backups-box can
|
||||
# read the dump directory. Trust points one way — the box can read backups, and
|
||||
|
|
@ -88,18 +73,14 @@
|
|||
mode: '0750'
|
||||
validate: "bash -n %s"
|
||||
|
||||
# The .service carries the push token in an Environment= line, so it is 0600.
|
||||
# The .timer holds nothing secret and stays world-readable.
|
||||
- name: "Install the {{ backup_source_name }}-backup systemd units"
|
||||
ansible.builtin.template:
|
||||
src: "backup.{{ item.unit }}.j2"
|
||||
dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}"
|
||||
src: "backup.{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {unit: service, mode: "0600"}
|
||||
- {unit: timer, mode: "0644"}
|
||||
mode: '0644'
|
||||
loop: [service, timer]
|
||||
notify: Reload systemd for backup units
|
||||
|
||||
- name: "Enable the {{ backup_source_name }}-backup timer"
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ After={{ backup_source_stop_service if '.' in backup_source_stop_service else ba
|
|||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh
|
||||
Environment=BACKUP_PUSH_URL={{ backup_source_push_url }}
|
||||
Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }}
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier={{ backup_source_name }}-backup
|
||||
|
|
|
|||
|
|
@ -41,76 +41,18 @@ chmod 700 "$BACKUP_DIR"
|
|||
# here or they accumulate forever.
|
||||
rm -f "${BACKUP_DIR}/${NAME}_"*.partial
|
||||
|
||||
# --- Reporting -------------------------------------------------------------
|
||||
# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed
|
||||
# backup even though the script "finished". Both are reported as failures.
|
||||
PUSH_URL="${BACKUP_PUSH_URL:-}"
|
||||
PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}"
|
||||
|
||||
report() {
|
||||
local success="$1" message="$2"
|
||||
[ -n "$PUSH_URL" ] || return 0
|
||||
local encoded
|
||||
encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g')
|
||||
curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \
|
||||
-H "Authorization: Bearer ${PUSH_TOKEN}" \
|
||||
"${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Reports on ANY exit path, so a dump that dies halfway still reports rather
|
||||
# than going quiet. The size of the FINISHED artefact decides success, not
|
||||
# merely reaching the end of the script.
|
||||
#
|
||||
# This is called FROM the single EXIT trap below - it must never register an
|
||||
# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather
|
||||
# than adding to it, so a second trap here silently discards the one that
|
||||
# restarts the service, and a backup run leaves the service stopped. That is
|
||||
# precisely the failure the restart trap exists to prevent.
|
||||
report_outcome() {
|
||||
local rc="$1"
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
report "false" "${NAME} dump exited ${rc}"
|
||||
elif [ ! -s "$ARTIFACT" ]; then
|
||||
report "false" "${NAME} produced no artefact at ${ARTIFACT}"
|
||||
else
|
||||
report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- One EXIT handler, doing both jobs -------------------------------------
|
||||
# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous
|
||||
# handler rather than adding to it. Registering a second one here would
|
||||
# silently discard the service restart and leave the service stopped after
|
||||
# every backup - which is the exact bug the restart exists to prevent, and it
|
||||
# is invisible until someone notices the service is down.
|
||||
on_exit() {
|
||||
local rc=$?
|
||||
{% if backup_source_stop_service or backup_source_stop_command %}
|
||||
log "Restarting ${SERVICE}..."
|
||||
eval "$START_CMD" || true
|
||||
{% endif %}
|
||||
report_outcome "$rc"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
{% if backup_source_stop_service or backup_source_stop_command %}
|
||||
# --- Stop the service; the trap above guarantees it comes back -------------
|
||||
# --- Stop the service, and guarantee it comes back ---
|
||||
# The trap is the point: without it a failed dump leaves the service down until
|
||||
# the next timer fires. Every hand-written script this replaced had that bug.
|
||||
# It is armed BEFORE the stop, so even a failure during the stop restarts.
|
||||
log "Stopping ${SERVICE}..."
|
||||
eval "$STOP_CMD"
|
||||
trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT
|
||||
{% endif %}
|
||||
|
||||
# --- Dump straight into age; plaintext never touches the disk ---
|
||||
log "Writing ${ARTIFACT}..."
|
||||
{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial"
|
||||
{% if backup_source_pull_user %}
|
||||
# Match the final ownership immediately, so even a partial left by a later
|
||||
# failure is not an unreadable obstacle to the pull.
|
||||
chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial"
|
||||
chmod 640 "${ARTIFACT}.partial"
|
||||
{% endif %}
|
||||
mv "${ARTIFACT}.partial" "$ARTIFACT"
|
||||
{% if backup_source_pull_user %}
|
||||
# Readable by the pull account and nobody else. The contents are age-encrypted
|
||||
|
|
|
|||
|
|
@ -9,21 +9,3 @@ backup_store_on_calendar: "*-*-* 04:00:00"
|
|||
# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/"
|
||||
# retention_days: 90
|
||||
backup_store_sources: []
|
||||
|
||||
# ── Reporting ────────────────────────────────────────────────────────────────
|
||||
# check-backups.sh reports one result PER SOURCE plus one for the store itself,
|
||||
# so the base URL is the endpoints collection and the script appends each key.
|
||||
# Empty is valid: the script still prints its report and exits 0/1.
|
||||
backup_store_check_push_base: ""
|
||||
backup_store_check_push_token: ""
|
||||
|
||||
# Every six hours, offset past the 04:00 pull so the first run of the day sees a
|
||||
# finished pull. The BACKUPS are daily, but this check is not - it reads the
|
||||
# source's dump timestamp out of the artefact filename, so running it more often
|
||||
# catches "the source stopped dumping" within hours rather than a day, and lets
|
||||
# the Gatus heartbeat be 7h instead of 30h.
|
||||
backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00"
|
||||
|
||||
# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the
|
||||
# pull is at 04:00, so 26h tolerates exactly one missed night before alarming.
|
||||
backup_store_check_max_age_hours: 26
|
||||
|
|
|
|||
|
|
@ -33,42 +33,6 @@
|
|||
validate: "bash -n %s"
|
||||
become: yes
|
||||
|
||||
# An assertion that last night actually worked. Generated from the same source
|
||||
# list as the puller, so it can never drift out of sync with what is supposed to
|
||||
# be arriving. Runs on a timer AND is useful by hand.
|
||||
- name: Install the backup check script
|
||||
ansible.builtin.template:
|
||||
src: check-backups.sh.j2
|
||||
dest: /usr/local/bin/check-backups.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
validate: "bash -n %s"
|
||||
become: yes
|
||||
|
||||
# The .service carries the push token, so it is 0600; the .timer is not secret.
|
||||
- name: Install the check-backups systemd units
|
||||
ansible.builtin.template:
|
||||
src: "check-backups.{{ item.unit }}.j2"
|
||||
dest: "/etc/systemd/system/check-backups.{{ item.unit }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {unit: service, mode: "0600"}
|
||||
- {unit: timer, mode: "0644"}
|
||||
become: yes
|
||||
|
||||
# restarted, not started: `started` is a no-op on an already-active timer, so a
|
||||
# changed schedule would never be picked up.
|
||||
- name: Enable the check-backups timer
|
||||
ansible.builtin.systemd:
|
||||
name: check-backups.timer
|
||||
enabled: yes
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
become: yes
|
||||
|
||||
- name: Install the pull-backups systemd units
|
||||
ansible.builtin.template:
|
||||
src: "pull-backups.{{ item }}.j2"
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
[Unit]
|
||||
Description=Verify the nightly backup pull actually worked
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User={{ ansible_user_id }}
|
||||
ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }}
|
||||
Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }}
|
||||
Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }}
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Assert the nightly backups actually worked.
|
||||
#
|
||||
# Run as {{ ansible_user_id }} on this host. Needs no sudo.
|
||||
#
|
||||
# What it CANNOT do: verify contents. The age identity lives only on lapy, so
|
||||
# this host cannot decrypt anything it holds — by design. These are freshness,
|
||||
# completeness and integrity checks. To verify content, decrypt on lapy:
|
||||
# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups/<svc>/<artefact>" \
|
||||
# | age -d -i ~/.age/counterweight_age | tar -tzf - | head
|
||||
#
|
||||
# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL.
|
||||
set -uo pipefail
|
||||
|
||||
STORE="{{ backup_store_dir }}"
|
||||
MAX_AGE_H="${1:-26}" # an artefact older than this is stale
|
||||
NOW=$(date +%s)
|
||||
fails=0; warns=0
|
||||
|
||||
# Colour only when attached to a terminal: this gets piped into files and, later,
|
||||
# probably into a notification.
|
||||
if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m'
|
||||
else R=''; Y=''; G=''; N=''; fi
|
||||
|
||||
# Per-source verdicts, so each source can be reported independently. A single
|
||||
# aggregate red light tells you backups are broken; it does not tell you which
|
||||
# one, which is the thing you need at 3am.
|
||||
declare -A SRC_FAIL SRC_MSG
|
||||
CURRENT=""
|
||||
|
||||
red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1));
|
||||
[ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; }
|
||||
yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); }
|
||||
ok() { printf ' %sok%s %s\n' "$G" "$N" "$*";
|
||||
[ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }
|
||||
|
||||
# --- Reporting -------------------------------------------------------------
|
||||
# Each source gets its own Gatus external endpoint, plus one for the store
|
||||
# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means
|
||||
# report nowhere, which is valid: the exit code is still the whole answer.
|
||||
PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}"
|
||||
PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}"
|
||||
|
||||
report() {
|
||||
local key="$1" success="$2" message="$3"
|
||||
[ -n "$PUSH_BASE" ] || return 0
|
||||
local encoded
|
||||
encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g')
|
||||
curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \
|
||||
-H "Authorization: Bearer ${PUSH_TOKEN}" \
|
||||
"${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true
|
||||
}
|
||||
|
||||
hours_since() { echo $(( (NOW - $1) / 3600 )); }
|
||||
|
||||
# Pull the dump timestamp out of <name>_YYYYmmdd_HHMMSS.<suffix>. This is when
|
||||
# the SOURCE produced it, which is the thing that actually matters: a source
|
||||
# whose timer died still pulls "ok" forever, because yesterday's artefact is
|
||||
# still sitting there. Checking only the pull would miss exactly that.
|
||||
dump_epoch() {
|
||||
local base ts
|
||||
base=$(basename "$1")
|
||||
ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1
|
||||
[ -n "$ts" ] || return 1
|
||||
date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null
|
||||
}
|
||||
|
||||
check_source() {
|
||||
local name="$1" keep="$2" dir="$STORE/$1"
|
||||
printf '\n%s\n' "== $name"
|
||||
CURRENT="$name"
|
||||
SRC_FAIL[$name]=0
|
||||
SRC_MSG[$name]=""
|
||||
|
||||
[ -d "$dir" ] || { red "$name: no directory $dir"; return; }
|
||||
|
||||
local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l)
|
||||
[ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; }
|
||||
|
||||
local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l)
|
||||
[ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these"
|
||||
|
||||
local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1)
|
||||
local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p)
|
||||
|
||||
# 1. Did the SOURCE dump recently?
|
||||
local de; de=$(dump_epoch "$newest")
|
||||
if [ -z "${de:-}" ]; then
|
||||
yell "$name: cannot parse a dump timestamp from $(basename "$newest")"
|
||||
else
|
||||
local dh; dh=$(hours_since "$de")
|
||||
if [ "$dh" -lt 0 ]; then
|
||||
# A future-dated artefact would otherwise stay "fresh" forever and the
|
||||
# staleness check would never fire again — the exact silent failure this
|
||||
# script exists to catch.
|
||||
red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?"
|
||||
elif [ "$dh" -gt "$MAX_AGE_H" ]; then
|
||||
red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run"
|
||||
else
|
||||
ok "$name: dumped ${dh}h ago"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Did the PULL bring it over recently?
|
||||
local ph; ph=$(hours_since "$(stat -c %Y "$newest")")
|
||||
if [ "$ph" -gt "$MAX_AGE_H" ]; then
|
||||
red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)"
|
||||
else
|
||||
ok "$name: pulled ${ph}h ago"
|
||||
fi
|
||||
|
||||
# 3. Is it plausibly a real backup?
|
||||
local sz; sz=$(stat -c %s "$newest")
|
||||
if [ "$sz" -eq 0 ]; then
|
||||
red "$name: newest artefact is ZERO bytes"
|
||||
elif [ -n "$prev" ]; then
|
||||
local psz; psz=$(stat -c %s "$prev")
|
||||
if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then
|
||||
# Not automatically wrong: headscale legitimately shrank 297K -> 20K when
|
||||
# a clean stop checkpointed its write-ahead log into the database.
|
||||
yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect"
|
||||
else
|
||||
ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)"
|
||||
fi
|
||||
else
|
||||
ok "$name: $(numfmt --to=iec "$sz") (first artefact)"
|
||||
fi
|
||||
|
||||
# 4. Is retention pruning? Allow generous slack for multiple dumps per day.
|
||||
if [ "$n" -gt $(( keep * 3 + 10 )) ]; then
|
||||
yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
echo "Artefacts older than ${MAX_AGE_H}h are treated as stale."
|
||||
|
||||
# --- the pull job itself ---
|
||||
# Reserved key, reported as backup-store_pull-job. The store's own machinery is
|
||||
# a different alarm from any one source being stale, and it is a LEADING
|
||||
# indicator where the per-source checks are lagging ones: those only fire once
|
||||
# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer,
|
||||
# a failed pull job or a filling disk are all visible here immediately, and they
|
||||
# name the cause instead of showing six stale sources with no explanation.
|
||||
CURRENT="__store"
|
||||
SRC_FAIL[__store]=0
|
||||
SRC_MSG[__store]=""
|
||||
printf '\n%s\n' "== pull-backups.service"
|
||||
result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null)
|
||||
status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null)
|
||||
when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null)
|
||||
[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})"
|
||||
if [ -n "$when" ]; then
|
||||
wh=$(hours_since "$(date -d "$when" +%s)")
|
||||
[ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?"
|
||||
fi
|
||||
systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \
|
||||
&& ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \
|
||||
|| red "pull-backups.timer is NOT enabled"
|
||||
|
||||
# --- each source ---
|
||||
{% for src in backup_store_sources %}
|
||||
check_source "{{ src.name }}" {{ src.retention_days }}
|
||||
CURRENT=""
|
||||
# The store key is what Gatus computes from group+name: sanitize("backup-store")
|
||||
# + "_" + sanitize("{{ src.name }}").
|
||||
report "backup-store_{{ src.name }}" \
|
||||
"$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \
|
||||
"${SRC_MSG[{{ src.name }}]:-no result}"
|
||||
{% endfor %}
|
||||
|
||||
# --- capacity ---
|
||||
CURRENT="__store"
|
||||
printf '\n%s\n' "== disk"
|
||||
use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9')
|
||||
avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ')
|
||||
if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free"
|
||||
elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free"
|
||||
else ok "store is ${use}% full, ${avail} free"; fi
|
||||
|
||||
CURRENT=""
|
||||
report "backup-store_pull-job" \
|
||||
"$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \
|
||||
"${SRC_MSG[__store]:-no result}"
|
||||
|
||||
printf '\n%s\n' "-----"
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "RESULT: $fails failure(s), $warns warning(s)"
|
||||
echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$warns" -gt 0 ]; then
|
||||
echo "RESULT: all checks passed, $warns warning(s)"
|
||||
else
|
||||
echo "RESULT: all checks passed"
|
||||
fi
|
||||
exit 0
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
[Unit]
|
||||
Description=Run the backup verification after the nightly pull
|
||||
Requires=check-backups.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar={{ backup_store_check_on_calendar }}
|
||||
# Run a missed occurrence on the next boot rather than skipping the day.
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -22,12 +22,7 @@ pull_{{ src.name | replace('-', '_') }}() {
|
|||
local dir="${STORE}/{{ src.name }}"
|
||||
mkdir -p "$dir"
|
||||
log "Pulling {{ src.name }} from {{ src.source }}..."
|
||||
# --exclude '*.partial': a dump that died mid-write leaves one behind, owned
|
||||
# root:root 0600 because the chown only happens after a successful mv. Without
|
||||
# this exclude the pull account cannot read it and rsync fails for the WHOLE
|
||||
# source — so one failed dump would silently block every subsequent pull of
|
||||
# that service. An incomplete artefact is never worth transferring anyway.
|
||||
if rsync -az --timeout=120 --exclude '*.partial' \
|
||||
if rsync -az --timeout=120 \
|
||||
-e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \
|
||||
"{{ src.source }}" "$dir/"; then
|
||||
log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))"
|
||||
|
|
|
|||
|
|
@ -1,85 +0,0 @@
|
|||
# `bitcoin_knots`
|
||||
|
||||
Builds Bitcoin Knots from source with PGP + SHA256 verification of the release
|
||||
tarball, runs it as a full node on `knots-box`, and keeps a health check on a
|
||||
systemd timer. The second play in the calling playbook publishes the P2P port
|
||||
from the edge host via `socket_proxy`.
|
||||
|
||||
Converted from `deploy_bitcoin_knots_playbook.yml` (892 lines) under Plan 6. The
|
||||
playbook is now 40 lines.
|
||||
|
||||
## The build is guarded; the chain is never touched
|
||||
|
||||
`build.yml` is 32 tasks, every one carrying
|
||||
`when: not bitcoind_binary_exists.stat.exists`. On a host that already has the
|
||||
binary the whole download / verify / 30-60 minute compile skips — **including the
|
||||
two `state: absent` deletions**, which target `/opt/bitcoin-knots/source` and the
|
||||
extracted build directory.
|
||||
|
||||
The chain lives elsewhere and nothing here touches it:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `bitcoin_knots_dir` | `/opt/bitcoin-knots` — build tree, safe to delete |
|
||||
| `bitcoin_data_dir` | `/var/lib/bitcoin` — config, logs, wallets |
|
||||
| `bitcoin_large_data_dir` | `/mnt/knots_data` — **~875 GB of blockchain** |
|
||||
|
||||
The signature-verification tasks are the security control of this role. They are
|
||||
copied verbatim; do not "simplify" them.
|
||||
|
||||
## ⚠ This node is half of the mining setup
|
||||
|
||||
`bitcoin.conf` carries a DATUM Gateway section that was hand-added on the node
|
||||
and was **missing from the playbook's template**:
|
||||
|
||||
```ini
|
||||
blockmaxsize=3985000
|
||||
blockmaxweight=3985000
|
||||
blocknotify=killall -USR1 datum_gateway
|
||||
maxmempool=1000
|
||||
blockreconstructionextratxn=1000000
|
||||
```
|
||||
|
||||
`blocknotify` is how `datum_gateway` learns a new block landed. Running the old
|
||||
playbook would have deleted all of it, and solo mining would have carried on
|
||||
grinding against a stale template — a silent failure that costs money rather
|
||||
than raising an error. The template now carries it behind
|
||||
`bitcoin_datum_gateway_enabled`.
|
||||
|
||||
**bitcoin-knots and datum-gateway are one system, not two services.** Changing
|
||||
either config means thinking about both.
|
||||
|
||||
## The restart handler, and why exactness matters now
|
||||
|
||||
The hand-written `Restart bitcoind` handler carried
|
||||
`when: uptime_kuma_enabled | default(false)`, so it had been inert since the
|
||||
decommissioning: `bitcoin.conf` and the systemd unit both notify it and neither
|
||||
could restart anything. A config change applied to disk, reported success, and
|
||||
never took effect.
|
||||
|
||||
It is ungated here — which raises the bar for the template. **Any** residual
|
||||
difference between the template and the live file, down to a trailing newline,
|
||||
means the task reports `changed` and restarts a Bitcoin node on every run. It
|
||||
took four rounds of `--check --diff` to reach `changed=0`: the DATUM section, an
|
||||
explanatory comment that was rendering into the deployed file (now a `{# #}`
|
||||
Jinja comment), a `# Pruning (optional)` comment the live file had, and one
|
||||
trailing blank line.
|
||||
|
||||
## `dbcache`
|
||||
|
||||
Computed as 90% of RAM unless `bitcoin_dbcache_mb_override` is set. The live node
|
||||
was hand-tuned to **200 MB**; the calculation produces 3528. As with fulcrum,
|
||||
`set_fact` outranks role defaults, so the *calculation* honours the override — a
|
||||
value pinned only in `defaults/` is silently ignored.
|
||||
|
||||
## Monitoring: one variable, no product knowledge
|
||||
|
||||
The check tests bitcoind's RPC and records the answer in its exit code, which
|
||||
systemd keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. Set
|
||||
`healthcheck_push_url` to report anywhere that accepts an HTTP ping.
|
||||
|
||||
The timer had last fired **2026-08-09** while still reporting `active` and
|
||||
`enabled` — the same `OnBootSec` + `OnUnitActiveSec` dead chain as fulcrum, where
|
||||
nothing re-arms it if the service does not run in a given boot. The role runs the
|
||||
check once after enabling, which both smoke-tests it and supplies the reference
|
||||
the timer schedules from.
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
# Bitcoin Knots Configuration Variables
|
||||
|
||||
# Version - REQUIRED: Specify exact version/tag to build
|
||||
# The only version string. There used to be a second, v-prefixed copy
|
||||
# (bitcoin_knots_version) that nothing read - two hand-maintained copies of one
|
||||
# fact, with nothing keeping them in step.
|
||||
bitcoin_knots_version_short: "29.2.knots20251110"
|
||||
|
||||
# Directories
|
||||
bitcoin_knots_dir: /opt/bitcoin-knots
|
||||
bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source"
|
||||
bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets
|
||||
bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate)
|
||||
bitcoin_conf_dir: /etc/bitcoin
|
||||
|
||||
# Network
|
||||
bitcoin_rpc_port: 8332
|
||||
# The edge host's socket-proxy/Caddy play needs this too, and a role default is
|
||||
# invisible outside this role. The authoritative value for the live deployment is
|
||||
# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the
|
||||
# protocol standard, so the role still works standalone.
|
||||
bitcoin_p2p_port: 8333
|
||||
bitcoin_rpc_bind: "0.0.0.0"
|
||||
|
||||
# Build options
|
||||
bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores
|
||||
bitcoin_build_prefix: /usr/local
|
||||
|
||||
# Configuration options
|
||||
bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX)
|
||||
bitcoin_max_connections: 125
|
||||
# dbcache will be calculated as 90% of host RAM automatically in playbook
|
||||
|
||||
# ZMQ Configuration
|
||||
bitcoin_zmq_enabled: true
|
||||
bitcoin_zmq_bind: "tcp://0.0.0.0"
|
||||
bitcoin_zmq_port_rawblock: 28332
|
||||
bitcoin_zmq_port_rawtx: 28333
|
||||
bitcoin_zmq_port_hashblock: 28334
|
||||
bitcoin_zmq_port_hashtx: 28335
|
||||
|
||||
# Service user
|
||||
bitcoin_user: bitcoin
|
||||
bitcoin_group: bitcoin
|
||||
|
||||
# --- Health check ----------------------------------------------------------
|
||||
# Checks bitcoind RPC and records the answer in its exit code, which systemd
|
||||
# keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`.
|
||||
#
|
||||
# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means
|
||||
# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping
|
||||
# works; nothing here is specific to a monitoring product.
|
||||
healthcheck_push_url: ""
|
||||
# Bearer token for the Gatus external endpoint. Required whenever a push URL
|
||||
# is set: Gatus rejects an unauthenticated push with 401.
|
||||
healthcheck_push_token: ""
|
||||
|
||||
# --- Logging ----------------------------------------------------------------
|
||||
# The live node logs to a file. Set to "" to use printtoconsole=1 (journald).
|
||||
bitcoin_logfile: "{{ bitcoin_data_dir }}/debug.log"
|
||||
|
||||
# --- dbcache ----------------------------------------------------------------
|
||||
# Computed as 90% of RAM unless this is set. The live node was hand-tuned to
|
||||
# 200 MB; the calculation would have produced 3528. As with fulcrum, note that
|
||||
# set_fact outranks role defaults, so the CALCULATION has to honour this - a
|
||||
# value pinned only in defaults/ is silently ignored.
|
||||
bitcoin_dbcache_mb_override: 200
|
||||
|
||||
# --- DATUM Gateway ----------------------------------------------------------
|
||||
# This node feeds block templates to datum_gateway on knots-box. These settings
|
||||
# were hand-added to bitcoin.conf and were missing from the template, so a
|
||||
# playbook run would have removed them and broken the mining setup.
|
||||
bitcoin_datum_gateway_enabled: true
|
||||
bitcoin_blockmaxsize: 3985000
|
||||
bitcoin_blockmaxweight: 3985000
|
||||
bitcoin_blocknotify: "killall -USR1 datum_gateway"
|
||||
bitcoin_maxmempool: 1000
|
||||
bitcoin_blockreconstructionextratxn: 1000000
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
---
|
||||
# Ungated on purpose. The hand-written handler carried
|
||||
# when: uptime_kuma_enabled | default(false)
|
||||
# so it has been inert since the decommissioning. Two tasks notify it —
|
||||
# bitcoin.conf and the systemd unit — and neither could actually restart
|
||||
# bitcoind. A configuration change to a Bitcoin node therefore applied to disk,
|
||||
# reported success, and silently never took effect.
|
||||
#
|
||||
# Restarting bitcoind re-opens the chainstate; it does not reindex.
|
||||
- name: Restart bitcoind
|
||||
systemd:
|
||||
name: bitcoind
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
|
@ -1,222 +0,0 @@
|
|||
---
|
||||
# Every task here is guarded by `when: not bitcoind_binary_exists.stat.exists`,
|
||||
# so on a host that already has the binary the whole download / verify / build
|
||||
# sequence skips — including the two `state: absent` deletions, which target
|
||||
# /opt/bitcoin-knots/{source,bitcoin-<version>} and never the chain data in
|
||||
# /mnt/knots_data.
|
||||
- name: Check if bitcoind binary already exists
|
||||
stat:
|
||||
path: "{{ bitcoin_build_prefix }}/bin/bitcoind"
|
||||
register: bitcoind_binary_exists
|
||||
changed_when: false
|
||||
|
||||
- name: Install gnupg for signature verification
|
||||
apt:
|
||||
name: gnupg
|
||||
state: present
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Import Luke Dashjr's Bitcoin Knots signing key
|
||||
command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964
|
||||
register: key_import
|
||||
changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout"
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
failed_when: key_import.rc != 0
|
||||
|
||||
- name: Display imported key fingerprint
|
||||
command: gpg --fingerprint 90C8019E36C2E964
|
||||
register: key_fingerprint
|
||||
changed_when: false
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Download SHA256SUMS file
|
||||
get_url:
|
||||
url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS"
|
||||
dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS"
|
||||
mode: '0644'
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Download SHA256SUMS.asc signature file
|
||||
get_url:
|
||||
url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc"
|
||||
dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc"
|
||||
mode: '0644'
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Verify PGP signature on SHA256SUMS file
|
||||
command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS
|
||||
register: sha256sums_verification
|
||||
changed_when: false
|
||||
failed_when: false # Don't fail here - check for 'Good signature' in next task
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
|
||||
- name: Display SHA256SUMS verification result
|
||||
debug:
|
||||
msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}"
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Fail if SHA256SUMS signature verification failed
|
||||
fail:
|
||||
msg: "SHA256SUMS signature verification failed. Aborting build."
|
||||
when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr)
|
||||
|
||||
- name: Remove any existing tarball to force fresh download
|
||||
file:
|
||||
path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz
|
||||
state: absent
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Download Bitcoin Knots source tarball
|
||||
get_url:
|
||||
url: "{{ bitcoin_source_tarball_url }}"
|
||||
dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz"
|
||||
mode: '0644'
|
||||
validate_certs: yes
|
||||
force: yes
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Calculate SHA256 checksum of downloaded tarball
|
||||
command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz
|
||||
register: tarball_checksum
|
||||
changed_when: false
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Extract expected checksum from SHA256SUMS file
|
||||
shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}'
|
||||
register: expected_checksum
|
||||
changed_when: false
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
failed_when: expected_checksum.stdout == ""
|
||||
|
||||
- name: Display checksum comparison
|
||||
debug:
|
||||
msg:
|
||||
- "Expected: {{ expected_checksum.stdout | trim }}"
|
||||
- "Actual: {{ tarball_checksum.stdout.split()[0] }}"
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Verify tarball checksum matches SHA256SUMS
|
||||
fail:
|
||||
msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}"
|
||||
when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0]
|
||||
|
||||
- name: Remove existing source directory if it exists (to force fresh extraction)
|
||||
file:
|
||||
path: "{{ bitcoin_knots_source_dir }}"
|
||||
state: absent
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Remove extracted directory if it exists (from previous runs)
|
||||
file:
|
||||
path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}"
|
||||
state: absent
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Extract verified source tarball
|
||||
unarchive:
|
||||
src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz
|
||||
dest: "{{ bitcoin_knots_dir }}"
|
||||
remote_src: yes
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Check if extracted directory exists
|
||||
stat:
|
||||
path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}"
|
||||
register: extracted_dir_stat
|
||||
changed_when: false
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Rename extracted directory to expected name
|
||||
command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}"
|
||||
when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists
|
||||
|
||||
- name: Check if CMakeLists.txt exists
|
||||
stat:
|
||||
path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt"
|
||||
register: cmake_exists
|
||||
changed_when: false
|
||||
when: not bitcoind_binary_exists.stat.exists
|
||||
|
||||
- name: Create CMake build directory
|
||||
file:
|
||||
path: "{{ bitcoin_knots_source_dir }}/build"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
|
||||
|
||||
- name: Configure Bitcoin Knots build with CMake
|
||||
command: >
|
||||
cmake
|
||||
-DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }}
|
||||
-DBUILD_BITCOIN_WALLET=OFF
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
-DWITH_ZMQ=ON
|
||||
..
|
||||
args:
|
||||
chdir: "{{ bitcoin_knots_source_dir }}/build"
|
||||
when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
|
||||
register: configure_result
|
||||
changed_when: true
|
||||
|
||||
- name: Verify CMake enabled ZMQ
|
||||
shell: |
|
||||
set -e
|
||||
cd "{{ bitcoin_knots_source_dir }}/build"
|
||||
cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ'
|
||||
when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
|
||||
register: zmq_check
|
||||
changed_when: false
|
||||
|
||||
- name: Fail if CMakeLists.txt not found
|
||||
fail:
|
||||
msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots."
|
||||
when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false))
|
||||
|
||||
- name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes)
|
||||
command: cmake --build . -j{{ bitcoin_build_jobs }}
|
||||
args:
|
||||
chdir: "{{ bitcoin_knots_source_dir }}/build"
|
||||
when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
|
||||
async: 3600
|
||||
poll: 0
|
||||
register: build_result
|
||||
changed_when: true
|
||||
|
||||
- name: Check build status
|
||||
async_status:
|
||||
jid: "{{ build_result.ansible_job_id }}"
|
||||
register: build_job_result
|
||||
until: build_job_result.finished
|
||||
retries: 120
|
||||
delay: 60
|
||||
when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined
|
||||
|
||||
- name: Fail if build failed
|
||||
fail:
|
||||
msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}"
|
||||
when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false)
|
||||
|
||||
- name: Install Bitcoin Knots binaries
|
||||
command: cmake --install .
|
||||
args:
|
||||
chdir: "{{ bitcoin_knots_source_dir }}/build"
|
||||
when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false)
|
||||
changed_when: true
|
||||
|
||||
- name: Verify bitcoind binary exists
|
||||
stat:
|
||||
path: "{{ bitcoin_build_prefix }}/bin/bitcoind"
|
||||
register: bitcoind_installed
|
||||
changed_when: false
|
||||
|
||||
- name: Verify bitcoin-cli binary exists
|
||||
stat:
|
||||
path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli"
|
||||
register: bitcoin_cli_installed
|
||||
changed_when: false
|
||||
|
||||
- name: Fail if binaries not found
|
||||
fail:
|
||||
msg: "Bitcoin Knots binaries not found after installation"
|
||||
when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
# Ownership copied verbatim from the playbook this replaces; verified
|
||||
# mechanically against `git show HEAD:` rather than retyped from memory.
|
||||
- name: Create bitcoin.conf configuration file
|
||||
ansible.builtin.template:
|
||||
src: bitcoin.conf.j2
|
||||
dest: "{{ bitcoin_conf_dir }}/bitcoin.conf"
|
||||
owner: "{{ bitcoin_user }}"
|
||||
group: "{{ bitcoin_group }}"
|
||||
mode: '0640'
|
||||
notify: Restart bitcoind
|
||||
|
||||
- name: Create systemd service file for bitcoind
|
||||
ansible.builtin.template:
|
||||
src: bitcoind.service.j2
|
||||
dest: /etc/systemd/system/bitcoind.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
notify: Restart bitcoind
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
---
|
||||
# Everything here answers "is bitcoind healthy" and records the answer. The
|
||||
# Uptime Kuma specifics that used to follow — an embedded Python script creating
|
||||
# monitors over the API, a /tmp credentials file, push-URL extraction and a
|
||||
# systemd Environment= rewrite — are gone. Where it reports is now one variable,
|
||||
# healthcheck_push_url. See the role README.
|
||||
- name: Install curl for health check script
|
||||
apt:
|
||||
name: curl
|
||||
state: present
|
||||
|
||||
- name: Create Bitcoin Knots health check script
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.sh.j2
|
||||
dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
validate: "bash -n %s"
|
||||
|
||||
- name: Create systemd service for Bitcoin Knots health check
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.service.j2
|
||||
dest: /etc/systemd/system/bitcoin-knots-healthcheck.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
- name: Create systemd timer for Bitcoin Knots health check
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.timer.j2
|
||||
dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd daemon for health check
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and restart the Bitcoin Knots health check timer
|
||||
systemd:
|
||||
name: bitcoin-knots-healthcheck.timer
|
||||
enabled: yes
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
||||
# Runs the check once, which is both a smoke test and the thing that actually
|
||||
# arms the timer. This timer is OnBootSec + OnUnitActiveSec with no OnCalendar:
|
||||
# OnBootSec elapses once, and OnUnitActiveSec needs the SERVICE to have run this
|
||||
# boot to have anything to schedule from. Restarting the timer does not supply
|
||||
# that reference; running the service does. The live timer had last fired on
|
||||
# 2026-08-09 while still reporting `active` and `enabled`.
|
||||
- name: Run the Bitcoin Knots health check once to arm the timer
|
||||
command: systemctl start bitcoin-knots-healthcheck.service
|
||||
changed_when: false
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
---
|
||||
- name: Calculate dbcache as a share of system RAM
|
||||
set_fact:
|
||||
bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}"
|
||||
when: bitcoin_dbcache_mb_override | string | length == 0
|
||||
|
||||
- name: Use the explicit dbcache override
|
||||
set_fact:
|
||||
bitcoin_dbcache_mb: "{{ bitcoin_dbcache_mb_override }}"
|
||||
when: bitcoin_dbcache_mb_override | string | length > 0
|
||||
changed_when: false
|
||||
|
||||
- name: Display calculated dbcache value
|
||||
debug:
|
||||
msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)"
|
||||
|
||||
|
||||
- name: Install build dependencies
|
||||
apt:
|
||||
name:
|
||||
- build-essential
|
||||
- libtool
|
||||
- autotools-dev
|
||||
- automake
|
||||
- pkg-config
|
||||
- bsdmainutils
|
||||
- python3
|
||||
- python3-pip
|
||||
- libevent-dev
|
||||
- libboost-system-dev
|
||||
- libboost-filesystem-dev
|
||||
- libboost-test-dev
|
||||
- libboost-thread-dev
|
||||
- libboost-chrono-dev
|
||||
- libboost-program-options-dev
|
||||
- libboost-dev
|
||||
- libssl-dev
|
||||
- libdb-dev
|
||||
- libminiupnpc-dev
|
||||
- libzmq3-dev
|
||||
- libnatpmp-dev
|
||||
- libsqlite3-dev
|
||||
- git
|
||||
- curl
|
||||
- wget
|
||||
- cmake
|
||||
state: present
|
||||
update_cache: yes
|
||||
|
||||
- name: Create bitcoin group
|
||||
group:
|
||||
name: "{{ bitcoin_group }}"
|
||||
system: yes
|
||||
state: present
|
||||
|
||||
- name: Create bitcoin user
|
||||
user:
|
||||
name: "{{ bitcoin_user }}"
|
||||
group: "{{ bitcoin_group }}"
|
||||
system: yes
|
||||
shell: /usr/sbin/nologin
|
||||
home: "{{ bitcoin_data_dir }}"
|
||||
create_home: yes
|
||||
state: present
|
||||
|
||||
- name: Create bitcoin-knots directory
|
||||
file:
|
||||
path: "{{ bitcoin_knots_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create bitcoin-knots source directory
|
||||
file:
|
||||
path: "{{ bitcoin_knots_source_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create bitcoin data directory (for config, logs, wallets)
|
||||
file:
|
||||
path: "{{ bitcoin_data_dir }}"
|
||||
state: directory
|
||||
owner: "{{ bitcoin_user }}"
|
||||
group: "{{ bitcoin_group }}"
|
||||
mode: '0750'
|
||||
|
||||
- name: Create bitcoin large data directory (for blockchain)
|
||||
file:
|
||||
path: "{{ bitcoin_large_data_dir }}"
|
||||
state: directory
|
||||
owner: "{{ bitcoin_user }}"
|
||||
group: "{{ bitcoin_group }}"
|
||||
mode: '0750'
|
||||
|
||||
- name: Create bitcoin config directory
|
||||
file:
|
||||
path: "{{ bitcoin_conf_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
---
|
||||
# import_tasks, not include_tasks: static imports stay visible to --list-tasks,
|
||||
# which is how this conversion was verified against the playbook it replaced.
|
||||
- ansible.builtin.import_tasks: install.yml
|
||||
- ansible.builtin.import_tasks: build.yml
|
||||
- ansible.builtin.import_tasks: configure.yml
|
||||
- ansible.builtin.import_tasks: service.yml
|
||||
- ansible.builtin.import_tasks: healthcheck.yml
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
---
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start bitcoind service
|
||||
systemd:
|
||||
name: bitcoind
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Wait for bitcoind RPC to be available
|
||||
uri:
|
||||
url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}"
|
||||
method: POST
|
||||
body_format: json
|
||||
body:
|
||||
jsonrpc: "1.0"
|
||||
id: "healthcheck"
|
||||
method: "getblockchaininfo"
|
||||
params: []
|
||||
user: "{{ bitcoin_rpc_user }}"
|
||||
password: "{{ bitcoin_rpc_password }}"
|
||||
status_code: 200
|
||||
timeout: 10
|
||||
register: rpc_check
|
||||
until: rpc_check.status == 200
|
||||
retries: 30
|
||||
delay: 5
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Display RPC connection status
|
||||
debug:
|
||||
msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
# Bitcoin Knots Configuration
|
||||
# Generated by Ansible
|
||||
|
||||
# Data directory (blockchain storage)
|
||||
datadir={{ bitcoin_large_data_dir }}
|
||||
|
||||
# RPC Configuration
|
||||
server=1
|
||||
rpcuser={{ bitcoin_rpc_user }}
|
||||
rpcpassword={{ bitcoin_rpc_password }}
|
||||
rpcbind={{ bitcoin_rpc_bind }}
|
||||
rpcport={{ bitcoin_rpc_port }}
|
||||
rpcallowip=0.0.0.0/0
|
||||
|
||||
# Network Configuration
|
||||
listen=1
|
||||
port={{ bitcoin_p2p_port }}
|
||||
maxconnections={{ bitcoin_max_connections }}
|
||||
|
||||
# Performance
|
||||
dbcache={{ bitcoin_dbcache_mb }}
|
||||
|
||||
# Transaction Index (optional)
|
||||
{% if bitcoin_enable_txindex %}
|
||||
txindex=1
|
||||
{% endif %}
|
||||
|
||||
{# The live node carries this comment and the template never produced it, so a
|
||||
run would have silently deleted it. Harmless in itself, but matching it keeps
|
||||
this task at `ok` - which means any future `changed` here is a real signal
|
||||
rather than known noise. #}
|
||||
# Pruning (optional)
|
||||
|
||||
# Logging
|
||||
logtimestamps=1
|
||||
{% if bitcoin_logfile %}
|
||||
logfile={{ bitcoin_logfile }}
|
||||
{% else %}
|
||||
printtoconsole=1
|
||||
{% endif %}
|
||||
|
||||
# ZMQ Configuration
|
||||
{% if bitcoin_zmq_enabled | default(false) %}
|
||||
zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }}
|
||||
zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }}
|
||||
zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }}
|
||||
zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }}
|
||||
{% endif %}
|
||||
|
||||
# Security
|
||||
disablewallet=1
|
||||
{% if bitcoin_datum_gateway_enabled %}
|
||||
|
||||
{# These were hand-added on the node and were NOT in this template, so running
|
||||
the playbook would have stripped them. blocknotify is how datum_gateway
|
||||
learns a new block landed; without it solo mining keeps grinding on a stale
|
||||
template - a silent failure that costs money rather than raising an error.
|
||||
Kept as a Jinja comment so the explanation stays in the repo and out of the
|
||||
deployed config. #}
|
||||
# Specific for DATUM gateway
|
||||
blockmaxsize={{ bitcoin_blockmaxsize }}
|
||||
blockmaxweight={{ bitcoin_blockmaxweight }}
|
||||
blocknotify={{ bitcoin_blocknotify }}
|
||||
maxmempool={{ bitcoin_maxmempool }}
|
||||
blockreconstructionextratxn={{ bitcoin_blockreconstructionextratxn }}
|
||||
|
||||
{% endif %}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
[Unit]
|
||||
Description=Bitcoin Knots daemon
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ bitcoin_user }}
|
||||
Group={{ bitcoin_group }}
|
||||
ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
TimeoutStopSec=600
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
[Unit]
|
||||
Description=Bitcoin Knots Health Check
|
||||
After=network.target bitcoind.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh
|
||||
Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }}
|
||||
Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }}
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Bitcoin Knots health check — managed by Ansible (roles/bitcoin_knots)
|
||||
#
|
||||
# The exit code is the answer and systemd keeps it:
|
||||
# systemctl is-failed bitcoin-knots-healthcheck.service
|
||||
# Reporting anywhere else is optional and generic.
|
||||
#
|
||||
#
|
||||
|
||||
RPC_HOST="{{ bitcoin_rpc_bind }}"
|
||||
RPC_PORT={{ bitcoin_rpc_port }}
|
||||
RPC_USER="{{ bitcoin_rpc_user }}"
|
||||
RPC_PASSWORD="{{ bitcoin_rpc_password }}"
|
||||
PUSH_URL="${HEALTHCHECK_PUSH_URL:-}"
|
||||
PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}"
|
||||
|
||||
# Check if bitcoind RPC is responding
|
||||
check_bitcoind() {
|
||||
local response
|
||||
response=$(curl -s --max-time 30 \
|
||||
--user "${RPC_USER}:${RPC_PASSWORD}" \
|
||||
--data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \
|
||||
--header 'Content-Type: application/json' \
|
||||
"http://${RPC_HOST}:${RPC_PORT}" 2>&1)
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
# Check if response contains a non-null error
|
||||
# Successful responses have "error": null, failures have "error": {...}
|
||||
if echo "$response" | grep -q '"error":null\|"error": null'; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
report() {
|
||||
local status=$1
|
||||
local msg=$2
|
||||
|
||||
# No push URL is normal, not an error: the exit code below is still a
|
||||
# complete answer for anything reading unit state.
|
||||
[ -n "$PUSH_URL" ] || return 0
|
||||
|
||||
# URL encode spaces in message
|
||||
local encoded_msg="${msg// /%20}"
|
||||
|
||||
# Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's
|
||||
# GET with ?status=up. The callers still pass up/down, so the mapping is
|
||||
# done here rather than at every call site.
|
||||
local _ok=false
|
||||
[ "${status}" = "up" ] && _ok=true
|
||||
if ! curl -s --max-time 15 --retry 2 -o /dev/null -X POST \
|
||||
-H "Authorization: Bearer ${PUSH_TOKEN}" \
|
||||
"${PUSH_URL}?success=${_ok}&error=${encoded_msg}"; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main health check
|
||||
if check_bitcoind; then
|
||||
report "up" "OK"
|
||||
exit 0
|
||||
else
|
||||
report "down" "bitcoind RPC not responding"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
[Unit]
|
||||
Description=Bitcoin Knots Health Check Timer
|
||||
Requires=bitcoind.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=1min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -15,8 +15,7 @@ caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"}
|
|||
# expression for the username silently passes through as literal text.
|
||||
caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}]
|
||||
|
||||
# Placement. This is now the only definition of caddy_sites_dir - services_config.yml
|
||||
# used to carry an identical copy, which was removed as redundant.
|
||||
# Placement. caddy_sites_dir comes from services_config.yml; this is the fallback.
|
||||
caddy_sites_dir: /etc/caddy/sites-enabled
|
||||
|
||||
# Rendered site files can carry credentials (basic_auth hashes), so --diff is
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
# `datum_gateway`
|
||||
|
||||
Builds and runs [DATUM Gateway](https://github.com/OCEAN-xyz/datum_gateway), the
|
||||
solo/pooled mining gateway, on `knots-box`. The calling playbook adds two more
|
||||
plays on the edge host: the dashboard via `caddy_site`, and the public Stratum
|
||||
port via `socket_proxy`.
|
||||
|
||||
Converted from `deploy_datum_gateway_playbook.yml` (802 lines) under Plan 6. The
|
||||
playbook is now 68 lines and keeps all three plays.
|
||||
|
||||
## ⚠ This is half of a system
|
||||
|
||||
The Bitcoin Knots node on the same host feeds this gateway through
|
||||
`blocknotify=killall -USR1 datum_gateway` in `bitcoin.conf` — see
|
||||
`roles/bitcoin_knots/README.md`, where that line was found to be missing from the
|
||||
template entirely. **Changing either config means thinking about both.**
|
||||
|
||||
Interrupting Stratum costs mining shares. Check before any run that restarts it:
|
||||
|
||||
```bash
|
||||
ss -tn state established '( sport = :23334 )'
|
||||
```
|
||||
|
||||
## Two pieces of drift where the node was right
|
||||
|
||||
The repo and the node had diverged on values that matter, and the deployment
|
||||
would have applied the repo's:
|
||||
|
||||
| | node (correct) | repo said |
|
||||
|---|---|---|
|
||||
| `datum_mining_address` | `bc1qvrj3g84…` | `bc1qdse9dsg…` |
|
||||
| `pool_pass_workers` / `_full_users` | `false` | `true` |
|
||||
|
||||
The address is the one that would have hurt: **it is where block rewards are
|
||||
paid**, and unlike fulcrum and bitcoin-knots the `Restart datum-gateway` handler
|
||||
here was *never* gated, so the change would have applied immediately rather than
|
||||
sitting inert. Both corrected in the vault and defaults, with notes.
|
||||
|
||||
Verify semantics rather than text when touching `config.json` — render it and
|
||||
compare parsed JSON, because the live file is single-line and the template is
|
||||
pretty-printed, so a textual diff is all noise:
|
||||
|
||||
```python
|
||||
json.load(open('live.json')) == json.load(open('rendered.json'))
|
||||
```
|
||||
|
||||
## `config.json` holds real secrets — diff is suppressed
|
||||
|
||||
The file carries `bitcoind.rpcpassword` and `api.admin_password`. `--diff`
|
||||
prints rendered content, so the task sets `diff: false` by default; pass
|
||||
`-e datum_reveal_config=true` to opt in.
|
||||
|
||||
Note `pool_pass_workers` / `pool_pass_full_users` are **booleans**, not
|
||||
passwords, despite the names — they control DATUM's pool-password passthrough.
|
||||
`mining.pool_address` is a Bitcoin address and public by nature.
|
||||
|
||||
## Expect `changed` on the compile every run
|
||||
|
||||
`Configure cmake build` and `Compile datum_gateway` are bare `command:` tasks
|
||||
with no `changed_when`, so they always report changed and always re-run. The
|
||||
build is reproducible — `Install datum_gateway binary` sees identical content and
|
||||
does not replace it, so the installed binary keeps its original timestamp — but
|
||||
the compile itself is wasted work on every run. That is the idempotent floor, not
|
||||
drift.
|
||||
|
||||
## Monitoring: one variable, no product knowledge
|
||||
|
||||
The check tests the gateway API and records the answer in its exit code, which
|
||||
systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. Set
|
||||
`healthcheck_push_url` to report anywhere accepting an HTTP ping.
|
||||
|
||||
Unlike the other services here, only the health-check *timer* handler was gated
|
||||
by `uptime_kuma_enabled`; the main deployment restart worked throughout.
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# DATUM Gateway Configuration Variables
|
||||
# https://github.com/OCEAN-xyz/datum_gateway
|
||||
|
||||
# Version - pin to a specific tag
|
||||
datum_gateway_version: "v0.4.1beta"
|
||||
|
||||
# Directories
|
||||
datum_gateway_dir: /opt/datum-gateway
|
||||
datum_gateway_source_dir: "{{ datum_gateway_dir }}/source"
|
||||
datum_gateway_config_dir: /etc/datum-gateway
|
||||
datum_gateway_log_dir: /var/log/datum-gateway
|
||||
|
||||
# Binary
|
||||
datum_gateway_bin_path: /usr/local/bin/datum_gateway
|
||||
|
||||
# Ports
|
||||
# The edge host's socket-proxy/Caddy play needs this too, and a role default is
|
||||
# invisible outside this role. The authoritative value for the live deployment is
|
||||
# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the
|
||||
# protocol standard, so the role still works standalone.
|
||||
datum_gateway_stratum_port: 23334
|
||||
datum_gateway_api_port: 7152
|
||||
|
||||
# Stratum settings
|
||||
datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher)
|
||||
|
||||
# Service user
|
||||
datum_gateway_user: datum
|
||||
datum_gateway_group: datum
|
||||
|
||||
# Build options
|
||||
datum_gateway_build_jobs: 4
|
||||
|
||||
# Bitcoin node connection
|
||||
# The gateway runs on the same host as Bitcoin Knots so localhost RPC works.
|
||||
# datum_bitcoin_rpc_url should include http:// and port.
|
||||
datum_bitcoin_rpc_url: "http://127.0.0.1:8332"
|
||||
# Note: bitcoin_rpc_user and bitcoin_rpc_password come from group_vars/all/vault.yml
|
||||
|
||||
# Mining config
|
||||
datum_coinbase_tag_primary: "DATUM"
|
||||
datum_coinbase_tag_secondary: "BY ORDER OF BIP110"
|
||||
|
||||
# Both false on the node; the vars file said true. Corrected 2026-09-13 to
|
||||
# match reality, on the same basis as datum_mining_address: the running node
|
||||
# is authoritative. These control DATUM's pool-password passthrough.
|
||||
datum_pool_pass_workers: false
|
||||
datum_pool_pass_full_users: false
|
||||
datum_pooled_mining_only: true
|
||||
|
||||
|
||||
# --- Health check -----------------------------------------------------------
|
||||
# Checks the DATUM Gateway API and records the answer in its exit code, which
|
||||
# systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`.
|
||||
#
|
||||
# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means
|
||||
# check, exit honestly, report nowhere.
|
||||
healthcheck_push_url: ""
|
||||
# Bearer token for the Gatus external endpoint. Required whenever a push URL
|
||||
# is set: Gatus rejects an unauthenticated push with 401.
|
||||
healthcheck_push_token: ""
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
- name: Restart datum-gateway
|
||||
systemd:
|
||||
name: datum-gateway
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
||||
# Ungated. This one carried `when: uptime_kuma_enabled | default(false)` while
|
||||
# the main Restart datum-gateway handler above did not — so on this service the
|
||||
# deployment restart worked and only the health-check timer restart was dead.
|
||||
- name: Restart datum-gateway health check timer
|
||||
systemd:
|
||||
name: datum-gateway-healthcheck.timer
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
---
|
||||
# Ownership copied verbatim from the playbook this replaces and verified
|
||||
# mechanically against `git show HEAD:`.
|
||||
- name: Write DATUM Gateway config.json
|
||||
ansible.builtin.template:
|
||||
src: config.json.j2
|
||||
dest: "{{ datum_gateway_config_dir }}/config.json"
|
||||
owner: "{{ datum_gateway_user }}"
|
||||
group: "{{ datum_gateway_group }}"
|
||||
mode: '0640'
|
||||
# config.json carries the bitcoind RPC password, the API admin password and
|
||||
# the pool passwords. `--diff` prints rendered content, so running with --diff
|
||||
# put all of them on the terminal and into any log capturing it. Suppressed by
|
||||
# default; pass -e datum_reveal_config=true when you genuinely need the diff.
|
||||
diff: "{{ datum_reveal_config | default(false) | bool }}"
|
||||
notify: Restart datum-gateway
|
||||
|
||||
- name: Create datum-gateway systemd service
|
||||
ansible.builtin.template:
|
||||
src: datum-gateway.service.j2
|
||||
dest: /etc/systemd/system/datum-gateway.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
notify: Restart datum-gateway
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
---
|
||||
# Everything here answers "is DATUM Gateway healthy" and records the answer. The
|
||||
# Uptime Kuma specifics that used to follow — an embedded Python script creating
|
||||
# monitors over the API, a /tmp credentials file, a push-URL file read back and
|
||||
# parsed, and a systemd Environment= rewrite — are gone. Where it reports is now
|
||||
# one variable, healthcheck_push_url.
|
||||
- name: Create DATUM Gateway health check script
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.sh.j2
|
||||
dest: /usr/local/bin/datum-gateway-healthcheck-push.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
validate: "bash -n %s"
|
||||
|
||||
- name: Create datum-gateway health check systemd service
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.service.j2
|
||||
dest: /etc/systemd/system/datum-gateway-healthcheck.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
notify: Restart datum-gateway health check timer
|
||||
|
||||
- name: Create datum-gateway health check systemd timer
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.timer.j2
|
||||
dest: /etc/systemd/system/datum-gateway-healthcheck.timer
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
notify: Restart datum-gateway health check timer
|
||||
|
||||
- name: Reload systemd daemon after health check units
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
# Ungated: enabling a timer is deployment, not monitoring.
|
||||
- name: Enable and restart the datum-gateway health check timer
|
||||
systemd:
|
||||
name: datum-gateway-healthcheck.timer
|
||||
enabled: yes
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
||||
# Arms the timer and smoke-tests the check. See roles/bitcoin_knots/README.md for
|
||||
# why restarting the timer alone is not enough with OnBootSec + OnUnitActiveSec.
|
||||
- name: Run the DATUM Gateway health check once to arm the timer
|
||||
command: systemctl start datum-gateway-healthcheck.service
|
||||
changed_when: false
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
---
|
||||
- name: Install DATUM Gateway build dependencies
|
||||
apt:
|
||||
name:
|
||||
- cmake
|
||||
- build-essential
|
||||
- git
|
||||
- libjansson-dev
|
||||
- libmicrohttpd-dev
|
||||
- libsodium-dev
|
||||
- libcurl4-openssl-dev
|
||||
# Runtime-only (netcat for health check)
|
||||
- netcat-openbsd
|
||||
state: present
|
||||
update_cache: yes
|
||||
|
||||
# ===========================================
|
||||
# System User and Directories
|
||||
# ===========================================
|
||||
- name: Create datum system user
|
||||
user:
|
||||
name: "{{ datum_gateway_user }}"
|
||||
system: yes
|
||||
shell: /usr/sbin/nologin
|
||||
home: "{{ datum_gateway_dir }}"
|
||||
create_home: no
|
||||
comment: "DATUM Gateway"
|
||||
|
||||
- name: Create DATUM Gateway directories
|
||||
file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
owner: "{{ item.owner }}"
|
||||
group: "{{ datum_gateway_group }}"
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" }
|
||||
- { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" }
|
||||
- { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" }
|
||||
- { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" }
|
||||
|
||||
# ===========================================
|
||||
# Build from Source
|
||||
# ===========================================
|
||||
- name: Clone DATUM Gateway repository at {{ datum_gateway_version }}
|
||||
git:
|
||||
repo: https://github.com/OCEAN-xyz/datum_gateway.git
|
||||
dest: "{{ datum_gateway_source_dir }}"
|
||||
version: "{{ datum_gateway_version }}"
|
||||
force: yes
|
||||
register: git_clone
|
||||
|
||||
- name: Configure cmake build
|
||||
command: cmake . -DCMAKE_BUILD_TYPE=Release
|
||||
args:
|
||||
chdir: "{{ datum_gateway_source_dir }}"
|
||||
|
||||
- name: Compile datum_gateway
|
||||
command: make -j{{ datum_gateway_build_jobs }}
|
||||
args:
|
||||
chdir: "{{ datum_gateway_source_dir }}"
|
||||
|
||||
- name: Install datum_gateway binary
|
||||
copy:
|
||||
src: "{{ datum_gateway_source_dir }}/datum_gateway"
|
||||
dest: "{{ datum_gateway_bin_path }}"
|
||||
remote_src: yes
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
notify: Restart datum-gateway
|
||||
|
||||
# ===========================================
|
||||
# Configuration
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
---
|
||||
# import_tasks, not include_tasks: static imports stay visible to --list-tasks.
|
||||
- ansible.builtin.import_tasks: install.yml
|
||||
- ansible.builtin.import_tasks: configure.yml
|
||||
- ansible.builtin.import_tasks: service.yml
|
||||
- ansible.builtin.import_tasks: healthcheck.yml
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
- name: Reload systemd daemon
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start datum-gateway
|
||||
systemd:
|
||||
name: datum-gateway
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
# ===========================================
|
||||
# Health Check Script + Systemd Timer
|
||||
# ===========================================
|
||||
# ═════════════════════════════════════════════════════════════════════════
|
||||
# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11.
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
{
|
||||
"bitcoind": {
|
||||
"rpcuser": "{{ bitcoin_rpc_user }}",
|
||||
"rpcpassword": "{{ bitcoin_rpc_password }}",
|
||||
"rpcurl": "{{ datum_bitcoin_rpc_url }}",
|
||||
"notify_fallback": true
|
||||
},
|
||||
"stratum": {
|
||||
"listen_port": {{ datum_gateway_stratum_port }},
|
||||
"vardiff_min": {{ datum_vardiff_min }}
|
||||
},
|
||||
"mining": {
|
||||
"pool_address": "{{ datum_mining_address }}",
|
||||
"coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}",
|
||||
"coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}"
|
||||
},
|
||||
"api": {
|
||||
"admin_password": "{{ datum_gateway_admin_password }}",
|
||||
"listen_port": {{ datum_gateway_api_port }},
|
||||
"modify_conf": false
|
||||
},
|
||||
"logger": {
|
||||
"log_to_console": true,
|
||||
"log_to_file": true,
|
||||
"log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log",
|
||||
"log_rotate_daily": true,
|
||||
"log_level_console": 2,
|
||||
"log_level_file": 1
|
||||
},
|
||||
"datum": {
|
||||
"pool_pass_workers": {{ datum_pool_pass_workers | lower }},
|
||||
"pool_pass_full_users": {{ datum_pool_pass_full_users | lower }},
|
||||
"pooled_mining_only": {{ datum_pooled_mining_only | lower }}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
[Unit]
|
||||
Description=DATUM Gateway - Bitcoin Mining Gateway
|
||||
Documentation=https://github.com/OCEAN-xyz/datum_gateway
|
||||
After=network.target bitcoind.service
|
||||
Wants=bitcoind.service
|
||||
|
||||
[Service]
|
||||
User={{ datum_gateway_user }}
|
||||
Group={{ datum_gateway_group }}
|
||||
Type=simple
|
||||
ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
# Prevent config from being read by other users
|
||||
ReadWritePaths={{ datum_gateway_log_dir }}
|
||||
ReadOnlyPaths={{ datum_gateway_config_dir }}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
[Unit]
|
||||
Description=DATUM Gateway Health Check
|
||||
After=network.target datum-gateway.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh
|
||||
Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }}
|
||||
Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }}
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
# DATUM Gateway health check — managed by Ansible (roles/datum_gateway)
|
||||
#
|
||||
# The exit code is the answer and systemd keeps it:
|
||||
# systemctl is-failed datum-gateway-healthcheck.service
|
||||
# Reporting anywhere else is optional and generic.
|
||||
PUSH_URL="${HEALTHCHECK_PUSH_URL:-}"
|
||||
PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}"
|
||||
STRATUM_PORT={{ datum_gateway_stratum_port }}
|
||||
|
||||
check_datum() {
|
||||
# Service must be active and stratum port must be listening
|
||||
systemctl is-active --quiet datum-gateway && \
|
||||
nc -z 127.0.0.1 "${STRATUM_PORT}"
|
||||
}
|
||||
|
||||
report() {
|
||||
local status=$1
|
||||
local msg=$2
|
||||
# No push URL is normal, not an error: the exit code below is still a
|
||||
# complete answer for anything reading unit state.
|
||||
[ -n "$PUSH_URL" ] || return 0
|
||||
# Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's
|
||||
# GET with ?status=up. The callers still pass up/down, so the mapping is
|
||||
# done here rather than at every call site.
|
||||
local _ok=false
|
||||
[ "${status}" = "up" ] && _ok=true
|
||||
curl -s --max-time 15 --retry 2 -o /dev/null -X POST \
|
||||
-H "Authorization: Bearer ${PUSH_TOKEN}" \
|
||||
"${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true
|
||||
}
|
||||
|
||||
if check_datum; then
|
||||
report "up" "OK"
|
||||
exit 0
|
||||
else
|
||||
report "down" "DATUM Gateway not responding"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[Unit]
|
||||
Description=DATUM Gateway Health Check Timer
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=1min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
# `forgejo_runner`
|
||||
|
||||
Installs and runs a Forgejo Actions runner, registers it with the Forgejo
|
||||
instance, and keeps a health check on a systemd timer.
|
||||
|
||||
Converted from `deploy_forgejo_runner_playbook.yml` (409 lines) under Plan 6.
|
||||
The playbook is now 16 lines.
|
||||
|
||||
## Phases
|
||||
|
||||
`tasks/main.yml` imports five files in order:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `prerequisites.yml` | Docker must be present |
|
||||
| `install.yml` | binary, system user, working directory |
|
||||
| `configure.yml` | config file, registration with the instance |
|
||||
| `service.yml` | systemd unit, start, assert it came up |
|
||||
| `healthcheck.yml` | check script, unit, timer |
|
||||
|
||||
`import_tasks`, not `include_tasks` — static imports are visible to
|
||||
`--list-tasks`, which is how the conversion was verified against the playbook it
|
||||
replaced.
|
||||
|
||||
## Monitoring: one variable, no product knowledge
|
||||
|
||||
This role contains **nothing specific to any monitoring system**. What used to
|
||||
be here — an ~80-line embedded Python script creating monitors over the Uptime
|
||||
Kuma API, a `/tmp` credentials file, token extraction, a systemd `Environment=`
|
||||
rewrite, and 8 `when: uptime_kuma_enabled` guards — is gone.
|
||||
|
||||
What remains answers the actual question, *is this service healthy*, and records
|
||||
it two ways:
|
||||
|
||||
- **the exit code**, which systemd keeps: `systemctl is-failed
|
||||
forgejo-runner-healthcheck.service` is a complete answer with no monitoring
|
||||
system involved at all;
|
||||
- **a log file** at `{{ healthcheck_log_file }}`.
|
||||
|
||||
To report health somewhere, set one variable:
|
||||
|
||||
```yaml
|
||||
healthcheck_push_url: "https://example/api/push/TOKEN"
|
||||
```
|
||||
|
||||
Any endpoint accepting an HTTP ping works. Empty (the default) means check, log,
|
||||
exit honestly, report nowhere — which is also the right setting for a *pull*-based
|
||||
monitor like Prometheus' textfile collector, since that reads unit state instead.
|
||||
|
||||
The push URL is a credential (anyone holding it can forge an "up"), so callers
|
||||
pass it from the vault rather than committing it.
|
||||
|
||||
## One behaviour change, deliberate
|
||||
|
||||
`Assert runner is running` used to be guarded by `uptime_kuma_enabled`, so it
|
||||
never ran. It is not a monitoring task — it is the deployment checking its own
|
||||
work — and the deprecation banner swept it up by mistake. It is ungated here,
|
||||
which means a runner that fails to start now fails the play instead of
|
||||
deploying "successfully" in silence.
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
---
|
||||
# Binary
|
||||
forgejo_runner_version: "6.3.1"
|
||||
forgejo_runner_arch: "linux-amd64"
|
||||
forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}"
|
||||
forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner"
|
||||
|
||||
# Runtime
|
||||
forgejo_runner_user: "runner"
|
||||
forgejo_runner_dir: "/opt/forgejo-runner"
|
||||
forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml"
|
||||
forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm"
|
||||
|
||||
# The Forgejo instance this runner registers with.
|
||||
forgejo_instance_url: "https://forgejo.contrapeso.xyz"
|
||||
# forgejo_runner_registration_token comes from the vault.
|
||||
|
||||
# --- Health check -----------------------------------------------------------
|
||||
# The check answers "is this service healthy" and records the answer two ways:
|
||||
# a log file, and its own exit code. The exit code is the durable artefact —
|
||||
# systemd stores it, so `systemctl is-failed forgejo-runner-healthcheck.service`
|
||||
# answers the question with no monitoring system involved at all.
|
||||
healthcheck_interval_seconds: 60
|
||||
healthcheck_script_dir: /opt/forgejo-runner-healthcheck
|
||||
healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh"
|
||||
healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log"
|
||||
healthcheck_service_name: forgejo-runner-healthcheck
|
||||
|
||||
# WHERE TO REPORT HEALTH — the one place to plug in monitoring.
|
||||
#
|
||||
# Empty means "check, log, exit honestly, report nowhere". Set it to any URL
|
||||
# that accepts an HTTP ping and the check will report there. Nothing in this
|
||||
# role is specific to a particular monitoring product: the Uptime Kuma API
|
||||
# calls, monitor creation and token handling that used to live here are gone.
|
||||
#
|
||||
# A pull-based monitor (Prometheus node_exporter textfile, say) needs this left
|
||||
# empty — it reads the systemd unit state instead.
|
||||
healthcheck_push_url: ""
|
||||
# Bearer token for the Gatus external endpoint. Required whenever a push URL
|
||||
# is set: Gatus rejects an unauthenticated push with 401.
|
||||
healthcheck_push_token: ""
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
---
|
||||
- name: Check if config already exists
|
||||
stat:
|
||||
path: "{{ forgejo_runner_config_path }}"
|
||||
register: config_stat
|
||||
|
||||
- name: Generate default config
|
||||
shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}"
|
||||
args:
|
||||
chdir: "{{ forgejo_runner_dir }}"
|
||||
when: not config_stat.stat.exists
|
||||
|
||||
- name: Set config file ownership
|
||||
file:
|
||||
path: "{{ forgejo_runner_config_path }}"
|
||||
owner: "{{ forgejo_runner_user }}"
|
||||
group: "{{ forgejo_runner_user }}"
|
||||
when: not config_stat.stat.exists
|
||||
|
||||
# ── 6. Register runner ─────────────────────────────────────────────
|
||||
- name: Check if runner is already registered
|
||||
stat:
|
||||
path: "{{ forgejo_runner_dir }}/.runner"
|
||||
register: runner_stat
|
||||
|
||||
- name: Register runner with Forgejo instance
|
||||
command: >
|
||||
{{ forgejo_runner_bin_path }} register --no-interactive
|
||||
--instance {{ forgejo_instance_url }}
|
||||
--token {{ forgejo_runner_registration_token }}
|
||||
--name forgejo-runner-box
|
||||
--labels "{{ forgejo_runner_labels }}"
|
||||
args:
|
||||
chdir: "{{ forgejo_runner_dir }}"
|
||||
when: not runner_stat.stat.exists
|
||||
|
||||
- name: Set runner registration file ownership
|
||||
file:
|
||||
path: "{{ forgejo_runner_dir }}/.runner"
|
||||
owner: "{{ forgejo_runner_user }}"
|
||||
group: "{{ forgejo_runner_user }}"
|
||||
when: not runner_stat.stat.exists
|
||||
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
---
|
||||
# Everything here answers "is the service healthy" and records the answer.
|
||||
# The Uptime Kuma specifics that used to surround it — an embedded Python script
|
||||
# that created monitors over the API, a /tmp credentials file, token extraction,
|
||||
# and a systemd Environment= rewrite — are gone. What reports where is now one
|
||||
# variable, healthcheck_push_url. See the role README.
|
||||
- name: Create healthcheck script directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ healthcheck_script_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Create forgejo-runner healthcheck script
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.sh.j2
|
||||
dest: "{{ healthcheck_script_path }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
validate: "bash -n %s"
|
||||
|
||||
- name: Create healthcheck systemd service
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.service.j2
|
||||
dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
- name: Create healthcheck systemd timer
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.timer.j2
|
||||
dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer"
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd for healthcheck units
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start healthcheck timer
|
||||
systemd:
|
||||
name: "{{ healthcheck_service_name }}.timer"
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Test healthcheck script
|
||||
command: "{{ healthcheck_script_path }}"
|
||||
register: healthcheck_test
|
||||
changed_when: false
|
||||
|
||||
- name: Verify healthcheck script works
|
||||
assert:
|
||||
that:
|
||||
- healthcheck_test.rc == 0
|
||||
fail_msg: "Healthcheck script failed to execute properly"
|
||||
|
||||
- name: Display deployment summary
|
||||
debug:
|
||||
msg: |
|
||||
Forgejo Runner deployed successfully!
|
||||
|
||||
Runner Name: forgejo-runner-box
|
||||
Instance: {{ forgejo_instance_url }}
|
||||
Working Directory: {{ forgejo_runner_dir }}
|
||||
Service: forgejo-runner.service ({{ runner_active.stdout }})
|
||||
|
||||
Healthcheck Monitor: {{ healthcheck_service_name }}
|
||||
Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s
|
||||
Reporting to: {{ healthcheck_push_url | default('', true) | regex_replace('/api/push/.*', '/api/push/***') | default('(nowhere - set healthcheck_push_url)', true) }}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
---
|
||||
- name: Download forgejo-runner binary
|
||||
get_url:
|
||||
url: "{{ forgejo_runner_url }}"
|
||||
dest: "{{ forgejo_runner_bin_path }}"
|
||||
mode: '0755'
|
||||
|
||||
# ── 3. Create runner system user ───────────────────────────────────
|
||||
- name: Create runner system user
|
||||
user:
|
||||
name: "{{ forgejo_runner_user }}"
|
||||
system: yes
|
||||
shell: /usr/sbin/nologin
|
||||
home: "{{ forgejo_runner_dir }}"
|
||||
create_home: no
|
||||
groups: docker
|
||||
append: yes
|
||||
comment: 'Forgejo Runner'
|
||||
|
||||
# ── 4. Create working directory ────────────────────────────────────
|
||||
- name: Create forgejo-runner working directory
|
||||
file:
|
||||
path: "{{ forgejo_runner_dir }}"
|
||||
state: directory
|
||||
owner: "{{ forgejo_runner_user }}"
|
||||
group: "{{ forgejo_runner_user }}"
|
||||
mode: '0750'
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
---
|
||||
# import_tasks, not include_tasks: these are unconditional phases, and static
|
||||
# imports are visible to `--list-tasks`. That matters because the task list is
|
||||
# how this refactor was verified against the playbook it replaced.
|
||||
- ansible.builtin.import_tasks: prerequisites.yml
|
||||
- ansible.builtin.import_tasks: install.yml
|
||||
- ansible.builtin.import_tasks: configure.yml
|
||||
- ansible.builtin.import_tasks: service.yml
|
||||
- ansible.builtin.import_tasks: healthcheck.yml
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
- name: Check if Docker is installed
|
||||
command: docker --version
|
||||
register: docker_check
|
||||
changed_when: false
|
||||
failed_when: docker_check.rc != 0
|
||||
|
||||
- name: Fail if Docker is not available
|
||||
assert:
|
||||
that:
|
||||
- docker_check.rc == 0
|
||||
fail_msg: "Docker is required for forgejo-runner but is not installed"
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
---
|
||||
- name: Create forgejo-runner systemd service
|
||||
ansible.builtin.template:
|
||||
src: forgejo-runner.service.j2
|
||||
dest: /etc/systemd/system/forgejo-runner.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Enable and start forgejo-runner service
|
||||
systemd:
|
||||
name: forgejo-runner
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
- name: Verify forgejo-runner is active
|
||||
command: systemctl is-active forgejo-runner
|
||||
register: runner_active
|
||||
changed_when: false
|
||||
|
||||
# Ungated on purpose. This was previously guarded by `uptime_kuma_enabled`, but
|
||||
# it is not a monitoring task — it is the deployment asserting its own success.
|
||||
# The deprecation banner swept it up along with the Kuma plumbing, which meant a
|
||||
# broken runner deployed "successfully" and silently.
|
||||
- name: Assert runner is running
|
||||
assert:
|
||||
that:
|
||||
- runner_active.stdout == "active"
|
||||
fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}"
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
[Unit]
|
||||
Description=Forgejo Runner
|
||||
Documentation=https://forgejo.org/docs/latest/admin/actions/
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ forgejo_runner_user }}
|
||||
Group={{ forgejo_runner_user }}
|
||||
WorkingDirectory={{ forgejo_runner_dir }}
|
||||
ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }}
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
[Unit]
|
||||
Description=Forgejo Runner Healthcheck
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ healthcheck_script_path }}
|
||||
Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }}
|
||||
Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }}
|
||||
User=root
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Forgejo Runner healthcheck — managed by Ansible (roles/forgejo_runner)
|
||||
#
|
||||
# Answers "is forgejo-runner healthy" and records it two ways: this log, and the
|
||||
# exit code. The exit code is the durable artefact — systemd keeps it, so
|
||||
# systemctl is-failed {{ healthcheck_service_name }}.service
|
||||
# answers the question with no monitoring system involved.
|
||||
#
|
||||
# Reporting is optional and generic: if a push URL is configured it also pings
|
||||
# it. Nothing here knows or cares which monitoring product is on the other end.
|
||||
|
||||
LOG_FILE="{{ healthcheck_log_file }}"
|
||||
# Read from the environment rather than templated in, so the unit file is the
|
||||
# only place the token lives and the script is not secret.
|
||||
PUSH_URL="${HEALTHCHECK_PUSH_URL:-}"
|
||||
PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}"
|
||||
|
||||
log_message() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Gatus external endpoint: a POST with a bearer token and success=true|false.
|
||||
#
|
||||
# This used to report ONLY success - it exited before pushing when the runner
|
||||
# was down - so a failure was invisible until the heartbeat window expired.
|
||||
# Reporting the failure is the whole point of having a check.
|
||||
report() {
|
||||
local ok="$1" msg="$2"
|
||||
[ -n "$PUSH_URL" ] || return 0
|
||||
curl -s --max-time 15 --retry 2 -o /dev/null -X POST \
|
||||
-H "Authorization: Bearer ${PUSH_TOKEN}" \
|
||||
"${PUSH_URL}?success=${ok}&error=${msg// /%20}" || true
|
||||
}
|
||||
|
||||
main() {
|
||||
if ! systemctl is-active --quiet forgejo-runner; then
|
||||
log_message "ERROR: forgejo-runner is not active"
|
||||
report false "forgejo-runner is not active"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_message "forgejo-runner is active"
|
||||
report true "active"
|
||||
exit 0
|
||||
}
|
||||
|
||||
main
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
[Unit]
|
||||
Description=Run Forgejo Runner Healthcheck every minute
|
||||
Requires={{ healthcheck_service_name }}.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30sec
|
||||
OnUnitActiveSec={{ healthcheck_interval_seconds }}sec
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# `fulcrum`
|
||||
|
||||
Deploys [Fulcrum](https://github.com/cculianu/Fulcrum), an Electrum server
|
||||
indexing the Bitcoin Knots node, on `fulcrum-box`. The second play in the
|
||||
calling playbook publishes its SSL port from the edge host via `socket_proxy`.
|
||||
|
||||
Converted from `deploy_fulcrum_playbook.yml` (685 lines) under Plan 6. The
|
||||
playbook is now 33 lines.
|
||||
|
||||
## The index is the expensive thing
|
||||
|
||||
`{{ fulcrum_db_dir }}` is ~192 GB and takes days to rebuild. Nothing in this role
|
||||
touches it beyond `state: directory` with the ownership it already has
|
||||
(`fulcrum:fulcrum 0755`). Restarting Fulcrum re-opens the database; it does not
|
||||
reindex.
|
||||
|
||||
## Three things this conversion fixed, all pre-existing
|
||||
|
||||
**`bitcoind` pointed at the wrong machine.** The vars file carried
|
||||
`bitcoin_rpc_host: "192.168.1.140"` commented "IP of knots_box_local", but `.140`
|
||||
is **fulcrum-box itself** — knots-box is `.135`. The DHCP leases had reshuffled.
|
||||
The live config had already been hand-corrected to `knots-box`; running the
|
||||
playbook would have reverted it and broken indexing. Now addressed by Tailscale
|
||||
name, like everything else in this repo.
|
||||
|
||||
**The restart handler was inert.** It carried
|
||||
`when: uptime_kuma_enabled | default(false)`, so the three tasks that notify it
|
||||
(SSL certificate, `fulcrum.conf`, systemd unit) could not restart anything. A
|
||||
configuration change would write to disk, report success, and silently never take
|
||||
effect. Ungated.
|
||||
|
||||
**`db_mem` was about to quadruple.** The role computes a share of RAM; on this
|
||||
5931 MB host 75% is 4448 MB, leaving ~1.4 GB for the OS and Fulcrum's non-cache
|
||||
memory. The live value had been hand-tuned to 2048. `fulcrum_db_mem_mb_override`
|
||||
pins it. Note `set_fact` outranks role defaults, so the *calculation* has to
|
||||
honour the override — pinning it in `defaults/` alone is silently ignored.
|
||||
|
||||
## The health check timer, and how to read it
|
||||
|
||||
The timer is `OnBootSec` + `OnUnitActiveSec` with no `OnCalendar`. That
|
||||
combination has a failure mode worth knowing: `OnBootSec` is monotonic and
|
||||
elapses once; `OnUnitActiveSec` schedules relative to the **service** last being
|
||||
active. If the service does not run in a given boot, there is no reference to
|
||||
schedule from and the timer sits `active` and `enabled` doing nothing. That is
|
||||
exactly what had happened here — last trigger **2026-02-17**, seven months of no
|
||||
health check, with every surface-level indicator green.
|
||||
|
||||
Restarting the timer does not supply that reference; running the service does.
|
||||
So the role runs the check once after enabling the timer, which is both the fix
|
||||
and a smoke test.
|
||||
|
||||
**Diagnosing this is easy to get wrong**: `NextElapseUSecRealtime` is always
|
||||
empty for a monotonic timer, so it looks broken even when it is fine. Read
|
||||
`NextElapseUSecMonotonic`, or just use `systemctl list-timers`.
|
||||
|
||||
The timer also no longer carries `Requires=fulcrum.service`. On a timer that
|
||||
means "stop watching when the watched thing stops", which is backwards for a
|
||||
health check.
|
||||
|
||||
## Monitoring: one variable, no product knowledge
|
||||
|
||||
The check tests the Electrum TCP port and records the answer in its exit code,
|
||||
which systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. To report
|
||||
elsewhere set `healthcheck_push_url` to any endpoint accepting an HTTP ping. The
|
||||
Uptime Kuma API calls, monitor creation and token handling are gone.
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
# Fulcrum Configuration Variables
|
||||
|
||||
# Version - Pinned to specific release
|
||||
fulcrum_version: "2.1.0" # Fulcrum version to install
|
||||
|
||||
# Directories
|
||||
fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount)
|
||||
fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path)
|
||||
fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk
|
||||
fulcrum_binary_path: /usr/local/bin/Fulcrum
|
||||
|
||||
# Network - Bitcoin RPC connection
|
||||
# Bitcoin Knots is on a different host (knots_box_local)
|
||||
# Using RPC user/password authentication (credentials from group_vars/all/vault.yml)
|
||||
# Addressed by Tailscale name, never a LAN IP. This was
|
||||
# bitcoin_rpc_host: "192.168.1.140" # IP of knots_box_local
|
||||
# but .140 is fulcrum-box ITSELF - knots-box is .135. The DHCP leases had
|
||||
# reshuffled (the same drift that transposed the inventory), so running this
|
||||
# playbook would have pointed Fulcrum at itself and broken indexing. The live
|
||||
# config had already been hand-corrected to knots-box; this makes the repo
|
||||
# agree with it.
|
||||
bitcoin_rpc_host: "knots-box"
|
||||
bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port
|
||||
# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml
|
||||
|
||||
# Network - Fulcrum server
|
||||
fulcrum_tcp_port: 50001
|
||||
# The edge host's socket-proxy/Caddy play needs this too, and a role default is
|
||||
# invisible outside this role. The authoritative value for the live deployment is
|
||||
# in host_vars/fulcrum_box_local/main.yml, which outranks this; the value here is the
|
||||
# protocol standard, so the role still works standalone.
|
||||
fulcrum_ssl_port: 50002
|
||||
# Binding address for Fulcrum TCP/SSL server:
|
||||
# - "127.0.0.1" = localhost only (use when Caddy is on the same box)
|
||||
# - "0.0.0.0" = all interfaces (use when Caddy is on a different box)
|
||||
# - Specific IP = bind to specific network interface
|
||||
fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box)
|
||||
fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port
|
||||
# If Caddy is on a different box, set this to the IP address that Caddy will use to connect
|
||||
|
||||
# SSL/TLS Configuration
|
||||
fulcrum_ssl_enabled: true
|
||||
fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt"
|
||||
fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key"
|
||||
fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert
|
||||
|
||||
|
||||
# Performance
|
||||
# db_mem will be calculated as 75% of available RAM automatically in playbook
|
||||
# db_mem is computed as this share of RAM unless fulcrum_db_mem_mb is set
|
||||
# explicitly. On a 5931 MB host 75% is 4448 MB, which leaves ~1.4 GB for the
|
||||
# OS and for Fulcrum's non-cache memory; the live config had been hand-tuned
|
||||
# down to 2048 and that setting is respected below.
|
||||
fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache
|
||||
|
||||
# Configuration options
|
||||
fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs
|
||||
fulcrum_peering: false # Disable peering with other Fulcrum servers
|
||||
fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications
|
||||
|
||||
# Service user
|
||||
fulcrum_user: fulcrum
|
||||
fulcrum_group: fulcrum
|
||||
|
||||
|
||||
# --- Health check -----------------------------------------------------------
|
||||
# Checks the Electrum TCP port and records the answer in its exit code, which
|
||||
# systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`.
|
||||
#
|
||||
# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means
|
||||
# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping
|
||||
# works; nothing here is specific to a monitoring product.
|
||||
healthcheck_push_url: ""
|
||||
# Bearer token for the Gatus external endpoint. Required whenever a push URL
|
||||
# is set: Gatus rejects an unauthenticated push with 401.
|
||||
healthcheck_push_token: ""
|
||||
|
||||
# Explicit db_mem in MB. When set it wins over fulcrum_db_mem_percent; empty
|
||||
# means compute from RAM. Set here because the live host had been hand-tuned to
|
||||
# 2048 and a silent jump to 4448 is not something a refactor should do.
|
||||
fulcrum_db_mem_mb_override: 2048
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
# Ungated on purpose. The hand-written handler carried
|
||||
# when: uptime_kuma_enabled | default(false)
|
||||
# so it has been inert since the decommissioning: three tasks notify it (the SSL
|
||||
# certificate, fulcrum.conf and the systemd unit), and none of them could
|
||||
# actually restart Fulcrum. A configuration change therefore applied to disk and
|
||||
# silently never took effect — the worst kind of quiet failure, because the
|
||||
# playbook reports success and the running service keeps its old settings.
|
||||
#
|
||||
# Restarting Fulcrum re-opens its database; it does not reindex.
|
||||
- name: Restart fulcrum
|
||||
systemd:
|
||||
name: fulcrum
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
---
|
||||
# Everything here answers "is Fulcrum healthy" and records the answer. The
|
||||
# Uptime Kuma specifics that used to follow — an embedded Python script creating
|
||||
# monitors over the API, a /tmp credentials file, push-URL extraction and a
|
||||
# systemd Environment= rewrite — are gone. Where it reports is now one variable,
|
||||
# healthcheck_push_url. See the role README.
|
||||
- name: Create Fulcrum health check script
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.sh.j2
|
||||
dest: /usr/local/bin/fulcrum-healthcheck-push.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
validate: "bash -n %s"
|
||||
|
||||
- name: Create systemd service for Fulcrum health check
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.service.j2
|
||||
dest: /etc/systemd/system/fulcrum-healthcheck.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
- name: Create systemd timer for Fulcrum health check
|
||||
ansible.builtin.template:
|
||||
src: healthcheck.timer.j2
|
||||
dest: /etc/systemd/system/fulcrum-healthcheck.timer
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Reload systemd daemon for health check
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
# state: restarted, not started. The hand-written timer had got itself stuck
|
||||
# `active` with no next elapse and had not fired since 2026-02-17; `started` on
|
||||
# an already-active timer is a no-op and would have left it stuck. Restarting
|
||||
# re-arms it. See the note in healthcheck.timer.j2.
|
||||
- name: Enable and restart the Fulcrum health check timer
|
||||
systemd:
|
||||
name: fulcrum-healthcheck.timer
|
||||
enabled: yes
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
|
||||
# Run the check once, which is both a smoke test and the thing that actually
|
||||
# arms the timer.
|
||||
#
|
||||
# This timer is OnBootSec + OnUnitActiveSec with no OnCalendar. OnBootSec is
|
||||
# monotonic and had long since elapsed; OnUnitActiveSec schedules relative to the
|
||||
# SERVICE last being active, and the service had not run since 2026-02-17 — so
|
||||
# there was no reference to schedule from and the timer sat `active` and
|
||||
# `enabled` with NextElapseUSecMonotonic=infinity. Restarting the timer alone
|
||||
# does not supply that reference; running the service does.
|
||||
#
|
||||
# (Diagnosing this is easy to get wrong: NextElapseUSecRealtime is always empty
|
||||
# for a monotonic timer, so it looks broken even when it is fine. Read
|
||||
# NextElapseUSecMonotonic, or just use `systemctl list-timers`.)
|
||||
- name: Run the Fulcrum health check once to arm the timer
|
||||
command: systemctl start fulcrum-healthcheck.service
|
||||
changed_when: false
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
---
|
||||
- name: Calculate db_mem as a share of system RAM
|
||||
set_fact:
|
||||
fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}"
|
||||
when: fulcrum_db_mem_mb_override | string | length == 0
|
||||
|
||||
- name: Use the explicit db_mem override
|
||||
set_fact:
|
||||
fulcrum_db_mem_mb: "{{ fulcrum_db_mem_mb_override }}"
|
||||
when: fulcrum_db_mem_mb_override | string | length > 0
|
||||
changed_when: false
|
||||
|
||||
- name: Display calculated db_mem value
|
||||
debug:
|
||||
msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)"
|
||||
|
||||
- name: Display Fulcrum version to install
|
||||
debug:
|
||||
msg: "Installing Fulcrum version {{ fulcrum_version }}"
|
||||
|
||||
- name: Install required packages
|
||||
apt:
|
||||
name:
|
||||
- curl
|
||||
- wget
|
||||
- openssl
|
||||
state: present
|
||||
update_cache: yes
|
||||
|
||||
- name: Create fulcrum group
|
||||
group:
|
||||
name: "{{ fulcrum_group }}"
|
||||
system: yes
|
||||
state: present
|
||||
|
||||
- name: Create fulcrum user
|
||||
user:
|
||||
name: "{{ fulcrum_user }}"
|
||||
group: "{{ fulcrum_group }}"
|
||||
system: yes
|
||||
shell: /usr/sbin/nologin
|
||||
home: /home/{{ fulcrum_user }}
|
||||
create_home: yes
|
||||
state: present
|
||||
|
||||
- name: Create Fulcrum database directory (heavy data on special mount)
|
||||
file:
|
||||
path: "{{ fulcrum_db_dir }}"
|
||||
state: directory
|
||||
owner: "{{ fulcrum_user }}"
|
||||
group: "{{ fulcrum_group }}"
|
||||
mode: '0755'
|
||||
|
||||
- name: Create Fulcrum config directory
|
||||
file:
|
||||
path: "{{ fulcrum_config_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: "{{ fulcrum_group }}"
|
||||
mode: '0755'
|
||||
|
||||
- name: Create Fulcrum lib directory (for banner and other data files)
|
||||
file:
|
||||
path: "{{ fulcrum_lib_dir }}"
|
||||
state: directory
|
||||
owner: "{{ fulcrum_user }}"
|
||||
group: "{{ fulcrum_group }}"
|
||||
mode: '0755'
|
||||
|
||||
# ===========================================
|
||||
# SSL Certificate Generation
|
||||
# ===========================================
|
||||
- name: Check if SSL certificate already exists
|
||||
stat:
|
||||
path: "{{ fulcrum_ssl_cert_path }}"
|
||||
register: fulcrum_ssl_cert_exists
|
||||
when: fulcrum_ssl_enabled | default(false)
|
||||
|
||||
- name: Generate self-signed SSL certificate for Fulcrum
|
||||
command: >
|
||||
openssl req -x509 -newkey rsa:4096
|
||||
-keyout {{ fulcrum_ssl_key_path }}
|
||||
-out {{ fulcrum_ssl_cert_path }}
|
||||
-sha256 -days {{ fulcrum_ssl_cert_days }}
|
||||
-nodes
|
||||
-subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local"
|
||||
args:
|
||||
creates: "{{ fulcrum_ssl_cert_path }}"
|
||||
when: fulcrum_ssl_enabled | default(false)
|
||||
notify: Restart fulcrum
|
||||
|
||||
- name: Set SSL certificate permissions
|
||||
file:
|
||||
path: "{{ fulcrum_ssl_cert_path }}"
|
||||
owner: "{{ fulcrum_user }}"
|
||||
group: "{{ fulcrum_group }}"
|
||||
mode: '0644'
|
||||
when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false)
|
||||
|
||||
- name: Set SSL key permissions
|
||||
file:
|
||||
path: "{{ fulcrum_ssl_key_path }}"
|
||||
owner: "{{ fulcrum_user }}"
|
||||
group: "{{ fulcrum_group }}"
|
||||
mode: '0600'
|
||||
when: fulcrum_ssl_enabled | default(false)
|
||||
|
||||
- name: Check if Fulcrum binary already exists
|
||||
stat:
|
||||
path: "{{ fulcrum_binary_path }}"
|
||||
register: fulcrum_binary_exists
|
||||
changed_when: false
|
||||
|
||||
- name: Download Fulcrum binary tarball
|
||||
get_url:
|
||||
url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz"
|
||||
dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz"
|
||||
mode: '0644'
|
||||
when: not fulcrum_binary_exists.stat.exists
|
||||
|
||||
- name: Extract Fulcrum binary
|
||||
unarchive:
|
||||
src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz"
|
||||
dest: "/tmp"
|
||||
remote_src: yes
|
||||
when: not fulcrum_binary_exists.stat.exists
|
||||
|
||||
- name: Install Fulcrum binary
|
||||
copy:
|
||||
src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum"
|
||||
dest: "{{ fulcrum_binary_path }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
remote_src: yes
|
||||
when: not fulcrum_binary_exists.stat.exists
|
||||
|
||||
- name: Verify Fulcrum binary installation
|
||||
command: "{{ fulcrum_binary_path }} --version"
|
||||
register: fulcrum_version_check
|
||||
changed_when: false
|
||||
|
||||
- name: Display Fulcrum version
|
||||
debug:
|
||||
msg: "{{ fulcrum_version_check.stdout_lines }}"
|
||||
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
---
|
||||
# import_tasks, not include_tasks: static imports stay visible to --list-tasks,
|
||||
# which is how this conversion was verified against the playbook it replaced.
|
||||
- ansible.builtin.import_tasks: install.yml
|
||||
- ansible.builtin.import_tasks: service.yml
|
||||
- ansible.builtin.import_tasks: healthcheck.yml
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue