diff --git a/.gitignore b/.gitignore index 6c0a084..471cd6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,23 @@ # OpenTofu / Terraform .terraform/ .tofu/ -.terraform.lock.hcl -.tofu.lock.hcl terraform.tfstate terraform.tfstate.* crash.log -*.tfvars -*.tfvars.json -test-inventory.ini -inventory.ini +# Provider credentials only. Non-secret infra config (the vms map) is committed +# as *.auto.tfvars, and *.lock.hcl is committed on purpose so provider hashes +# are pinned. +terraform.tfvars +terraform.tfvars.json +*secrets.auto.tfvars + venv/* .env -# Secrets and sensitive files -*_secrets.yml -*_secrets.yaml -secrets/ -.secrets/ +# Secrets are ansible-vault encrypted and ARE committed. +# Anything matching *_secrets.plain.yml is a working decryption — never commit those. +*_secrets.plain.yml + +# Vault password — never commit +ansible/.vault_pass \ No newline at end of file diff --git a/01_infra_setup.md b/01_infra_setup.md index 6ef0978..a8ba9c0 100644 --- a/01_infra_setup.md +++ b/01_infra_setup.md @@ -162,6 +162,12 @@ Note that, by applying these playbooks, both the root user and the `counterweigh ```bash cp ansible/infra_secrets.yml.example ansible/infra_secrets.yml ``` + > **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server + > deployment was removed from this repo; what it monitored and how it was set up is + > preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in + > the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic + > survives for whatever replaces it. The credentials below no longer exist in the vault. + * Edit `ansible/infra_secrets.yml` and add your Uptime Kuma credentials: ```yaml uptime_kuma_username: "admin" diff --git a/02_vps_core_services_setup.md b/02_vps_core_services_setup.md index 19cd122..5d75de9 100644 --- a/02_vps_core_services_setup.md +++ b/02_vps_core_services_setup.md @@ -49,6 +49,12 @@ Checklist: ## Uptime Kuma +> **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server +> deployment was removed from this repo; what it monitored and how it was set up is +> preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in +> the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic +> survives for whatever replaces it. The credentials below no longer exist in the vault. + Uptime Kuma gets used to monitor the availability of services, keep track of their uptime and notify issues. ### Deploy diff --git a/README.md b/README.md index f343cbc..89ef092 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ My repo documenting my personal infra, along with artifacts, scripts, etc. Go through the different numbered markdowns in the repo root to do the different parts. +## How to edit secrets + +`ansible-vault edit ansible/your_file_with_secrets.yml` + +Assumes that you've set `ansible/.vault_pass` with `chmod 600`. + ## Overview ### Services @@ -16,7 +22,7 @@ Go through the different numbered markdowns in the repo root to do the different + Plan install + File based config + Crossbackup to Desky via rsync -* Uptime Kuma +* ~~Uptime Kuma~~ — decommissioned 2026-09-11, see `archive/uptime_kuma/` + Deployed on Vipy + Crossbackup to Desky via rsync * Vaultwarden diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..aa10181 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,14 @@ +[defaults] +inventory = inventory.ini +roles_path = roles +collections_path = collections +interpreter_python = auto_silent +stdout_callback = yaml +retry_files_enabled = False +host_key_checking = True +forks = 10 +vault_password_file = .vault_pass + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=300s diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml new file mode 100644 index 0000000..f77ae48 --- /dev/null +++ b/ansible/group_vars/all/main.yml @@ -0,0 +1,66 @@ +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: contrapeso.xyz + +# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the +# playbooks are kept deliberately — the check logic is meant to be rewired to +# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. + +# age recipient for all backup artefacts +age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" + +# Public key small-backups-box pulls with +# Authorised on each source host for an unprivileged, dedicated user only +backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" + + +# ───────────────────────────────────────────────────────────────────────────── +# Subdomains. Global because the edge host proxies for services that live on +# other machines, so no single inventory group covers the readers. Combine with +# root_domain above to build an FQDN. +# +# Moved here from services_config.yml, which 30 plays had to remember to name in +# vars_files: - a file everyone must opt into is a file someone will forget. +# ───────────────────────────────────────────────────────────────────────────── +subdomains: + # Monitoring + gatus: status + + # VPN infrastructure (spacey) + headscale: headscale + + # Core services (vipy) + vaultwarden: vault + forgejo: forgejo + lnbits: wallet + + # Secondary services (vipy) + ntfy_emergency_app: avisame + personal_blog: pablohere + + # Memos (memos-box) + memos: memos + + # Mempool block explorer (mempool-box, proxied via vipy) + mempool: mempool + + # DATUM Gateway dashboard (knots-box, proxied via vipy) + datum_gateway: datum + +# Read by plays across several groups, so global rather than group_vars/. +headscale_namespace: counter-net + +# ───────────────────────────────────────────────────────────────────────────── +# Domains whose registration expiry is monitored (infra/402_public_monitoring). +# +# Registration renewal is a manual act at the registrar, and losing a domain is +# not recoverable in the way losing a host is - so these are checked daily and +# alarm with two weeks of runway. +# +# root_domain is the estate's own domain; the rest are domains we own that are +# served from it or from a host in the inventory. +# ───────────────────────────────────────────────────────────────────────────── +monitored_domains: + - "{{ root_domain }}" + - arbret.com diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml new file mode 100644 index 0000000..76eb7bf --- /dev/null +++ b/ansible/group_vars/all/vault.yml @@ -0,0 +1,164 @@ +$ANSIBLE_VAULT;1.1;AES256 +34383033613438623432656438313763613639313139626432303637383363643339613166626665 +6537663336353661373437303636613364643736663031300a623461353135623537343763383333 +37323731646262613436646231656263396532616132316237353762376363383062646132303435 +3435316666316235660a323163613932616661303935336633656332353766393536393736356539 +61396164386266303464373262373862363361363365363235383363613335663933313062663565 +39613466353161356462656539613335613536363532333431393935303430373435373737363537 +62383266323339373062313134316464303263313830646161333530383736336637333865373764 +31313165396366653763343466643561383236643836616662383439666165343139326232326364 +32633133373863616163343365656231353939366435346534383462396134653064663566343566 +66343464373865383864313264646635316233336133346365396634626261613561373965643339 +30353831393037316334313632376537393361336366326561343832373531303139323937613531 +34386164303430316162363536386137316330336638303365393462386465633862663932376563 +34323862636435306563333065383134343362333733623066656439613339376666353331373665 +32353938663364313036373237323062393430306661633732356434333232376338313835663330 +32306536393666323464663238306562613936376236613935616664653865373330626365626330 +64613066653632646638306338393331376430383165316363323437373362613866313962663432 +62356663636630356239613831353066303530363966396135376134633936313530643262353965 +38383330613962313935366633353635363638303937316362303866343433353437363334396431 +32393134626133353564643233616632376633303934653065613262353436333533376137303862 +35303861383833326537376433336634633730373833313036643365373431666438653033653930 +33346230396661636333613031353133353631306331666362386366633062666534303539316238 +39366634656333646634643032393738333330636230663366623265666533356534623465316665 +35336535326237343231653661613736626239316262386363666264613636326332366232333964 +38353635616662663133376534306366643037613732343233373336653166316330643638353539 +35353532643534613864333839393365313439343663333337373639633832383630316164306132 +63636162373034363839656638353534613733663737626432636436373164303239316436313566 +66623466396663353136323565643262383865323830616232373466373431373261306537646463 +61353966623062356438626635663461396165343366393132393965383734633632666335313030 +37333631373239303231396264393031373162373936303462363736313538343562343737356332 +63663762633032356130303964386664366137373432353533386566613037613163383963623732 +32343765376535343630656633333362393765666131653632613361363163383038323138333537 +37373333316466316562613439356365316163336366623435666166653835616531653563343664 +30303464663838646333373837363961356465613234636138633162366636633030383337643131 +62636236663066316630396132333934353139383465363034363033656161383663656462313939 +62353738396132343734343363636164323163616237613861643862303433633934656165343263 +35313737373333386335663731643664393630653839396464376639306231653934386336353630 +61393735336662663435613431386361373561363531643232303831363163636139653538393931 +66313965646562616237383439343637623835303065333730613865666638383131616261346463 +31396464306537613135653830653138323539393731626264326335336432666333333735646534 +30616439393464326631633635363466303336613135346231626232313361303666323661616638 +64363937343231363536353034363966326132333734386638613737636130646232363666643633 +61353563636466356630346232613761636432303430336461663434333636623962643764336639 +35303565613861323431393036303534303061666437366538306236373930313439353330313632 +61363363303562316664303065613863613339626632643931386438326330373938613762303334 +33316430363931393262373661623137633835656136313235613666613932313236343966306331 +36343031303632323432366337633637336335343638393564313738386162386164613339343662 +62383939663265373265613932633265626263623939303638383838646531343433393864363235 +37313761396436623833376435393537376136373162393465343764326533666533393061643965 +61306365346562333737313962633764313232323161623861336566343735353737633539656162 +34323939306661373662663331653333346264323930663636633134666532623438323537383165 +66343033363962633766303331623866306462623235373838616565653066646132633034363535 +31613462356361613535383963643662373162363363653334393937313039666266366363653537 +64356166396139393262313565303731346534646462333638316661383139346362333364613466 +38303561316263636136303431393366343936653161336238613439366266346136636465303337 +61373634376464333037303862616335623031656133636165636264386265643261373735663266 +39396265366138363430343035356337303438346165366361316230656239326633653537626339 +39613334316563376133343336343563363564653237633764393532336437303334363830396635 +31333963666462313830333163646465356337643263636462356363613266323630393866626535 +35336436663063333364353663616564626535396133386461326433323936323161613633353539 +66323935626433353162666335613839346561343264323763303034613366663233613037343538 +66376236373462383430313030373235373337653338333033306431616562353435656433356365 +37653063363435366532306334373438306632306435653334396630343863623938666335656164 +37386634366430383762316435396237313362313965353634613266616532633465646464313662 +35303433363236393465396161346431306231666432666531626136623966663864376663373562 +37653239633732346536323265643434616136326561666330633435383264333937353366653736 +61353430633363323263323163646639663330663632343038623964396433626337353232373334 +32343938356539303938666137323831646562393233323733336464316165623232326132316362 +61313631626234336164656533616231393663363364623130313439623466396231383537656562 +35393163323131383838356130643863633736613635306635306135666261623731613239303962 +61373432336638353339646665653561653738353731373032366361313562306466373365653333 +35306236336331333961613864346165613763383266613236343066623432303766333434333331 +66376263363132616132366165393235626336323665373135653462653533376138346632393363 +35396165383966326635333134663138366136336430393532373935373264616530613762386430 +37643862316232613261343430616438333831623835663239656531356666613032653837373531 +30386232643537306530323936386561646265616339376265353763663833623539653831656634 +63626564633964343663656435636635656562666236386639353332356566316631643431353330 +32643238623932643732353332633535643738663066383830376432346463393464333236386163 +64376165333266323166613564313061643832656239333165393035343836376434626162643061 +31373861393534636236383765313834356638613332626666653762343537363339383939666263 +61366230623930643534353565316165656132396664393138326139613536343738396330653735 +37623463343834626133313231393130646134363331393238663065643930383237326439343831 +32393536613361616637373761393361303936363863333838666633343063386235316237323531 +62633362316436623761616366383336613765373362393161303765613464643632666462343565 +63616430396164346263666436336463623638313130656231643131336663336565616135373433 +39326533643733333564373434333738343634383033353036646534646632383730613161393130 +62653836376435663435353739373536366466646332653030653332353761626466353631663163 +38613734383634663662376464653733306564313761336566336336656366653536643834653631 +35333963306562393234623039613338643266333762333236366331333431343830393864623830 +34323439386133613035363861623332663933313734313637623739613163663730316134303434 +35363239363836666463366334356435666632396132306163303037663430383535393934323537 +30353266373864346662333035393761646632313032383738346439343232663666343833333565 +35363564623038376666363436663830356165323234653230636264303066666230643632356237 +66633734346633366533623966393462313438333965623033343337323135353730613561363236 +30636333613034626162393137323762306432663761393766303163393536656338366664643432 +34636630323965396166356538623165663335333739323034333237306531393734306138343436 +36613937373932633234303336373530663533646635396664343935326334333239646534343336 +35396666353938326337646431353965653836323731313861613031356561376262363965623934 +65336164623134366365303062393161353965363937646363396564313138323938656330333964 +64663862363766613630333531623463616331363533343962653164376665303463336134346662 +66376238663632663932656230323638323135653666363736643065336330636236323064386361 +37646631303138616565383665663634333338613336346438366333636165366635313337373635 +62656236653961393061373262326534393339363236353336613339656462336661663739396261 +35363636653237316436656662633462306333313336313865333037636661383731336466666536 +36646537653431666261353232653233653930346538383438633466346636656432383963333765 +62346363633265306665366630383831303434336139343837616361666539656262633735363064 +30376138386531336430613563366666333964663231363465326230316233323532366632663930 +63653164373931353861653833326130643365373530663039363166366331303134366336613438 +63633336363539636265396331653630633162316366316130326531363463616464376635376538 +32376638353337343639633434613663666437313564333264383766623561393933333561663235 +37666163626538353736383435373530663438363636643665626266316338336532623235353337 +61653762383630616266353438386137653137616532633066623165383663646437343565366237 +33333463613539346136613135386466656135616136663036313661323738363066633038656139 +38353230313861666463383737303038613731316334333031643438343463666164633536313031 +39613663376436643165333362636462303230656336333234363336633262336363643736336231 +66316630613762383832613836333263303038306235663662323162633932653733626130356665 +65316630346534356261373961353130393334396365376237353439326261336339393336303030 +63323439666439313831626639323735653730356565643938376564316337613333616664643865 +30633530383865663731383635366566613565393632336337306334663937396135303539363536 +65656331386532306438383031313164626237393762303636663831666239336637666465633630 +63383961336333663537666561376532376631343132363638633364616531306363623432396332 +31343864626335613131653762363338393738343531626330663663376531336230656365363661 +64363530333965386439376161313534333435393966656231613161633034303964653564363436 +63396131623562303962383137666439373132633330363237353031306561306362666561376435 +66383535333233663736363762323662363264393365643135323834366565653932313863353437 +38363536323739336165356561613335653464363637636234313637383066633865353239393961 +31613532373361383062393238303430353662383930373764373237383035633364323661623661 +62653661623234366666616532663563316234663137616338393036333937633963323635366165 +61613032363531666338353132346365323661626265626461396362646338373536306362303338 +31646564343434633433663039366533656665643235336136333134366231376336343465336539 +37363139346239666232313366613432646265626564313838346364306133306434313337626562 +32346430316532623037633838303363346130623636386134313566383333613565316135373264 +38393261313435323433326130303032393538333963646430646366653363613830613534353832 +32643434613964613239363966376138376661636335373130356430336631333461623734383535 +35356461326136636234663738343738303064386536336462303632303461303733333666383036 +38383563393739343134363038623638393766316533373163336439336238613734656132623163 +32626564666430663339316163393934323133306238353562323866633738363737643937376435 +66306430623334643766623564363239346361393666663766306637313265313833396435626234 +35366239616132623863343365663833363934316362636638303536656631633364646235366566 +39386637613735373761643339396132323031323438316633363464336636316534643435323861 +62663732366430646538626663313035616235643537643234356434323635373962343633336266 +30323031306438656537626430356231393466313334633934623632663661306361313330633339 +33396234386536336331616161363132343765306562303932313963623037366633383765663134 +62636466636433666530363930633765626531613539363832313361666565323635326334326533 +36336439363836663033643262336139613437623633616138656564393032393263393132303031 +31626336663039633362366663306662333432643261613464303939326562653562346239383261 +37373966383635626433363831653936333964626262363839383936356634633233336365623765 +62313031353538636464303234383865613932323164386336316362323731303263346637346534 +32353734343130343364646161643431353230336364366330363261326334613936633234646264 +39623163346663613431323630303034383761663835373565663166366239303130386139396236 +62393565353761636636366462353038366664663430616332326465373364396264323064666536 +61653035633562316634336263636332363733663666626131383161656535383133613939623034 +66363632613733306336323165336534346337386163656631343332323636353539353737316131 +34616336343865656565646338653037333838663736333330376330663834373138363739633064 +61646439636365323838376131663266636235333062616532613936616339633661303634323531 +63356335386534353639323566376134373565353137333134363761666532366431633634333430 +39326662633765626230326130386333393463323433366162363432613234336634663439313339 +33653737333830336264343839396563316462613032376335356634383436353962613636333866 +32333032623635323430356636623138366635386533646133626164316438393937626462313239 +31636635363064363763396566306234363965346438653738333961623435303233396634643763 +65343763386464366636383734336335646464306262623363303934393734333666636635356365 +65353931303862343765343665373139376530656263356365366333323135383838383030613166 +30303963383833643937636539656164336537383337346436303534303035313935323633663765 +333464336235373230316337333632326138 diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml new file mode 100644 index 0000000..1072b51 --- /dev/null +++ b/ansible/group_vars/nodito_vms.yml @@ -0,0 +1,24 @@ +--- +# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. +# +# ansible_host is a MagicDNS name. If tailscaled is not running on the control +# node that name does not resolve, the probe fails, and the LAN address recorded +# as lan_ip in inventory.ini takes over. +# +# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`: +# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a +# single `nc -w5` silently tears down the SSH session after five quiet seconds. +# That produced intermittent "Data could not be sent to remote host" failures on +# exactly the long, quiet operations (apt) where a dropped connection costs most. +# `nc -z` probes, then `exec nc` carries the session with no timeout at all. +# +# Safe against the LAN addresses drifting again (which is how fulcrum/mempool +# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if +# lan_ip ever points at a different machine the host key will not match and ssh +# aborts. Verified by pointing fulcrum-box at mempool-box's address: +# "Host key verification failed." +# +# lan_ip is a convenience, not an identity. If it goes stale the fallback stops +# working; it will never connect you to the wrong box. +ansible_ssh_common_args: >- + -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'" diff --git a/ansible/host_vars/forgejo_runner_local/main.yml b/ansible/host_vars/forgejo_runner_local/main.yml new file mode 100644 index 0000000..6e852ce --- /dev/null +++ b/ansible/host_vars/forgejo_runner_local/main.yml @@ -0,0 +1,12 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - forgejo-runner diff --git a/ansible/host_vars/fulcrum_box_local/main.yml b/ansible/host_vars/fulcrum_box_local/main.yml new file mode 100644 index 0000000..aa189b0 --- /dev/null +++ b/ansible/host_vars/fulcrum_box_local/main.yml @@ -0,0 +1,17 @@ +# fulcrum-box: the Electrum server. +# +# Read by the fulcrum role here and by the socket-proxy play on the edge host, +# which publishes the port. See host_vars/knots_box_local/main.yml for why this +# lives in host_vars rather than in the role's defaults. +fulcrum_ssl_port: 50002 + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - fulcrum diff --git a/ansible/host_vars/knots_box_local/main.yml b/ansible/host_vars/knots_box_local/main.yml new file mode 100644 index 0000000..3866541 --- /dev/null +++ b/ansible/host_vars/knots_box_local/main.yml @@ -0,0 +1,26 @@ +# knots-box: Bitcoin Knots and the DATUM Gateway. +# +# These ports are read twice: by the role that deploys the service here, and by +# the socket-proxy / Caddy plays that run on the EDGE host and publish them. +# A role default is invisible to that second play, which is why these live in +# host_vars rather than roles//defaults/ - the edge play reads them as +# hostvars['knots_box_local']., and the role picks them up automatically +# because host_vars outranks role defaults. +# +# They used to live in services_config.yml, a file 30 plays had to remember to +# name in vars_files: and that four role defaults silently depended on. +bitcoin_p2p_port: 8333 +datum_gateway_api_port: 7152 +datum_gateway_stratum_port: 23334 + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - bitcoind + - datum-gateway diff --git a/ansible/host_vars/memos_box_local/main.yml b/ansible/host_vars/memos_box_local/main.yml new file mode 100644 index 0000000..f1fc19b --- /dev/null +++ b/ansible/host_vars/memos_box_local/main.yml @@ -0,0 +1,12 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - memos diff --git a/ansible/host_vars/mempool_box_local/main.yml b/ansible/host_vars/mempool_box_local/main.yml new file mode 100644 index 0000000..f46a1a5 --- /dev/null +++ b/ansible/host_vars/mempool_box_local/main.yml @@ -0,0 +1,6 @@ +# mempool-box: the Mempool block explorer. +# +# Read by the mempool role here and by the Caddy play on the edge host, which +# proxies to it. See host_vars/knots_box_local/main.yml for why this lives in +# host_vars rather than in the role's defaults. +mempool_frontend_port: 8080 diff --git a/ansible/host_vars/monitoring/main.yml b/ansible/host_vars/monitoring/main.yml new file mode 100644 index 0000000..2ef805d --- /dev/null +++ b/ansible/host_vars/monitoring/main.yml @@ -0,0 +1,12 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - caddy diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/host_vars/nodito/main.yml similarity index 52% rename from ansible/infra/nodito/nodito_vars.yml rename to ansible/host_vars/nodito/main.yml index c0002f3..c8176af 100644 --- a/ansible/infra/nodito/nodito_vars.yml +++ b/ansible/host_vars/nodito/main.yml @@ -14,7 +14,12 @@ systemd_service_name: nodito-cpu-temp-monitor # ZFS Pool Configuration zfs_pool_name: "proxmox-tank-1" -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the +# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently +# replaced and the repo never caught up. Pool creation is guarded by +# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been +# wrong on any disaster-recovery run. +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" @@ -26,3 +31,15 @@ ups_port: auto ups_user: counterweight ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - nut-server + - nut-monitor diff --git a/ansible/host_vars/nodito/vault.yml b/ansible/host_vars/nodito/vault.yml new file mode 100644 index 0000000..e64085e --- /dev/null +++ b/ansible/host_vars/nodito/vault.yml @@ -0,0 +1,11 @@ +$ANSIBLE_VAULT;1.1;AES256 +30333035323663393939343061323234336164396465623665346165393534646366333332376463 +3364373463333664363334373964323838336531353364310a636636373539623464336630666164 +61376532616339376562373238383436306664313564663266303534346461666466383965323538 +3163313239626663310a613033336332653165333537313366636361663036383031376561613761 +31313563373062333033323037653939663762343161656264633436343361663737626366663732 +39366666626338323436383134646263643538333564313566346336323563663534653161396136 +39333565393538366238643563323630346166643461643063393631643665363566623631373762 +36643866646637306231653837363838656163613766636265383139333838396535626335343163 +32613935353330636263616333666230323436663935326133636362343836323535623237646235 +6266316363366335323162663039366137633865396237373632 diff --git a/ansible/host_vars/spacey/main.yml b/ansible/host_vars/spacey/main.yml new file mode 100644 index 0000000..a50d116 --- /dev/null +++ b/ansible/host_vars/spacey/main.yml @@ -0,0 +1,13 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - headscale + - caddy diff --git a/ansible/host_vars/vipy/main.yml b/ansible/host_vars/vipy/main.yml new file mode 100644 index 0000000..3b077df --- /dev/null +++ b/ansible/host_vars/vipy/main.yml @@ -0,0 +1,15 @@ +--- + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - forgejo + - lnbits + - caddy + - phoenixd diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index 13e5149..0e2c914 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,7 +1,5 @@ - name: Secure Debian - hosts: all - vars_files: - - ../infra_vars.yml + hosts: managed become: true tasks: diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index e83cbcb..309db56 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,7 +1,5 @@ - name: Secure Debian - hosts: all - vars_files: - - ../infra_vars.yml + hosts: managed become: true tasks: diff --git a/ansible/infra/400_host_monitoring.yml b/ansible/infra/400_host_monitoring.yml new file mode 100644 index 0000000..3f6b119 --- /dev/null +++ b/ansible/infra/400_host_monitoring.yml @@ -0,0 +1,216 @@ +--- +# Host-level monitoring for the whole estate, reported to Gatus. +# +# Every check here PUSHES. Gatus never reaches out, which matters because nodito +# and its VMs sit behind NAT, and because four of the five checks are internal +# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool +# health and UPS mains status cannot be observed from outside the machine. +# +# Liveness is a push too, and that is a choice rather than a limitation. A +# heartbeat proves the host is running AND can reach the internet; an ICMP probe +# from one vantage point only proves it answers pings from there. And because +# Gatus alerts when a heartbeat window expires, a check that stops running +# raises the alarm by itself - a dead timer looks exactly like a dead host, +# which is the correct reading. +# +# Each host has ONE bearer token, shared across its own checks: a token can only +# write results for that host's endpoints, so a compromised host can lie about +# itself, which it could do anyway. +# +# The push URL must use Gatus's own key format (config/key/key.go): +# key = sanitize(group) + "_" + sanitize(name) +# where sanitize lowercases and replaces / _ . , space # + & with "-". So +# knots_box_local becomes knots-box-local in the URL but stays readable in the +# name. host_key below is the Jinja equivalent; do not hand-write these. + +# ───────────────────────────────────────────────────────────────────────────── +# Register everything with Gatus. +# +# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the +# host plays below take minutes, so every endpoint exists before its first push +# arrives. Registering afterwards would 404 every first report. +# +# Heartbeat windows are several times the check interval, so one missed run - a +# slow apt run, a reboot - does not raise an alarm, but a check that has +# genuinely stopped does. +# ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the host checks with Gatus + hosts: observability + become: yes + vars: + monitored: "{{ groups['managed'] | sort }}" + + tasks: + - name: Build the liveness endpoint list + ansible.builtin.set_fact: + liveness_endpoints: "{{ liveness_endpoints | default([]) + [{ + 'name': item, + 'group': 'liveness', + 'token': gatus_push_tokens[item], + 'heartbeat': '11m'}] }}" + loop: "{{ monitored }}" + + - name: Build the disk endpoint list + ansible.builtin.set_fact: + disk_endpoints: "{{ disk_endpoints | default([]) + [{ + 'name': item, + 'group': 'disk', + 'token': gatus_push_tokens[item], + 'heartbeat': '7h'}] }}" + loop: "{{ monitored }}" + + - name: Register liveness endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: liveness + gatus_endpoint_external: "{{ liveness_endpoints }}" + + - name: Register disk endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: disk + gatus_endpoint_external: "{{ disk_endpoints }}" + + - name: Register the hypervisor endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: hypervisor + gatus_endpoint_external: + - name: cpu + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "11m" + - name: zfs + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "7h" + - name: ups + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "11m" + +- name: Deploy host liveness and disk checks + hosts: managed + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" + host_token: "{{ gatus_push_tokens[inventory_hostname] }}" + + tasks: + - name: Is the host up? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: liveness + healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}" + healthcheck_check: liveness + healthcheck_interval: "5min" + healthcheck_boot_delay: "1min" + healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is the disk packed? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: disk-usage + healthcheck_description: "Disk usage for {{ inventory_hostname }}" + healthcheck_check: disk-usage + # Every 6h rather than daily. Disk usage itself moves slowly, but the + # heartbeat can only be as tight as the push frequency - a daily push + # forces a >24h window, and a stuck check then hides for a day and a + # half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the + # hosts so twelve boxes do not all report in the same second. + healthcheck_on_calendar: "*-*-* 00/6:00:00" + healthcheck_randomized_delay: "900" + healthcheck_boot_delay: "5min" + healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external" + healthcheck_push_token: "{{ host_token }}" + +- name: Deploy the hypervisor-only checks + hosts: hypervisor + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_token: "{{ gatus_push_tokens[inventory_hostname] }}" + + tasks: + - name: Is the CPU hot? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: cpu-temp + healthcheck_description: "CPU temperature for {{ inventory_hostname }}" + healthcheck_check: cpu-temp + healthcheck_packages: [curl, lm-sensors] + healthcheck_interval: "5min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is ZFS broken? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: zfs-health + healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}" + healthcheck_check: zfs-health + healthcheck_packages: [curl, jq] + healthcheck_zfs_pool: "{{ zfs_pool_name }}" + healthcheck_on_calendar: "*-*-* 00/6:20:00" + healthcheck_boot_delay: "10min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is the UPS online? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: ups-status + healthcheck_description: "UPS mains status for {{ ups_name }}" + healthcheck_check: ups-status + healthcheck_ups_name: "{{ ups_name }}" + healthcheck_interval: "5min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external" + healthcheck_push_token: "{{ host_token }}" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml new file mode 100644 index 0000000..ab564a1 --- /dev/null +++ b/ansible/infra/401_service_monitoring.yml @@ -0,0 +1,107 @@ +--- +# Is each systemd-deployed service actually running? +# +# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before +# it alarms, so a reboot or a slow check does not page anyone, but a host that +# stops reporting does. +# +# This closes the gap that let a real bug run unnoticed: a backup script left +# forgejo, lnbits, headscale and memos stopped, and NOTHING caught it. The dumps +# exited 0, the artefacts were correct, the deploy said failed=0, and liveness +# only proves the HOST is up - not that anything on it is serving. +# +# One endpoint PER UNIT, not per host. A host running four services needs four +# endpoints, or a single red light says "something on vipy is down" without +# saying which - and that is the question you actually have at 3am. But only ONE +# timer per host: the check iterates that host's units and pushes a result for +# each, the same way check-backups.sh reports per source. Four units on vipy +# would otherwise mean four scripts, four services and four timers. +# +# Which units each host runs is in host_vars//main.yml as +# monitored_services, because "what runs here" is a property of the machine. +# +# Keys are host-qualified because unit names collide - caddy runs on four +# machines. Gatus computes sanitize(group)_sanitize(name), so group "services" +# and name "vipy/caddy" give services_vipy-caddy. + +# ───────────────────────────────────────────────────────────────────────────── +# Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the +# host play above takes minutes, so every endpoint exists before its first push. +# ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the service checks with Gatus + hosts: observability + become: yes + + tasks: + # Two plain steps rather than one clever expression: first collect which + # units each host declares, then flatten that into endpoints. + - name: Collect the units each host declares + ansible.builtin.set_fact: + host_units: "{{ host_units | default([]) + [{'host': item, 'units': hostvars[item].monitored_services}] }}" + loop: "{{ groups['managed'] | sort }}" + when: hostvars[item].monitored_services | default([]) | length > 0 + + - name: Build one endpoint per unit + ansible.builtin.set_fact: + service_endpoints: "{{ service_endpoints | default([]) + [{ + 'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1, + 'group': 'services', + 'token': gatus_push_tokens[item.0.host], + 'heartbeat': '11m'}] }}" + loop: "{{ host_units | subelements('units') }}" + + - name: Register the service endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: services + gatus_endpoint_external: "{{ service_endpoints }}" + +- name: Monitor systemd services on every host that has them + hosts: managed + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" + + tasks: + - name: Is every deployed service running? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: service-health + healthcheck_description: "systemd services on {{ inventory_hostname }}" + healthcheck_check: systemd-units + healthcheck_units: "{{ monitored_services }}" + healthcheck_units_key_prefix: "services_{{ host_key }}" + healthcheck_interval: "5min" + healthcheck_boot_delay: "2min" + # The per-unit results go to keys under this collection; the role's own + # single-result push is unused here, so only the base is set. + healthcheck_push_base: "{{ gatus_api }}" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + when: monitored_services | default([]) | length > 0 diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml new file mode 100644 index 0000000..d924468 --- /dev/null +++ b/ansible/infra/402_public_monitoring.yml @@ -0,0 +1,141 @@ +--- +# Domain expiry, DNS correctness, and public endpoint reachability. +# +# These are the first checks in the estate that PULL rather than push, and that +# is the right way round for them: all three are about how the outside world +# sees us, so they must be measured from outside. Gatus polls from the +# observability host and needs nothing installed anywhere else - there is no +# script, no timer and no token, because nothing is reporting in. +# +# That also means these have no heartbeat. A heartbeat answers "did the thing +# that was supposed to report in do so"; when Gatus does the checking itself, +# failure is immediate and self-evident. + +- name: Register the public-facing checks with Gatus + hosts: observability + become: yes + + vars: + # Expected A records, derived from inventory rather than written down again. + # The estate's recurring bug is an address recorded in a second place and + # then left behind when the machine moved, so the check asserts against + # ansible_host - if a box is renumbered, inventory is the one edit. + dns_records: + - {sub: "{{ subdomains.gatus }}", host: monitoring} + - {sub: "{{ subdomains.headscale }}", host: spacey} + - {sub: "{{ subdomains.vaultwarden }}", host: vipy} + - {sub: "{{ subdomains.forgejo }}", host: vipy} + - {sub: "{{ subdomains.lnbits }}", host: vipy} + - {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy} + - {sub: "{{ subdomains.personal_blog }}", host: vipy} + - {sub: "{{ subdomains.memos }}", host: vipy} + - {sub: "{{ subdomains.mempool }}", host: vipy} + - {sub: "{{ subdomains.datum_gateway }}", host: vipy} + + # A public resolver on purpose: this must test what the internet sees, not + # what a local cache or the tailnet's MagicDNS happens to answer. + dns_resolver: "1.1.1.1" + + # Expected status per site, checked live before being written down. + # 401 is the CORRECT answer for the two behind basic auth - asserting 200 + # there would go green precisely when the auth broke. + public_sites: + - {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401} + - {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200} + - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200} + - {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200} + - {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200} + - {name: avisame, sub: "{{ subdomains.ntfy_emergency_app }}", path: "/", status: 200} + - {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200} + - {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200} + - {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200} + - {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401} + + # Ports published from the edge host by socket_proxy. + public_tcp: + - {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"} + - {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"} + - {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"} + + tasks: + # ── Domain expiry ──────────────────────────────────────────────────────── + # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the + # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole + # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so + # whatever the apex serves - a real site, or the registrar's parking page - + # is irrelevant. + # + # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway + # because it uses a free whois service that must not be hammered. + # 336h = 14 days of runway, because renewal is a manual act at the registrar. + - name: Build the domain endpoints + ansible.builtin.set_fact: + domain_endpoints: "{{ domain_endpoints | default([]) + [{ + 'name': item, + 'group': 'domain', + 'url': 'https://' ~ item, + 'interval': '24h', + 'conditions': ['[DOMAIN_EXPIRATION] > 336h'], + 'alerts': [{'type': 'signal', 'failure-threshold': 1, + 'success-threshold': 1, 'send-on-resolved': true, + 'minimum-reminder-interval': '168h'}]}] }}" + loop: "{{ monitored_domains }}" + + # ── DNS ────────────────────────────────────────────────────────────────── + # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The + # domain check stays at 24h because it does a WHOIS/RDAP lookup against a + # free service. Alert on the first failure - at a 6h interval, waiting for + # three would be nearly a day. + - name: Build the DNS endpoints + ansible.builtin.set_fact: + dns_endpoints: "{{ dns_endpoints | default([]) + [{ + 'name': item.sub ~ '.' ~ root_domain, + 'group': 'dns', + 'url': dns_resolver, + 'interval': '6h', + 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, + 'conditions': ['[DNS_RCODE] == NOERROR', + '[BODY] == ' ~ hostvars[item.host].ansible_host], + 'alerts': [{'type': 'signal', 'failure-threshold': 1, + 'success-threshold': 1, 'send-on-resolved': true, + 'minimum-reminder-interval': '24h'}]}] }}" + loop: "{{ dns_records }}" + + # ── Public HTTP ────────────────────────────────────────────────────────── + # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has + # no heartbeat window, so unlike the push checks the tolerance has to live + # in the threshold - and one failed poll of a public site is usually a blip. + - name: Build the public HTTP endpoints + ansible.builtin.set_fact: + http_endpoints: "{{ http_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'public', + 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, + 'interval': '5m', + 'conditions': ['[STATUS] == ' ~ item.status, + '[CERTIFICATE_EXPIRATION] > 168h'], + 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'success-threshold': 2, 'send-on-resolved': true, + 'minimum-reminder-interval': '6h'}]}] }}" + loop: "{{ public_sites }}" + + # ── Public TCP ─────────────────────────────────────────────────────────── + - name: Build the public TCP endpoints + ansible.builtin.set_fact: + tcp_endpoints: "{{ tcp_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'public', + 'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port, + 'interval': '5m', + 'conditions': ['[CONNECTED] == true'], + 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'success-threshold': 2, 'send-on-resolved': true, + 'minimum-reminder-interval': '6h'}]}] }}" + loop: "{{ public_tcp }}" + + - name: Register the public-facing endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: public + gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}" diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml new file mode 100644 index 0000000..c92b767 --- /dev/null +++ b/ansible/infra/403_service_probe_registration.yml @@ -0,0 +1,71 @@ +--- +# The per-service health probes. +# +# These are NOT the same thing as the systemd checks in infra/401. Those answer +# "is the unit running"; these answer "does the service actually respond" - an +# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch +# from the Mempool backend. A process can be perfectly `active` and useless, +# which is precisely the gap these close. +# +# The checks themselves live in each service's own role, deployed by that +# service's playbook. This play only registers where they report, because the +# endpoints must exist in Gatus before the first push arrives. +# +# They used to push to Uptime Kuma. The scripts now POST with a bearer token +# instead of GETting ?status=up, and each host uses its own token. + +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the per-service probes with Gatus + hosts: observability + become: yes + + vars: + probes: + - {name: bitcoin-knots, host: knots_box_local} + - {name: datum-gateway, host: knots_box_local} + - {name: fulcrum, host: fulcrum_box_local} + - {name: phoenixd, host: vipy} + - {name: forgejo-runner, host: forgejo_runner_local} + - {name: mempool-mariadb, host: mempool_box_local} + - {name: mempool-backend, host: mempool_box_local} + - {name: mempool-frontend, host: mempool_box_local} + + tasks: + - name: Build the probe endpoint list + ansible.builtin.set_fact: + probe_endpoints: "{{ probe_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'probe', + 'token': gatus_push_tokens[item.host], + 'heartbeat': '11m'}] }}" + loop: "{{ probes }}" + + - name: Register the probe endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + gatus_endpoint_name: probes + gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/infra/409_remove_legacy_monitoring.yml b/ansible/infra/409_remove_legacy_monitoring.yml new file mode 100644 index 0000000..8f57e46 --- /dev/null +++ b/ansible/infra/409_remove_legacy_monitoring.yml @@ -0,0 +1,109 @@ +--- +# Remove the Uptime-Kuma-era monitoring that 400/401/402 replaced. +# +# Deleting the playbooks that installed these is NOT enough: the units are on +# the hosts, enabled, and keep firing regardless of what the repo says. Two of +# them still push to https://uptime.contrapeso.xyz every 15 minutes. A playbook +# that is deleted without a cleanup leaves its output running forever, with +# nothing in the repo left to explain it. +# +# What replaced what, all verified against the deployed scripts before removal: +# +# disk-usage-monitor -> disk-usage-healthcheck (infra/400) +# The old one checked ONLY "/" at 80%. The replacement walks every real +# filesystem, excluding tmpfs/devtmpfs/squashfs/overlay, at 85%. Strictly +# more coverage, so nothing is lost. +# +# system-healthcheck -> liveness-healthcheck (infra/400) +# The old script computed uptime and pushed. That is exactly a liveness +# heartbeat and nothing more. +# +# nodito-cpu-temp-monitor -> cpu-temp-healthcheck (infra/400) +# zfs-health-monitor -> zfs-health-healthcheck (infra/400) +# The ZFS check logic was ported verbatim - same five conditions - so only +# the reporting transport changed. +# +# NOT removed, because they are not monitoring: +# zfs-monthly-scrub.{timer,service} the actual scrub (infra/nodito/32) +# pull-backups, check-backups the backup machinery (playbooks/backups) +# +# This play is idempotent and kept permanently rather than run once and deleted: +# on a host that never had these it does nothing, and it guarantees a rebuilt or +# restored machine cannot quietly bring them back. + +- name: Remove the legacy Uptime Kuma monitoring units + hosts: managed + become: yes + + vars: + legacy_units: + - disk-usage-monitor + - system-healthcheck + - nodito-cpu-temp-monitor + - zfs-health-monitor + legacy_dirs: + - /opt/disk-monitoring + - /opt/system-healthcheck + - /opt/nodito-monitoring + - /opt/zfs-monitoring + + tasks: + - name: Find which legacy units exist here + ansible.builtin.stat: + path: "/etc/systemd/system/{{ item.0 }}.{{ item.1 }}" + register: legacy_unit_files + loop: "{{ legacy_units | product(['timer', 'service']) | list }}" + + # Stop and disable BEFORE deleting the unit file: systemd cannot disable a + # unit whose file has already gone, which would leave a dangling symlink in + # multi-user.target.wants and a warning on every daemon-reload. + - name: Stop and disable the legacy units + ansible.builtin.systemd: + name: "{{ item.item.0 }}.{{ item.item.1 }}" + state: stopped + enabled: no + loop: "{{ legacy_unit_files.results }}" + loop_control: + label: "{{ item.item.0 }}.{{ item.item.1 }}" + when: item.stat.exists + failed_when: false + + - name: Remove the legacy unit files + ansible.builtin.file: + path: "/etc/systemd/system/{{ item.item.0 }}.{{ item.item.1 }}" + state: absent + loop: "{{ legacy_unit_files.results }}" + loop_control: + label: "{{ item.item.0 }}.{{ item.item.1 }}" + when: item.stat.exists + + - name: Reload systemd + ansible.builtin.systemd: + daemon_reload: yes + + - name: Remove the legacy monitoring scripts and their logs + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ legacy_dirs }}" + + # An orphan predating all of this: mode 0644, not executable, referenced by + # no unit and no cron entry, pushing to a Kuma monitor. Superseded by + # ups-status-healthcheck. + - name: Remove the orphaned hand-written UPS heartbeat + ansible.builtin.file: + path: /usr/local/bin/ups-heartbeat.sh + state: absent + + - name: Confirm nothing still pushes to Uptime Kuma + ansible.builtin.shell: >- + grep -rl "uptime.contrapeso.xyz" /etc/systemd/system /usr/local/bin /opt 2>/dev/null || true + register: kuma_refs + changed_when: false + + - name: Report any remaining references + ansible.builtin.debug: + msg: >- + {{ 'clean - nothing references Uptime Kuma' + if kuma_refs.stdout | trim | length == 0 + else 'STILL REFERENCING KUMA: ' ~ kuma_refs.stdout_lines | join(', ') }} diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml deleted file mode 100644 index de02f53..0000000 --- a/ansible/infra/410_disk_usage_alerts.yml +++ /dev/null @@ -1,330 +0,0 @@ -- name: Deploy Disk Usage Monitoring - hosts: all - become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml - - ../infra_secrets.yml - - vars: - disk_usage_threshold_percent: 80 - disk_check_interval_minutes: 15 - monitored_mount_point: "/" - monitoring_script_dir: /opt/disk-monitoring - monitoring_script_path: "{{ monitoring_script_dir }}/disk_usage_monitor.sh" - log_file: "{{ monitoring_script_dir }}/disk_usage_monitor.log" - systemd_service_name: disk-usage-monitor - # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname and mount point - set_fact: - monitor_name: "disk-usage-{{ host_name.stdout }}-{{ monitored_mount_point | replace('/', 'root') }}" - monitor_friendly_name: "Disk Usage: {{ host_name.stdout }} ({{ monitored_mount_point }})" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': True, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when usage exceeds {{ disk_usage_threshold_percent }}%" - "{{ (disk_check_interval_minutes * 60) + 60 }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_disk_usage_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for disk monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create disk usage monitoring script - copy: - dest: "{{ monitoring_script_path }}" - content: | - #!/bin/bash - - # Disk Usage Monitoring Script - # Monitors disk usage and sends alerts to Uptime Kuma - # Mode: "No news is good news" - only sends alerts when disk usage is HIGH - - LOG_FILE="{{ log_file }}" - USAGE_THRESHOLD="{{ disk_usage_threshold_percent }}" - UPTIME_KUMA_URL="{{ uptime_kuma_disk_usage_push_url }}" - MOUNT_POINT="{{ monitored_mount_point }}" - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to get disk usage percentage - get_disk_usage() { - local mount_point="$1" - local usage="" - - # Get disk usage percentage (without % sign) - usage=$(df -h "$mount_point" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') - - if [ -z "$usage" ]; then - log_message "ERROR: Could not read disk usage for $mount_point" - return 1 - fi - - echo "$usage" - } - - # Function to get disk usage details - get_disk_details() { - local mount_point="$1" - df -h "$mount_point" 2>/dev/null | awk 'NR==2 {print "Used: "$3" / Total: "$2" ("$5" full)"}' - } - - # Function to send alert to Uptime Kuma when disk usage exceeds threshold - # With upside-down mode enabled, sending status=up will trigger an alert - send_uptime_kuma_alert() { - local usage="$1" - local details="$2" - local message="DISK FULL WARNING: ${MOUNT_POINT} is ${usage}% full (Threshold: ${USAGE_THRESHOLD}%) - ${details}" - - log_message "ALERT: $message" - - # Send push notification to Uptime Kuma with status=up - # In upside-down mode, status=up is treated as down/alert - response=$(curl -s -w "\n%{http_code}" -G \ - --data-urlencode "status=up" \ - --data-urlencode "msg=$message" \ - "$UPTIME_KUMA_URL" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" - else - log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" - fi - } - - # Main monitoring logic - main() { - log_message "Starting disk usage check for $MOUNT_POINT" - - # Get current disk usage - current_usage=$(get_disk_usage "$MOUNT_POINT") - - if [ $? -ne 0 ] || [ -z "$current_usage" ]; then - log_message "ERROR: Could not read disk usage" - exit 1 - fi - - # Get disk details - disk_details=$(get_disk_details "$MOUNT_POINT") - - log_message "Current disk usage: ${current_usage}% - $disk_details" - - # Check if usage exceeds threshold - if [ "$current_usage" -gt "$USAGE_THRESHOLD" ]; then - log_message "WARNING: Disk usage ${current_usage}% exceeds threshold ${USAGE_THRESHOLD}%" - send_uptime_kuma_alert "$current_usage" "$disk_details" - else - log_message "Disk usage is within normal range - no alert needed (no news is good news)" - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for disk usage monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.service" - content: | - [Unit] - Description=Disk Usage Monitor - After=network.target - - [Service] - Type=oneshot - ExecStart={{ monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for disk usage monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" - content: | - [Unit] - Description=Run Disk Usage Monitor every {{ disk_check_interval_minutes }} minute(s) - Requires={{ systemd_service_name }}.service - - [Timer] - OnBootSec={{ disk_check_interval_minutes }}min - OnUnitActiveSec={{ disk_check_interval_minutes }}min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start disk usage monitoring timer - systemd: - name: "{{ systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test disk usage monitoring script - command: "{{ monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "Disk usage monitoring script failed to execute properly" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml deleted file mode 100644 index 2580ff0..0000000 --- a/ansible/infra/420_system_healthcheck.yml +++ /dev/null @@ -1,312 +0,0 @@ -- name: Deploy System Healthcheck Monitoring - hosts: all - become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml - - ../infra_secrets.yml - - vars: - healthcheck_interval_seconds: 60 # Send healthcheck every 60 seconds (1 minute) - healthcheck_timeout_seconds: 90 # Uptime Kuma should alert if no ping received within 90s - healthcheck_retries: 1 # Number of retries before alerting - monitoring_script_dir: /opt/system-healthcheck - monitoring_script_path: "{{ monitoring_script_dir }}/system_healthcheck.sh" - log_file: "{{ monitoring_script_dir }}/system_healthcheck.log" - systemd_service_name: system-healthcheck - # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "system-healthcheck-{{ host_name.stdout }}" - monitor_friendly_name: "System Healthcheck: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_healthcheck_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_healthcheck_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Regular healthcheck ping every {{ healthcheck_interval_seconds }}s" - "{{ healthcheck_timeout_seconds }}" - "{{ healthcheck_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_healthcheck_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for healthcheck monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create system healthcheck script - copy: - dest: "{{ monitoring_script_path }}" - content: | - #!/bin/bash - - # System Healthcheck Script - # Sends regular heartbeat pings to Uptime Kuma - # This ensures the system is running and able to communicate - - LOG_FILE="{{ log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_healthcheck_push_url }}" - HOSTNAME=$(hostname) - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to send healthcheck ping to Uptime Kuma - send_healthcheck() { - local uptime_seconds=$(awk '{print int($1)}' /proc/uptime) - local uptime_days=$((uptime_seconds / 86400)) - local uptime_hours=$(((uptime_seconds % 86400) / 3600)) - local uptime_minutes=$(((uptime_seconds % 3600) / 60)) - - local message="System healthy - Uptime: ${uptime_days}d ${uptime_hours}h ${uptime_minutes}m" - - log_message "Sending healthcheck ping: $message" - - # Send push notification to Uptime Kuma with status=up - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Healthcheck ping sent successfully (HTTP $http_code)" - else - log_message "ERROR: Failed to send healthcheck ping (HTTP $http_code)" - return 1 - fi - } - - # Main healthcheck logic - main() { - log_message "Starting system healthcheck for $HOSTNAME" - - # Send healthcheck ping - if send_healthcheck; then - log_message "Healthcheck completed successfully" - else - log_message "ERROR: Healthcheck failed" - exit 1 - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for system healthcheck - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.service" - content: | - [Unit] - Description=System Healthcheck Monitor - After=network.target - - [Service] - Type=oneshot - ExecStart={{ monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for system healthcheck - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" - content: | - [Unit] - Description=Run System Healthcheck every minute - Requires={{ systemd_service_name }}.service - - [Timer] - OnBootSec=30sec - OnUnitActiveSec={{ healthcheck_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start system healthcheck timer - systemd: - name: "{{ systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test system healthcheck script - command: "{{ monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "System healthcheck script failed to execute properly" - - - name: Display monitor information - debug: - msg: | - ✓ System healthcheck monitoring deployed successfully! - - Monitor Name: {{ monitor_friendly_name }} - Monitor Group: {{ uptime_kuma_monitor_group }} - Healthcheck Interval: Every {{ healthcheck_interval_seconds }} seconds (1 minute) - Timeout: {{ healthcheck_timeout_seconds }} seconds (90s) - Retries: {{ healthcheck_retries }} - - The system will send a heartbeat ping every minute. - Uptime Kuma will alert if no ping is received within 90 seconds (with 1 retry). - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_healthcheck_monitor.py - state: absent - delegate_to: localhost - become: no - diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml deleted file mode 100644 index 3b87102..0000000 --- a/ansible/infra/430_cpu_temp_alerts.yml +++ /dev/null @@ -1,316 +0,0 @@ -- name: Deploy CPU Temperature Monitoring - hosts: nodito_host - become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml - - ../infra_secrets.yml - - vars: - temp_threshold_celsius: 80 - temp_check_interval_minutes: 1 - monitoring_script_dir: /opt/nodito-monitoring - monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" - log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" - systemd_service_name: nodito-cpu-temp-monitor - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "cpu-temp-{{ host_name.stdout }}" - monitor_friendly_name: "CPU Temperature: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma CPU temperature monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_cpu_temp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - monitors = api.get_monitors() - notifications = api.get_notifications() - - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name=group_name) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': True, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - api.edit_monitor(existing_monitor['id'], **monitor_data) - else: - api.add_monitor(**monitor_data) - - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_cpu_temp_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when temperature exceeds {{ temp_threshold_celsius }}°C" - "{{ (temp_check_interval_minutes * 60) + 60 }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_cpu_temp_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for temperature monitoring - package: - name: - - lm-sensors - - curl - - jq - - bc - state: present - - - name: Create monitoring script directory - file: - path: "{{ monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create CPU temperature monitoring script - copy: - dest: "{{ monitoring_script_path }}" - content: | - #!/bin/bash - - # CPU Temperature Monitoring Script - # Monitors CPU temperature and sends alerts to Uptime Kuma - - LOG_FILE="{{ log_file }}" - TEMP_THRESHOLD="{{ temp_threshold_celsius }}" - UPTIME_KUMA_URL="{{ uptime_kuma_cpu_temp_push_url }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - get_cpu_temp() { - local temp="" - - if command -v sensors >/dev/null 2>&1; then - temp=$(sensors 2>/dev/null | grep -E "Core 0|Package id 0|Tdie|Tctl" | head -1 | grep -oE '[0-9]+\.[0-9]+°C' | grep -oE '[0-9]+\.[0-9]+') - fi - - if [ -z "$temp" ] && [ -f /sys/class/thermal/thermal_zone0/temp ]; then - temp=$(cat /sys/class/thermal/thermal_zone0/temp) - temp=$(echo "scale=1; $temp/1000" | bc -l 2>/dev/null || echo "$temp") - fi - - if [ -z "$temp" ] && command -v acpi >/dev/null 2>&1; then - temp=$(acpi -t 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1) - fi - - echo "$temp" - } - - send_uptime_kuma_alert() { - local temp="$1" - local message="CPU Temperature Alert: ${temp}°C (Threshold: ${TEMP_THRESHOLD}°C)" - - log_message "ALERT: $message" - - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/°/%C2%B0/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g') - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" - else - log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" - fi - } - - main() { - log_message "Starting CPU temperature check" - - current_temp=$(get_cpu_temp) - - if [ -z "$current_temp" ]; then - log_message "ERROR: Could not read CPU temperature" - exit 1 - fi - - log_message "Current CPU temperature: ${current_temp}°C" - - if (( $(echo "$current_temp > $TEMP_THRESHOLD" | bc -l) )); then - log_message "WARNING: CPU temperature ${current_temp}°C exceeds threshold ${TEMP_THRESHOLD}°C" - send_uptime_kuma_alert "$current_temp" - else - log_message "CPU temperature is within normal range" - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for CPU temperature monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.service" - content: | - [Unit] - Description=CPU Temperature Monitor - After=network.target - - [Service] - Type=oneshot - ExecStart={{ monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for CPU temperature monitoring - copy: - dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" - content: | - [Unit] - Description=Run CPU Temperature Monitor every {{ temp_check_interval_minutes }} minute(s) - Requires={{ systemd_service_name }}.service - - [Timer] - OnBootSec={{ temp_check_interval_minutes }}min - OnUnitActiveSec={{ temp_check_interval_minutes }}min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start CPU temperature monitoring timer - systemd: - name: "{{ systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test CPU temperature monitoring script - command: "{{ monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "CPU temperature monitoring script failed to execute properly" - - - name: Display monitoring configuration - debug: - msg: - - "CPU Temperature Monitoring configured successfully" - - "Temperature threshold: {{ temp_threshold_celsius }}°C" - - "Check interval: {{ temp_check_interval_minutes }} minute(s)" - - "Monitor Name: {{ monitor_friendly_name }}" - - "Monitor Group: {{ uptime_kuma_monitor_group }}" - - "Uptime Kuma Push URL: {{ uptime_kuma_cpu_temp_push_url }}" - - "Monitoring script: {{ monitoring_script_path }}" - - "Systemd Service: {{ systemd_service_name }}.service" - - "Systemd Timer: {{ systemd_service_name }}.timer" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_cpu_temp_monitor.py - state: absent - delegate_to: localhost - become: no - diff --git a/ansible/infra/900_install_rsync.yml b/ansible/infra/900_install_rsync.yml index c0b7318..b690009 100644 --- a/ansible/infra/900_install_rsync.yml +++ b/ansible/infra/900_install_rsync.yml @@ -1,7 +1,5 @@ - name: Install rsync - hosts: all - vars_files: - - ../infra_vars.yml + hosts: managed become: true tasks: diff --git a/ansible/infra/910_docker_playbook.yml b/ansible/infra/910_docker_playbook.yml index f137b6a..62b7147 100644 --- a/ansible/infra/910_docker_playbook.yml +++ b/ansible/infra/910_docker_playbook.yml @@ -1,5 +1,5 @@ - name: Install Docker and Docker Compose on Debian 12 - hosts: all + hosts: managed become: yes tasks: diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 8d06d44..4decb5d 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -1,14 +1,10 @@ - name: Join machine to headscale mesh network - hosts: all + hosts: managed become: yes - vars_files: - - ../infra_vars.yml - - ../services_config.yml vars: headscale_host_name: "spacey" headscale_subdomain: "{{ subdomains.headscale }}" headscale_domain: "https://{{ headscale_subdomain }}.{{ root_domain }}" - headscale_namespace: "{{ service_settings.headscale.namespace }}" tasks: - name: Set facts for headscale server connection diff --git a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml index 02c6679..4edab06 100644 --- a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml +++ b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml @@ -1,8 +1,6 @@ - name: Bootstrap Nodito SSH Key Access - hosts: nodito_host + hosts: hypervisor become: true - vars_files: - - ../infra_vars.yml tasks: - name: Install sudo package diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml index b0be2ef..0fab184 100644 --- a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -1,8 +1,6 @@ - name: Switch Proxmox VE from Enterprise to Community Repositories - hosts: nodito_host + hosts: hypervisor become: true - vars_files: - - ../infra_vars.yml tasks: - name: Check for deb822 sources format diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index cb72328..2efbf8c 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -1,9 +1,6 @@ - name: Setup ZFS RAID 1 Pool for Proxmox Storage - hosts: nodito_host + hosts: hypervisor become: true - vars_files: - - ../infra_vars.yml - - nodito_vars.yml tasks: - name: Verify Proxmox VE is running @@ -139,17 +136,18 @@ Config file exists: {{ storage_cfg_file.stat.exists }} Storage check result: {{ storage_exists_check.rc }} Pool exists: {{ zfs_pool_exists.rc == 0 }} - Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }} Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }} - - name: Remove existing storage if it exists - command: pvesm remove {{ zfs_pool_name }} - register: pvesm_remove_result - failed_when: false - when: - - zfs_pool_exists.rc == 0 - - storage_exists_check.rc == 0 - + # Registration is add-only on purpose. There used to be a "Remove existing + # storage if it exists" task here that ran `pvesm remove` whenever the + # storage WAS present, paired with an add that only ran when it was ABSENT. + # The two conditions are mutually exclusive, so a real run against a + # correctly-configured hypervisor removed the storage entry backing every VM + # and never put it back. It would also have dropped `mountpoint /var/lib/vz`, + # which the live entry has and which `pvesm add` below does not set. + # + # If the storage entry ever needs its options changed, edit + # /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here. - name: Add ZFS pool storage to Proxmox using pvesm command: > pvesm add zfspool {{ zfs_pool_name }} @@ -171,498 +169,56 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" -- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs - hosts: nodito +# ───────────────────────────────────────────────────────────────────────────── +# The monthly scrub. +# +# The ZFS HEALTH CHECK that used to share this play is gone: it is now the +# zfs-health check in infra/400_host_monitoring.yml, which carries the same five +# conditions - pool state, device states, resilver in progress, read/write/ +# checksum errors, and errors from the last scan - but reports to Gatus like +# every other host check instead of owning its own push plumbing. +# +# The scrub itself stays here, because it is not monitoring: it is the +# maintenance that gives the health check something true to report. A pool that +# is never scrubbed has no idea whether it is healthy. +# ───────────────────────────────────────────────────────────────────────────── +- name: Schedule the monthly ZFS scrub + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml vars: - zfs_check_interval_seconds: 86400 # 24 hours - zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) - zfs_check_retries: 1 - zfs_monitoring_script_dir: /opt/zfs-monitoring - zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" - zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" - zfs_systemd_health_service_name: zfs-health-monitor zfs_systemd_scrub_service_name: zfs-monthly-scrub - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "zfs-health-{{ host_name.stdout }}" - monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma ZFS health monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_zfs_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma ZFS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_zfs_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" - "{{ zfs_check_timeout_seconds }}" - "{{ zfs_check_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for ZFS monitoring - package: - name: - - curl - - jq - state: present - - - name: Create monitoring script directory - file: - path: "{{ zfs_monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create ZFS health monitoring script - copy: - dest: "{{ zfs_monitoring_script_path }}" - content: | - #!/bin/bash - - # ZFS Pool Health Monitoring Script - # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy - # If any issues detected, does NOT send heartbeat (triggers timeout alert) - - LOG_FILE="{{ zfs_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" - POOL_NAME="{{ zfs_pool_name }}" - HOSTNAME=$(hostname) - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to check pool health using JSON output - check_pool_health() { - local pool="$1" - local issues_found=0 - - # Get pool status as JSON - local pool_json - pool_json=$(zpool status -j "$pool" 2>&1) - - if [ $? -ne 0 ]; then - log_message "ERROR: Failed to get pool status for $pool" - log_message " -> $pool_json" - return 1 - fi - - # Check 1: Pool state must be ONLINE - local pool_state - pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') - - if [ "$pool_state" != "ONLINE" ]; then - log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" - issues_found=1 - else - log_message "OK: Pool state is ONLINE" - fi - - # Check 2: Check all vdevs and devices for non-ONLINE states - local bad_states - bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.state? and .state != "ONLINE") | - "\(.name // "unknown"): \(.state)" - ' 2>/dev/null) - - if [ -n "$bad_states" ]; then - log_message "ISSUE: Found devices not in ONLINE state:" - echo "$bad_states" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: All devices are ONLINE" - fi - - # Check 3: Check for resilvering in progress - local scan_function scan_state - scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - - if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then - local resilver_progress - resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') - log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" - issues_found=1 - fi - - # Check 4: Check for read/write/checksum errors on all devices - # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison - local devices_with_errors - devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | - "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" - ' 2>/dev/null) - - if [ -n "$devices_with_errors" ]; then - log_message "ISSUE: Found devices with I/O errors:" - echo "$devices_with_errors" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: No read/write/checksum errors detected" - fi - - # Check 5: Check for scan errors (from last scrub/resilver) - local scan_errors - scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') - - if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then - log_message "ISSUE: Last scan reported $scan_errors errors" - issues_found=1 - else - log_message "OK: No scan errors" - fi - - return $issues_found - } - - # Function to get last scrub info for status message - get_scrub_info() { - local pool="$1" - local pool_json - pool_json=$(zpool status -j "$pool" 2>/dev/null) - - local scan_func scan_state scan_start - scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') - - if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then - echo "scrub in progress (started $scan_start)" - elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then - echo "last scrub: $scan_start" - else - echo "no scrub history" - fi - } - - # Function to send heartbeat to Uptime Kuma - send_heartbeat() { - local message="$1" - - log_message "Sending heartbeat to Uptime Kuma: $message" - - # URL encode the message - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent successfully (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - # Main health check logic - main() { - log_message "==========================================" - log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" - - # Run all health checks - if check_pool_health "$POOL_NAME"; then - # All checks passed - send heartbeat - local scrub_info - scrub_info=$(get_scrub_info "$POOL_NAME") - - local message="Pool $POOL_NAME healthy ($scrub_info)" - send_heartbeat "$message" - - log_message "Health check completed: ALL OK" - exit 0 - else - # Issues found - do NOT send heartbeat (will trigger timeout alert) - log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" - log_message "Uptime Kuma will alert after timeout due to missing heartbeat" - exit 1 - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for ZFS health monitoring - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" - content: | - [Unit] - Description=ZFS Pool Health Monitor - After=zfs.target network.target - - [Service] - Type=oneshot - ExecStart={{ zfs_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for daily ZFS health monitoring - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Pool Health Monitor daily - Requires={{ zfs_systemd_health_service_name }}.service - - [Timer] - OnBootSec=5min - OnUnitActiveSec={{ zfs_check_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - name: Create systemd service for ZFS monthly scrub - copy: + template: + src: templates/zfs-monthly-scrub.service.j2 dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" - content: | - [Unit] - Description=ZFS Monthly Scrub for {{ zfs_pool_name }} - After=zfs.target - - [Service] - Type=oneshot - ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target owner: root group: root mode: '0644' - name: Create systemd timer for monthly ZFS scrub - copy: + template: + src: templates/zfs-monthly-scrub.timer.j2 dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Scrub on last day of every month at 4:00 AM - Requires={{ zfs_systemd_scrub_service_name }}.service - - [Timer] - OnCalendar=*-*~01 04:00:00 - Persistent=true - - [Install] - WantedBy=timers.target owner: root group: root mode: '0644' - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start ZFS health monitoring timer - systemd: - name: "{{ zfs_systemd_health_service_name }}.timer" - enabled: yes - state: started - - - name: Enable and start ZFS monthly scrub timer + - name: Enable and start the monthly scrub timer systemd: name: "{{ zfs_systemd_scrub_service_name }}.timer" enabled: yes state: started + daemon_reload: yes - - name: Test ZFS health monitoring script - command: "{{ zfs_monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "ZFS health monitoring script failed - check pool health" - - - name: Display monitoring configuration + - name: Report the scrub schedule debug: - msg: | - ✓ ZFS Pool Health Monitoring deployed successfully! - - Monitor Name: {{ monitor_friendly_name }} - Monitor Group: {{ uptime_kuma_monitor_group }} - Pool Name: {{ zfs_pool_name }} - - Health Check: - - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) - - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) - - Script: {{ zfs_monitoring_script_path }} - - Log: {{ zfs_log_file }} - - Service: {{ zfs_systemd_health_service_name }}.service - - Timer: {{ zfs_systemd_health_service_name }}.timer - - Monthly Scrub: - - Schedule: Last day of month at 4:00 AM - - Service: {{ zfs_systemd_scrub_service_name }}.service - - Timer: {{ zfs_systemd_scrub_service_name }}.timer - - Conditions monitored: - - Pool state (must be ONLINE) - - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) - - Resilver status (alerts if resilvering) - - Read/Write/Checksum errors - - Scrub errors - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_zfs_monitor.py - state: absent - delegate_to: localhost - become: no + msg: >- + Monthly scrub of {{ zfs_pool_name }}: + last day of each month at 04:00. + Health is reported separately by the zfs-health check + (infra/400_host_monitoring.yml). diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index e8f8332..faf93a5 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -1,9 +1,6 @@ - name: Create Proxmox template from Debian cloud image (no VM clone) - hosts: nodito_host + hosts: hypervisor become: true - vars_files: - - ../../infra_vars.yml - - nodito_vars.yml vars: # Defaults (override via vars_files or --extra-vars as needed) diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index 02468d5..12fe720 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -1,12 +1,33 @@ - name: Setup NUT (Network UPS Tools) for CyberPower UPS - hosts: nodito_host + hosts: hypervisor become: true - vars_files: - - ../../infra_vars.yml - - nodito_vars.yml - - nodito_secrets.yml tasks: + # ------------------------------------------------------------------ + # Safety catch + # + # /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by + # hand in January 2026 and carry a working password. host_vars/nodito/vault.yml + # (formerly infra/nodito/nodito_secrets.yml) still holds the literal string + # CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that + # working pair with a placeholder and restart NUT - leaving the hypervisor's + # UPS unmonitored and unable to trigger a clean shutdown on mains loss. + # + # Until the real password is put in the vault, stop here. + # ansible-vault edit host_vars/nodito/vault.yml + # ------------------------------------------------------------------ + - name: Refuse to run with a placeholder UPS password + assert: + that: + - ups_password is defined + - ups_password | length > 0 + - ups_password != "CHANGE_ME_TO_SECURE_PASSWORD" + fail_msg: >- + ups_password is unset or still the placeholder. Applying this play would + overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on + nodito and restart NUT. Put the real password in the vault first: + ansible-vault edit host_vars/nodito/vault.yml + # ------------------------------------------------------------------ # Installation # ------------------------------------------------------------------ @@ -75,90 +96,45 @@ # Configuration files # ------------------------------------------------------------------ - name: Configure NUT mode (standalone) - copy: + template: dest: /etc/nut/nut.conf - content: | - # Managed by Ansible - MODE=standalone + src: templates/nut.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure UPS device - copy: + template: dest: /etc/nut/ups.conf - content: | - # Managed by Ansible - [{{ ups_name }}] - driver = {{ ups_driver }} - port = {{ ups_port }} - desc = "{{ ups_desc }}" - offdelay = {{ ups_offdelay }} - ondelay = {{ ups_ondelay }} + src: templates/ups.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd to listen on localhost - copy: + template: dest: /etc/nut/upsd.conf - content: | - # Managed by Ansible - LISTEN 127.0.0.1 3493 + src: templates/upsd.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd users - copy: + template: dest: /etc/nut/upsd.users - content: | - # Managed by Ansible - [{{ ups_user }}] - password = {{ ups_password }} - upsmon master + src: templates/upsd.users.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsmon - copy: + template: dest: /etc/nut/upsmon.conf - content: | - # Managed by Ansible - MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master - - MINSUPPLIES 1 - SHUTDOWNCMD "/sbin/shutdown -h +0" - POLLFREQ 5 - POLLFREQALERT 5 - HOSTSYNC 15 - DEADTIME 15 - POWERDOWNFLAG /etc/killpower - - # Notifications - NOTIFYMSG ONLINE "UPS %s on line power" - NOTIFYMSG ONBATT "UPS %s on battery" - NOTIFYMSG LOWBATT "UPS %s battery is low" - NOTIFYMSG FSD "UPS %s: forced shutdown in progress" - NOTIFYMSG COMMOK "Communications with UPS %s established" - NOTIFYMSG COMMBAD "Communications with UPS %s lost" - NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" - NOTIFYMSG REPLBATT "UPS %s battery needs replacing" - - # Log all events to syslog - NOTIFYFLAG ONLINE SYSLOG - NOTIFYFLAG ONBATT SYSLOG - NOTIFYFLAG LOWBATT SYSLOG - NOTIFYFLAG FSD SYSLOG - NOTIFYFLAG COMMOK SYSLOG - NOTIFYFLAG COMMBAD SYSLOG - NOTIFYFLAG SHUTDOWN SYSLOG - NOTIFYFLAG REPLBATT SYSLOG + src: templates/upsmon.conf.j2 owner: root group: nut mode: "0640" @@ -249,321 +225,11 @@ - nut-server - nut-monitor - -- name: Setup UPS Heartbeat Monitoring with Uptime Kuma - hosts: nodito - become: true - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml - - nodito_secrets.yml - - vars: - ups_heartbeat_interval_seconds: 60 - ups_heartbeat_timeout_seconds: 120 - ups_heartbeat_retries: 1 - ups_monitoring_script_dir: /opt/ups-monitoring - ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" - ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" - ups_systemd_service_name: ups-heartbeat - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "ups-{{ host_name.stdout }}" - monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma UPS monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_ups_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - monitors = api.get_monitors() - notifications = api.get_notifications() - - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name=group_name) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - api.edit_monitor(existing_monitor['id'], **monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - api.add_monitor(**monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma UPS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_ups_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" - "{{ ups_heartbeat_timeout_seconds }}" - "{{ ups_heartbeat_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL as fact - set_fact: - uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - - name: Install required packages for UPS monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ ups_monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create UPS heartbeat monitoring script - copy: - dest: "{{ ups_monitoring_script_path }}" - content: | - #!/bin/bash - - # UPS Heartbeat Monitoring Script - # Sends heartbeat to Uptime Kuma only when UPS is on mains power - # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) - - LOG_FILE="{{ ups_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" - UPS_NAME="{{ ups_name }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - send_heartbeat() { - local message="$1" - - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent: $message (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - main() { - local status charge runtime load - - status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) - - if [ -z "$status" ]; then - log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" - exit 1 - fi - - charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) - runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) - load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) - - if [[ "$status" == *"OL"* ]]; then - local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" - send_heartbeat "$message" - exit 0 - else - log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" - exit 1 - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for UPS heartbeat - copy: - dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" - content: | - [Unit] - Description=UPS Heartbeat Monitor - After=network.target nut-monitor.service - - [Service] - Type=oneshot - ExecStart={{ ups_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for UPS heartbeat - copy: - dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" - content: | - [Unit] - Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds - Requires={{ ups_systemd_service_name }}.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start UPS heartbeat timer - systemd: - name: "{{ ups_systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test UPS heartbeat script - command: "{{ ups_monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "UPS heartbeat script failed - check UPS status and communication" - - - name: Display monitoring configuration - debug: - msg: - - "UPS Monitoring configured successfully" - - "" - - "NUT Configuration:" - - " UPS Name: {{ ups_name }}" - - " UPS Description: {{ ups_desc }}" - - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" - - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" - - "" - - "Uptime Kuma Monitoring:" - - " Monitor Name: {{ monitor_friendly_name }}" - - " Monitor Group: {{ uptime_kuma_monitor_group }}" - - " Push URL: {{ uptime_kuma_ups_push_url }}" - - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" - - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" - - "" - - "Scripts and Services:" - - " Script: {{ ups_monitoring_script_path }}" - - " Log: {{ ups_log_file }}" - - " Service: {{ ups_systemd_service_name }}.service" - - " Timer: {{ ups_systemd_service_name }}.timer" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_ups_monitor.py - state: absent - delegate_to: localhost - become: no +# The UPS heartbeat play that used to live here is gone. What it deployed - +# /opt/ups-monitoring plus a ups-heartbeat timer - is now the ups-status check +# in infra/400_host_monitoring.yml, which reports to Gatus like every other +# host check instead of carrying its own push plumbing. +# +# This playbook is now purely NUT setup: the driver, upsd, upsmon and the +# shutdown behaviour. Monitoring whether the UPS is on mains is a separate +# concern and belongs with the other host checks. diff --git a/ansible/infra/nodito/templates/nut.conf.j2 b/ansible/infra/nodito/templates/nut.conf.j2 new file mode 100644 index 0000000..1f8a72f --- /dev/null +++ b/ansible/infra/nodito/templates/nut.conf.j2 @@ -0,0 +1,2 @@ +# Managed by Ansible +MODE=standalone diff --git a/ansible/infra/nodito/templates/ups.conf.j2 b/ansible/infra/nodito/templates/ups.conf.j2 new file mode 100644 index 0000000..0fa0a60 --- /dev/null +++ b/ansible/infra/nodito/templates/ups.conf.j2 @@ -0,0 +1,9 @@ +# Managed by Ansible +maxretry = 3 + +[{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} diff --git a/ansible/infra/nodito/templates/upsd.conf.j2 b/ansible/infra/nodito/templates/upsd.conf.j2 new file mode 100644 index 0000000..baddf21 --- /dev/null +++ b/ansible/infra/nodito/templates/upsd.conf.j2 @@ -0,0 +1,2 @@ +# Managed by Ansible +LISTEN 127.0.0.1 3493 diff --git a/ansible/infra/nodito/templates/upsd.users.j2 b/ansible/infra/nodito/templates/upsd.users.j2 new file mode 100644 index 0000000..8d43784 --- /dev/null +++ b/ansible/infra/nodito/templates/upsd.users.j2 @@ -0,0 +1,4 @@ +# Managed by Ansible +[{{ ups_user }}] + password = {{ ups_password }} + upsmon master diff --git a/ansible/infra/nodito/templates/upsmon.conf.j2 b/ansible/infra/nodito/templates/upsmon.conf.j2 new file mode 100644 index 0000000..dc301e1 --- /dev/null +++ b/ansible/infra/nodito/templates/upsmon.conf.j2 @@ -0,0 +1,34 @@ +# Managed by Ansible +MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + +MINSUPPLIES 1 +SHUTDOWNCMD "/sbin/shutdown -h +0" +POLLFREQ 5 +POLLFREQALERT 5 +HOSTSYNC 15 +DEADTIME 15 +POWERDOWNFLAG "/etc/killpower" +OFFDURATION 30 +RBWARNTIME 43200 +NOCOMMWARNTIME 300 +FINALDELAY 5 + +# Notifications +NOTIFYMSG ONLINE "UPS %s on line power" +NOTIFYMSG ONBATT "UPS %s on battery" +NOTIFYMSG LOWBATT "UPS %s battery is low" +NOTIFYMSG FSD "UPS %s: forced shutdown in progress" +NOTIFYMSG COMMOK "Communications with UPS %s established" +NOTIFYMSG COMMBAD "Communications with UPS %s lost" +NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" +NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + +# Log all events to syslog +NOTIFYFLAG ONLINE SYSLOG +NOTIFYFLAG ONBATT SYSLOG +NOTIFYFLAG LOWBATT SYSLOG +NOTIFYFLAG FSD SYSLOG +NOTIFYFLAG COMMOK SYSLOG +NOTIFYFLAG COMMBAD SYSLOG +NOTIFYFLAG SHUTDOWN SYSLOG +NOTIFYFLAG REPLBATT SYSLOG diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 new file mode 100644 index 0000000..04b3514 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=ZFS Monthly Scrub for {{ zfs_pool_name }} +After=zfs.target + +[Service] +Type=oneshot +ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} +User=root +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 new file mode 100644 index 0000000..36ad3d7 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Run ZFS Scrub on last day of every month at 4:00 AM +Requires={{ zfs_systemd_scrub_service_name }}.service + +[Timer] +OnCalendar=*-*~01 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example deleted file mode 100644 index d539282..0000000 --- a/ansible/infra_secrets.yml.example +++ /dev/null @@ -1,40 +0,0 @@ -# Uptime Kuma login credentials -# Used by the disk monitoring playbook to create monitors automatically - -uptime_kuma_username: "admin" -uptime_kuma_password: "your_password_here" - -# ntfy credentials -# Used for notification channel setup in Uptime Kuma - -ntfy_username: "your_ntfy_username" -ntfy_password: "your_ntfy_password" - -# headscale-ui credentials -# Used for HTTP basic authentication via Caddy -# Provide either: -# - headscale_ui_password: plain text password (will be hashed automatically) -# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate) - -headscale_ui_username: "admin" -headscale_ui_password: "your_secure_password_here" -# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password - -bitcoin_rpc_user: "bitcoinrpc" -bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" - -# Mempool MariaDB credentials -# Used by: services/mempool/deploy_mempool_playbook.yml -mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" - -# Forgejo Runner registration token -# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml -# See: services/forgejo-runner/SETUP.md for how to obtain this token -forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE" - -# DATUM Gateway secrets -# Used by: services/datum-gateway/deploy_datum_gateway_playbook.yml -datum_mining_address: "YOUR_BITCOIN_ADDRESS_FOR_BLOCK_REWARDS" -datum_gateway_admin_password: "CHANGE_ME_TO_SECURE_PASSWORD" -datum_dashboard_username: "admin" -datum_dashboard_password_hash: "$2a$14$..." # Generate with: caddy hash-password diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml deleted file mode 100644 index 952df93..0000000 --- a/ansible/infra_vars.yml +++ /dev/null @@ -1,4 +0,0 @@ -new_user: counterweight -ssh_port: 22 -allow_ssh_from: "any" -root_domain: contrapeso.xyz diff --git a/ansible/inventory.ini b/ansible/inventory.ini new file mode 100644 index 0000000..c39439d --- /dev/null +++ b/ansible/inventory.ini @@ -0,0 +1,74 @@ +[vps] +vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +monitoring ansible_host=64.226.70.190 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +[nodito_host] +nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +# Requires the tailnet to be up on the control node. +[nodito_vms] +knots_box_local ansible_host=knots-box lan_ip=192.168.1.135 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +fulcrum_box_local ansible_host=fulcrum-box lan_ip=192.168.1.140 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +mempool_box_local ansible_host=mempool-box lan_ip=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +# Local connection to laptop: this assumes you're running ansible commands from your personal laptop +[lapy] +localhost ansible_connection=local ansible_user=counterweight + +[arbret] +prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +[edge] +vipy + +# The group is `observability`, NOT `monitoring` — there is a HOST named +# `monitoring` on line 5, and a group with the same name makes `hosts: monitoring` +# ambiguous. Ansible resolved it to the host and warned: +# [WARNING]: Found both group and host with same name: monitoring +[observability] +monitoring + +[vpn_control] +spacey + +[hypervisor] +nodito + +[bitcoin] +knots_box_local + +[electrum] +fulcrum_box_local + +[mempool] +mempool_box_local + +[memos] +memos_box_local + +[ci_runner] +forgejo_runner_local + +[control] +localhost + +# Every machine Ansible may configure as a server. +# Deliberately EXCLUDES [control] (your laptop) and [arbret]. +[managed:children] +vps +nodito_host +nodito_vms + +# Hosts that run Caddy and therefore have /etc/caddy/sites-enabled. +[caddy:children] +edge +observability +vpn_control + +[backup_store] +small_backups_local diff --git a/ansible/inventory.ini.example b/ansible/inventory.ini.example deleted file mode 100644 index bde96dd..0000000 --- a/ansible/inventory.ini.example +++ /dev/null @@ -1,16 +0,0 @@ -[vps] -vipy ansible_host=your.services.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key -watchtower ansible_host=your.monitoring.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key -spacey ansible_host=your.headscale.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key - -[nodito_host] -nodito ansible_host=your.proxmox.ip.here ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key ansible_ssh_pass=your_root_password - -[nodito_vms] -# Example node, replace with your VM names and addresses -# memos_box ansible_host=192.168.1.150 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key - -# Local connection to laptop: this assumes you're running ansible commands from your personal laptop -# Make sure to adjust the username -[lapy] -localhost ansible_connection=local ansible_user=your laptop user gpg_recipient=your_email@example.com gpg_key_id=your_gpg_key_id_here \ No newline at end of file diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml new file mode 100644 index 0000000..0be3807 --- /dev/null +++ b/ansible/playbooks/backups.yml @@ -0,0 +1,107 @@ +- name: Configure the offsite backup pull + hosts: backup_store + gather_facts: yes + + tasks: + - name: Ensure the box pulls every source on a timer + ansible.builtin.include_role: + name: backup_store + vars: + # check-backups.sh reports one result per source plus one for the store + # itself, so it needs the collection URL and appends each key. + backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + backup_store_sources: + - name: arbret + source: "arbret@prd-arbret:/opt/arbret/backups/" + retention_days: 90 + - name: headscale + source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" + retention_days: 90 + - name: memos + source: "backup-pull@memos-box:/opt/backups/memos/" + retention_days: 90 + - name: vaultwarden + source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/" + retention_days: 90 + - name: lnbits + source: "backup-pull@prd-vipy:/opt/backups/lnbits/" + retention_days: 90 + - name: forgejo + source: "backup-pull@prd-vipy:/opt/backups/forgejo/" + retention_days: 14 + +# ───────────────────────────────────────────────────────────────────────────── +# Register the backup checks with Gatus. +# +# Two groups on purpose, because they answer different questions and fail for +# different reasons: +# +# backup-dump did the SOURCE produce an artefact? Pushed by each dump right +# after it runs, so a broken dump is visible within minutes. +# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is +# retention pruning? Pushed by check-backups.sh at 05:30. +# +# The store alone could catch almost everything, because the artefact filename +# carries the source's dump timestamp - a source whose timer died still pulls +# "ok" forever, but the timestamp gives it away. What the source side adds is +# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it +# cannot tell you whether the dump broke or the pull did. +# +# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` +# deliberately excludes, so nothing of ours runs there. It is store-checked only. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the backup checks with Gatus + hosts: observability + become: yes + vars: + # Sources we deploy the dump for, and the host each one runs on. + dump_sources: + - {name: headscale, host: spacey} + - {name: memos, host: memos_box_local} + - {name: vaultwarden, host: vipy} + - {name: lnbits, host: vipy} + - {name: forgejo, host: vipy} + store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] + + tasks: + # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter + # than a day plus slack. The store-side check catches the same fault within + # 6h by reading the artefact's dump timestamp out of the filename, so this is + # the slow backstop rather than the primary signal. + - name: Build the dump endpoint list + ansible.builtin.set_fact: + dump_endpoints: "{{ dump_endpoints | default([]) + [{ + 'name': item.name, + 'group': 'backup-dump', + 'token': gatus_push_tokens[item.host], + 'heartbeat': '26h'}] }}" + loop: "{{ dump_sources }}" + + - name: Build the store endpoint list + ansible.builtin.set_fact: + store_endpoints: "{{ store_endpoints | default([]) + [{ + 'name': item, + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '7h'}] }}" + loop: "{{ store_sources }}" + + - name: Register the backup endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + # Push endpoints: the heartbeat window is the tolerance, so alert on + # the first failure rather than waiting for three 7h windows to pass. + gatus_endpoint_default_alerts: + - type: signal + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 12h + gatus_endpoint_name: backups + gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ + 'name': 'pull job', + 'group': 'backup-store', + 'token': gatus_push_tokens['small_backups_local'], + 'heartbeat': '7h'}] }}" diff --git a/ansible/requirements.yml b/ansible/requirements.yml index dd7eef4..16ef1fd 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,11 +1,10 @@ ---- # Ansible Galaxy Collections Requirements # Install with: ansible-galaxy collection install -r requirements.yml -collections: - # Uptime Kuma Ansible Collection - # Used by: infra/41_disk_usage_alerts.yml - # Provides modules to manage Uptime Kuma monitors programmatically - - name: lucasheld.uptime_kuma - version: ">=1.0.0" +# No collections are currently required. +# +# lucasheld.uptime_kuma was pinned here but never used — every monitor was created +# by hand-rolled Python instead. Removed 2026-09-11 along with Uptime Kuma itself. +# See archive/uptime_kuma/. +collections: [] diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md new file mode 100644 index 0000000..a7794ab --- /dev/null +++ b/ansible/roles/backup_source/README.md @@ -0,0 +1,126 @@ +# `backup_source` + +Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a +local directory, prune, on a systemd timer. `small-backups-box` pulls the +directory later (see `backup_store`). + +Modelled on `prd-arbret`, which has been doing exactly this correctly since +before the rest of the estate was migrated. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 +``` + +Produces `/opt/backups/headscale/headscale_.tar.gz.age`, +`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`. + +## Why the source encrypts, not the destination + +`age -r ` is asymmetric and the host holds only the **public** key, so +a compromised host cannot read its own backups — or anyone else's. The scripts +this replaces encrypted with GPG *on the laptop, after the data had already +crossed the network*, which protects the artefact at rest but not in transit. + +The matching identity lives only on lapy and is escrowed. **Lose it and every +artefact everywhere becomes noise**, including arbret's. + +## `backup_source_dump_command` writes to STDOUT + +The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` +with relative paths in `tar` rather than absolute ones: it avoids tar's "removing +leading /" and makes the restore target explicit. + +## Services that are not systemd + +`backup_source_stop_service` runs `systemctl stop/start`. For anything else, +give the pair explicitly — vaultwarden is a docker compose stack, so +`systemctl stop vaultwarden` silently does nothing: + +```yaml +backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" +backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" +``` + +The same EXIT trap wraps both forms. The assert refuses a stop command without a +matching start command, because that combination fails in the one way you would +not notice: the service stops and never comes back. + +## More than one thing to back up + +`tar` takes several paths, so multiple files or directories are normally **one** +artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together, +lnbits captures its data directory and its `.env`. + +Prefer one artefact. A backup should be a consistent snapshot, and two artefacts +written by two runs can drift — you can end up restoring an `.env` that does not +match the database it configures. Pulling a single file back out needs no +unpacking: + +```bash +age -d -i | tar -xzO opt/lnbits/lnbits/.env +``` + +If you genuinely need separate artefacts, call the role twice with different +`backup_source_name`s rather than extending it — but only one call may set +`backup_source_stop_service`, or the service is stopped twice per night. + +The case this shape cannot express is a **database dump plus a file tree** +(`pg_dump` and a media directory, say): you cannot merge those into one stream +without staging plaintext on disk, which is exactly what this design avoids. +None of the current services need it — all are file trees, all stopped for the +dump. A future one that does should use two role calls. + +## Everything here is sqlite, so everything stops + +All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files +present). A live copy of a WAL-mode database can be torn or stale, so each is +stopped for the duration. Measured downtime: under a second for headscale and +memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos +and database. That last one is the real cost of a consistent snapshot; if it +becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy, +not skipping the stop. + +## The trap is the reason this role exists + +When `backup_source_stop_service` is set, the script stops the unit and installs +an EXIT trap that starts it again. Without it, a failed dump leaves the service +down until the next timer fires — **every hand-written script this replaced had +that bug**, and it was only ever masked because their `systemctl stop` failed +first, before anything was stopped. + +Verified on spacey: with the dump forced to fail, the log shows +`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit +failed rather than hiding it), and headscale is `active` afterwards. + +If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed — +which is correct, because nothing was stopped. + +## `.partial` + +The dump writes `.partial` and only `mv`s it into place on success, so +a truncated file is never mistaken for a backup. A failure inside the pipeline +does leave one behind, and the prune glob cannot match it (it ends `.partial`, +not `.tar.gz.age`), so the script clears stale partials at the **start** of each +run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run. + +## `backup_source_stop_service` may be a bare name + +`headscale` and `headscale.service` both work. The unit template normalises it, +because systemd rejects a bare name in `After=` with +`Failed to add dependency ... Invalid argument` — which it logs and then ignores, +so the unit appears to work while carrying no ordering at all. + +## Retention is two-tier + +`backup_source_retention_days` is **local** and short — these hosts are +disk-constrained. The long tail lives on `small-backups-box`, which decides its +own retention per source. Losing the local copy is expected and fine. diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml new file mode 100644 index 0000000..b195190 --- /dev/null +++ b/ansible/roles/backup_source/defaults/main.yml @@ -0,0 +1,44 @@ +--- +# Required +backup_source_name: "" # "headscale" -> headscale_.tar.gz.age +backup_source_description: "" # "Headscale" +backup_source_dump_command: "" # must write the payload to STDOUT + +# Placement +backup_source_dir: "/opt/backups/{{ backup_source_name }}" +backup_source_artifact_suffix: "tar.gz.age" + +# Encryption. Asymmetric: the host holds only the public key and cannot decrypt +# what it produces. +backup_source_recipient: "{{ age_backup_recipient }}" + +# The unprivileged account small-backups-box pulls as. It owns the dump +# directory and nothing else; it deliberately has no sudo. +backup_source_pull_user: backup-pull +backup_source_pull_key: "{{ backup_pull_public_key }}" + +# Safety. Give either a systemd unit, or an explicit pair of commands for +# services that are not systemd-managed (vaultwarden is a docker compose stack). +# Whichever is used, a trap guarantees the restart. +backup_source_stop_service: "" # systemd unit stopped for the dump +backup_source_stop_command: "" # overrides stop_service when set +backup_source_start_command: "" # required alongside stop_command + +# Retention here is LOCAL and short; small-backups-box keeps the long tail. +backup_source_retention_days: 7 + +# Schedule. The box pulls at 04:00, so dumps must land before that. +backup_source_on_calendar: "*-*-* 02:00:00" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Where to report that this dump ran and produced a plausible artefact. +# Gatus external endpoint: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty is valid and is not an error: the unit's exit code is still the answer, +# and the STORE will independently notice a stale dump within ~26h because the +# artefact filename carries this dump's timestamp. Reporting here only buys +# earlier detection and tells you it was the DUMP that broke rather than the +# pull. +backup_source_push_url: "" +backup_source_push_token: "" diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml new file mode 100644 index 0000000..37a7f2b --- /dev/null +++ b/ansible/roles/backup_source/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd for backup units + ansible.builtin.systemd: + daemon_reload: yes diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml new file mode 100644 index 0000000..a964c5d --- /dev/null +++ b/ansible/roles/backup_source/tasks/main.yml @@ -0,0 +1,110 @@ +--- +- name: Assert backup_source parameters are sane + ansible.builtin.assert: + that: + - backup_source_name | length > 0 + - backup_source_description | length > 0 + - backup_source_dump_command | length > 0 + - backup_source_recipient | length > 0 + - backup_source_recipient is match('^age1[0-9a-z]{58}$') + - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0) + fail_msg: >- + backup_source: '{{ backup_source_name | default("") }}' needs a name, + description, dump command and a valid age recipient (age1... 62 chars). + backup_source_stop_command must be paired with backup_source_start_command. + quiet: true + +# Declared here rather than assumed. Stage 1 installed it by hand; this is what +# makes a rebuilt host get it too. +# Cache refresh is best-effort on purpose. An unrelated third-party repo with a +# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource +# key) makes `apt-get update` return warnings, which the apt module treats as a +# hard failure — and that must not stop backups being configured. Installing the +# package is NOT best-effort: if age is genuinely unavailable, the next task fails. +- name: Refresh the apt cache (best effort) + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 3600 + failed_when: false + changed_when: false + +- name: Ensure age is installed + ansible.builtin.apt: + name: + - age + # curl is needed only when backup_source_push_url is set, but installing it + # unconditionally keeps the task idempotent and it is present on every + # Debian host here anyway. + - curl + state: present + +# The pull account: unprivileged, no sudo, exists only so small-backups-box can +# read the dump directory. Trust points one way — the box can read backups, and +# can do nothing else on this host. +- name: "Ensure the {{ backup_source_pull_user }} account exists" + ansible.builtin.user: + name: "{{ backup_source_pull_user }}" + system: yes + shell: /bin/sh # rsync-over-ssh needs a shell; nologin breaks it + home: "/var/lib/{{ backup_source_pull_user }}" + create_home: yes + password: '!' # no password login, ever + when: backup_source_pull_user | length > 0 + +- name: "Authorise the backup box's key for {{ backup_source_pull_user }}" + ansible.posix.authorized_key: + user: "{{ backup_source_pull_user }}" + key: "{{ backup_source_pull_key }}" + key_options: "restrict" # no pty, no forwarding, no user rc + exclusive: yes + state: present + when: backup_source_pull_user | length > 0 + +# The shared container above the per-service directories. It must be traversable +# or the pull account cannot reach its own directory. The script's `mkdir -p` +# runs under `umask 077` and would otherwise create this 0700. +- name: "Ensure {{ backup_source_dir | dirname }} is traversable" + ansible.builtin.file: + path: "{{ backup_source_dir | dirname }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: "Ensure {{ backup_source_dir }} exists" + ansible.builtin.file: + path: "{{ backup_source_dir }}" + state: directory + owner: root + group: "{{ backup_source_pull_user | default('root', true) }}" + mode: '0750' + +- name: "Install the {{ backup_source_name }} backup script" + ansible.builtin.template: + src: backup.sh.j2 + dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh" + owner: root + group: root + mode: '0750' + validate: "bash -n %s" + +# The .service carries the push token in an Environment= line, so it is 0600. +# The .timer holds nothing secret and stays world-readable. +- name: "Install the {{ backup_source_name }}-backup systemd units" + ansible.builtin.template: + src: "backup.{{ item.unit }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} + notify: Reload systemd for backup units + +- name: "Enable the {{ backup_source_name }}-backup timer" + ansible.builtin.systemd: + name: "{{ backup_source_name }}-backup.timer" + enabled: yes + state: started + daemon_reload: yes diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 new file mode 100644 index 0000000..7a1df21 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description={{ backup_source_description }} backup +{% if backup_source_stop_service %} +{# systemd rejects a bare name here ("Failed to add dependency ... Invalid + argument"), so normalise to a full unit name. #} +After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }} +{% endif %} + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +Environment=BACKUP_PUSH_URL={{ backup_source_push_url }} +Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }} +StandardOutput=journal +StandardError=journal +SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 new file mode 100644 index 0000000..018c313 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source) +# +# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}. +# The host holds only the age PUBLIC key, so it cannot read its own backups. +set -euo pipefail +umask 077 + +BACKUP_DIR="{{ backup_source_dir }}" +RETENTION_DAYS={{ backup_source_retention_days }} +RECIPIENT="{{ backup_source_recipient }}" +SUFFIX="{{ backup_source_artifact_suffix }}" +NAME="{{ backup_source_name }}" +{% if backup_source_stop_service or backup_source_stop_command %} +STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }} +START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }} +SERVICE="{{ backup_source_stop_service or backup_source_description }}" # label for the log only +{% endif %} + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}" + +die() { echo "FATAL: $*" >&2; exit 1; } +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } + +# --- Pre-flight --- +[[ -n "$RECIPIENT" ]] || die "no age recipient configured" +command -v age >/dev/null || die "age is not installed" + +# Mode must agree with what the role sets, or each undoes the other every run. +mkdir -p "$BACKUP_DIR" +{% if backup_source_pull_user %} +chown root:{{ backup_source_pull_user }} "$BACKUP_DIR" +chmod 750 "$BACKUP_DIR" +{% else %} +chmod 700 "$BACKUP_DIR" +{% endif %} + +# A run that died mid-dump leaves a .partial. It is not a backup, and the prune +# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them +# here or they accumulate forever. +rm -f "${BACKUP_DIR}/${NAME}_"*.partial + +# --- Reporting ------------------------------------------------------------- +# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed +# backup even though the script "finished". Both are reported as failures. +PUSH_URL="${BACKUP_PUSH_URL:-}" +PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}" + +report() { + local success="$1" message="$2" + [ -n "$PUSH_URL" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true +} + +# Reports on ANY exit path, so a dump that dies halfway still reports rather +# than going quiet. The size of the FINISHED artefact decides success, not +# merely reaching the end of the script. +# +# This is called FROM the single EXIT trap below - it must never register an +# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather +# than adding to it, so a second trap here silently discards the one that +# restarts the service, and a backup run leaves the service stopped. That is +# precisely the failure the restart trap exists to prevent. +report_outcome() { + local rc="$1" + if [ "$rc" -ne 0 ]; then + report "false" "${NAME} dump exited ${rc}" + elif [ ! -s "$ARTIFACT" ]; then + report "false" "${NAME} produced no artefact at ${ARTIFACT}" + else + report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)" + fi +} + +# --- One EXIT handler, doing both jobs ------------------------------------- +# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous +# handler rather than adding to it. Registering a second one here would +# silently discard the service restart and leave the service stopped after +# every backup - which is the exact bug the restart exists to prevent, and it +# is invisible until someone notices the service is down. +on_exit() { + local rc=$? +{% if backup_source_stop_service or backup_source_stop_command %} + log "Restarting ${SERVICE}..." + eval "$START_CMD" || true +{% endif %} + report_outcome "$rc" +} +trap on_exit EXIT + +{% if backup_source_stop_service or backup_source_stop_command %} +# --- Stop the service; the trap above guarantees it comes back ------------- +# The trap is the point: without it a failed dump leaves the service down until +# the next timer fires. Every hand-written script this replaced had that bug. +# It is armed BEFORE the stop, so even a failure during the stop restarts. +log "Stopping ${SERVICE}..." +eval "$STOP_CMD" +{% endif %} + +# --- Dump straight into age; plaintext never touches the disk --- +log "Writing ${ARTIFACT}..." +{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +{% if backup_source_pull_user %} +# Match the final ownership immediately, so even a partial left by a later +# failure is not an unreadable obstacle to the pull. +chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial" +chmod 640 "${ARTIFACT}.partial" +{% endif %} +mv "${ARTIFACT}.partial" "$ARTIFACT" +{% if backup_source_pull_user %} +# Readable by the pull account and nobody else. The contents are age-encrypted +# regardless, so this is depth rather than the actual protection. +chown root:{{ backup_source_pull_user }} "$ARTIFACT" +chmod 640 "$ARTIFACT" +{% else %} +chmod 600 "$ARTIFACT" +{% endif %} +log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))" + +# --- Prune --- +log "Pruning local artefacts older than ${RETENTION_DAYS} days..." +find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete + +log "Done." diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2 new file mode 100644 index 0000000..d9a7e6a --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description={{ backup_source_description }} backup + +[Timer] +OnCalendar={{ backup_source_on_calendar }} +# Persistent: a window missed while the host was down runs on next boot. cron on +# a laptop had no equivalent, which is how two backups went unnoticed for months. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md new file mode 100644 index 0000000..13d9f5a --- /dev/null +++ b/ansible/roles/backup_store/README.md @@ -0,0 +1,55 @@ +# `backup_store` + +Pulls already-encrypted backup artefacts from every source host onto +`small-backups-box`, on a timer, and expires them per source. + +Generalises the hand-written `pull-backups.sh` that had one hardcoded source +(`arbret`). That job's behaviour is preserved exactly: same source path, same +90 days, same destination directory. + +## This host holds no key + +Everything pulled here is ciphertext produced by `backup_source` on the source +host. The box cannot read any of it — the age identity lives only on lapy. That +is deliberate: the machine holding every backup should not also be able to open +them. + +## One failing source must not stop the others + +The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own +function, failures are counted, and the script exits non-zero at the end so +systemd marks the unit failed. A dead host costs you that one source, not the +whole run. + +This is the specific failure the whole plan exists to prevent: the laptop jobs +aborted on first error and then silently produced empty directories for nine +months. + +## Trust points one way + +The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated +account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised +with `restrict`. That account can read one directory and do nothing else — no +sudo, no pty, no forwarding. A compromised backup box cannot reach into +production. + +## Addressing: names, never IPs + +Sources are addressed by name. The job this replaced hardcoded spacey's IP; the +droplet was later rebuilt, the address was recycled to a stranger, and the +backup failed silently from 2025-12-01 while the directory listing still looked +healthy. + +Two kinds of name are in play: + +- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a + headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without + it the box cannot even resolve the peer, let alone reach it. +- **spacey** is *not* a tailnet member — it is the headscale control server — so + its backup is pulled over the public internet via `headscale.contrapeso.xyz`, + which follows the host if the droplet is rebuilt. + +## Retention here is the long tail + +Sources keep a few days locally; this box keeps 90 (or whatever the source entry +says). Losing the source's local copy is expected. diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml new file mode 100644 index 0000000..e6d17a3 --- /dev/null +++ b/ansible/roles/backup_store/defaults/main.yml @@ -0,0 +1,29 @@ +--- +backup_store_dir: "{{ ansible_env.HOME }}/backups" +backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull" +backup_store_on_calendar: "*-*-* 04:00:00" + +# One entry per source. `retention_days` is the LONG tail; the source keeps its +# own short local retention. +# - name: headscale +# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" +# retention_days: 90 +backup_store_sources: [] + +# ── Reporting ──────────────────────────────────────────────────────────────── +# check-backups.sh reports one result PER SOURCE plus one for the store itself, +# so the base URL is the endpoints collection and the script appends each key. +# Empty is valid: the script still prints its report and exits 0/1. +backup_store_check_push_base: "" +backup_store_check_push_token: "" + +# Every six hours, offset past the 04:00 pull so the first run of the day sees a +# finished pull. The BACKUPS are daily, but this check is not - it reads the +# source's dump timestamp out of the artefact filename, so running it more often +# catches "the source stopped dumping" within hours rather than a day, and lets +# the Gatus heartbeat be 7h instead of 30h. +backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00" + +# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the +# pull is at 04:00, so 26h tolerates exactly one missed night before alarming. +backup_store_check_max_age_hours: 26 diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml new file mode 100644 index 0000000..632be35 --- /dev/null +++ b/ansible/roles/backup_store/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload systemd for pull-backups + ansible.builtin.systemd: + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml new file mode 100644 index 0000000..55500c6 --- /dev/null +++ b/ansible/roles/backup_store/tasks/main.yml @@ -0,0 +1,89 @@ +--- +- name: Assert backup_store sources are sane + ansible.builtin.assert: + that: + - backup_store_sources | length > 0 + - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length + fail_msg: "backup_store: every source needs name, source and retention_days" + quiet: true + +- name: Ensure rsync is installed + ansible.builtin.apt: + name: rsync + state: present + update_cache: yes + cache_valid_time: 3600 + become: yes + +- name: Ensure the backup store directory exists + ansible.builtin.file: + path: "{{ backup_store_dir }}" + state: directory + mode: '0700' + +- name: Install the pull-backups script + ansible.builtin.template: + src: pull-backups.sh.j2 + dest: /usr/local/bin/pull-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + +# An assertion that last night actually worked. Generated from the same source +# list as the puller, so it can never drift out of sync with what is supposed to +# be arriving. Runs on a timer AND is useful by hand. +- name: Install the backup check script + ansible.builtin.template: + src: check-backups.sh.j2 + dest: /usr/local/bin/check-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + +# The .service carries the push token, so it is 0600; the .timer is not secret. +- name: Install the check-backups systemd units + ansible.builtin.template: + src: "check-backups.{{ item.unit }}.j2" + dest: "/etc/systemd/system/check-backups.{{ item.unit }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {unit: service, mode: "0600"} + - {unit: timer, mode: "0644"} + become: yes + +# restarted, not started: `started` is a no-op on an already-active timer, so a +# changed schedule would never be picked up. +- name: Enable the check-backups timer + ansible.builtin.systemd: + name: check-backups.timer + enabled: yes + state: restarted + daemon_reload: yes + become: yes + +- name: Install the pull-backups systemd units + ansible.builtin.template: + src: "pull-backups.{{ item }}.j2" + dest: "/etc/systemd/system/pull-backups.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + become: yes + notify: Reload systemd for pull-backups + +- name: Enable the pull-backups timer + ansible.builtin.systemd: + name: pull-backups.timer + enabled: yes + state: started + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/templates/check-backups.service.j2 b/ansible/roles/backup_store/templates/check-backups.service.j2 new file mode 100644 index 0000000..3e50b6a --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Verify the nightly backup pull actually worked +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }} +Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }} +Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 new file mode 100644 index 0000000..37c9916 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.sh.j2 @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Assert the nightly backups actually worked. +# +# Run as {{ ansible_user_id }} on this host. Needs no sudo. +# +# What it CANNOT do: verify contents. The age identity lives only on lapy, so +# this host cannot decrypt anything it holds — by design. These are freshness, +# completeness and integrity checks. To verify content, decrypt on lapy: +# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups//" \ +# | age -d -i ~/.age/counterweight_age | tar -tzf - | head +# +# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL. +set -uo pipefail + +STORE="{{ backup_store_dir }}" +MAX_AGE_H="${1:-26}" # an artefact older than this is stale +NOW=$(date +%s) +fails=0; warns=0 + +# Colour only when attached to a terminal: this gets piped into files and, later, +# probably into a notification. +if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' +else R=''; Y=''; G=''; N=''; fi + +# Per-source verdicts, so each source can be reported independently. A single +# aggregate red light tells you backups are broken; it does not tell you which +# one, which is the thing you need at 3am. +declare -A SRC_FAIL SRC_MSG +CURRENT="" + +red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); + [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; } +yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } +ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; + [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; } + +# --- Reporting ------------------------------------------------------------- +# Each source gets its own Gatus external endpoint, plus one for the store +# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means +# report nowhere, which is valid: the exit code is still the whole answer. +PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}" +PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}" + +report() { + local key="$1" success="$2" message="$3" + [ -n "$PUSH_BASE" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true +} + +hours_since() { echo $(( (NOW - $1) / 3600 )); } + +# Pull the dump timestamp out of _YYYYmmdd_HHMMSS.. This is when +# the SOURCE produced it, which is the thing that actually matters: a source +# whose timer died still pulls "ok" forever, because yesterday's artefact is +# still sitting there. Checking only the pull would miss exactly that. +dump_epoch() { + local base ts + base=$(basename "$1") + ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1 + [ -n "$ts" ] || return 1 + date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null +} + +check_source() { + local name="$1" keep="$2" dir="$STORE/$1" + printf '\n%s\n' "== $name" + CURRENT="$name" + SRC_FAIL[$name]=0 + SRC_MSG[$name]="" + + [ -d "$dir" ] || { red "$name: no directory $dir"; return; } + + local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l) + [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; } + + local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l) + [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these" + + local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1) + local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p) + + # 1. Did the SOURCE dump recently? + local de; de=$(dump_epoch "$newest") + if [ -z "${de:-}" ]; then + yell "$name: cannot parse a dump timestamp from $(basename "$newest")" + else + local dh; dh=$(hours_since "$de") + if [ "$dh" -lt 0 ]; then + # A future-dated artefact would otherwise stay "fresh" forever and the + # staleness check would never fire again — the exact silent failure this + # script exists to catch. + red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?" + elif [ "$dh" -gt "$MAX_AGE_H" ]; then + red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run" + else + ok "$name: dumped ${dh}h ago" + fi + fi + + # 2. Did the PULL bring it over recently? + local ph; ph=$(hours_since "$(stat -c %Y "$newest")") + if [ "$ph" -gt "$MAX_AGE_H" ]; then + red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)" + else + ok "$name: pulled ${ph}h ago" + fi + + # 3. Is it plausibly a real backup? + local sz; sz=$(stat -c %s "$newest") + if [ "$sz" -eq 0 ]; then + red "$name: newest artefact is ZERO bytes" + elif [ -n "$prev" ]; then + local psz; psz=$(stat -c %s "$prev") + if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then + # Not automatically wrong: headscale legitimately shrank 297K -> 20K when + # a clean stop checkpointed its write-ahead log into the database. + yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect" + else + ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)" + fi + else + ok "$name: $(numfmt --to=iec "$sz") (first artefact)" + fi + + # 4. Is retention pruning? Allow generous slack for multiple dumps per day. + if [ "$n" -gt $(( keep * 3 + 10 )) ]; then + yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working" + fi +} + +echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" +echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." + +# --- the pull job itself --- +# Reserved key, reported as backup-store_pull-job. The store's own machinery is +# a different alarm from any one source being stale, and it is a LEADING +# indicator where the per-source checks are lagging ones: those only fire once +# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer, +# a failed pull job or a filling disk are all visible here immediately, and they +# name the cause instead of showing six stale sources with no explanation. +CURRENT="__store" +SRC_FAIL[__store]=0 +SRC_MSG[__store]="" +printf '\n%s\n' "== pull-backups.service" +result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) +status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) +when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null) +[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})" +if [ -n "$when" ]; then + wh=$(hours_since "$(date -d "$when" +%s)") + [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?" +fi +systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ + && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \ + || red "pull-backups.timer is NOT enabled" + +# --- each source --- +{% for src in backup_store_sources %} +check_source "{{ src.name }}" {{ src.retention_days }} +CURRENT="" +# The store key is what Gatus computes from group+name: sanitize("backup-store") +# + "_" + sanitize("{{ src.name }}"). +report "backup-store_{{ src.name }}" \ + "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[{{ src.name }}]:-no result}" +{% endfor %} + +# --- capacity --- +CURRENT="__store" +printf '\n%s\n' "== disk" +use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') +avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') +if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" +elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" +else ok "store is ${use}% full, ${avail} free"; fi + +CURRENT="" +report "backup-store_pull-job" \ + "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \ + "${SRC_MSG[__store]:-no result}" + +printf '\n%s\n' "-----" +if [ "$fails" -gt 0 ]; then + echo "RESULT: $fails failure(s), $warns warning(s)" + echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager" + exit 1 +fi +if [ "$warns" -gt 0 ]; then + echo "RESULT: all checks passed, $warns warning(s)" +else + echo "RESULT: all checks passed" +fi +exit 0 diff --git a/ansible/roles/backup_store/templates/check-backups.timer.j2 b/ansible/roles/backup_store/templates/check-backups.timer.j2 new file mode 100644 index 0000000..ba5c5a8 --- /dev/null +++ b/ansible/roles/backup_store/templates/check-backups.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run the backup verification after the nightly pull +Requires=check-backups.service + +[Timer] +OnCalendar={{ backup_store_check_on_calendar }} +# Run a missed occurrence on the next boot rather than skipping the day. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2 new file mode 100644 index 0000000..c65516c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Pull encrypted backups from production + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/pull-backups.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier=pull-backups diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 new file mode 100644 index 0000000..1db7d9c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Pull encrypted backups from production — managed by Ansible (roles/backup_store) +# +# Everything here is already ciphertext: this host only moves and expires files, +# and holds no key that can read them. +set -uo pipefail # deliberately NOT -e; see the loop below + +SSH_KEY="{{ backup_store_ssh_key }}" +STORE="{{ backup_store_dir }}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } +fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); } + +failures=0 + +# One source failing must not stop the others. The whole point of this box is +# that a single dead host cannot silently take the rest of the backups with it — +# which is exactly how the laptop-based jobs failed unnoticed for nine months. +{% for src in backup_store_sources %} +# --- {{ src.name }} --- +pull_{{ src.name | replace('-', '_') }}() { + local dir="${STORE}/{{ src.name }}" + mkdir -p "$dir" + log "Pulling {{ src.name }} from {{ src.source }}..." + # --exclude '*.partial': a dump that died mid-write leaves one behind, owned + # root:root 0600 because the chown only happens after a successful mv. Without + # this exclude the pull account cannot read it and rsync fails for the WHOLE + # source — so one failed dump would silently block every subsequent pull of + # that service. An incomplete artefact is never worth transferring anyway. + if rsync -az --timeout=120 --exclude '*.partial' \ + -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ + "{{ src.source }}" "$dir/"; then + log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" + else + fail "{{ src.name }}: rsync failed" + return 1 + fi + log " {{ src.name }}: pruning older than {{ src.retention_days }} days" + find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete +} +pull_{{ src.name | replace('-', '_') }} || true + +{% endfor %} +if [ "$failures" -gt 0 ]; then + log "FAILED: $failures source(s) did not pull" + exit 1 +fi +log "All sources pulled." diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2 new file mode 100644 index 0000000..336e1ca --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Daily offsite backup pull + +[Timer] +OnCalendar={{ backup_store_on_calendar }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/bitcoin_knots/README.md b/ansible/roles/bitcoin_knots/README.md new file mode 100644 index 0000000..8e35900 --- /dev/null +++ b/ansible/roles/bitcoin_knots/README.md @@ -0,0 +1,85 @@ +# `bitcoin_knots` + +Builds Bitcoin Knots from source with PGP + SHA256 verification of the release +tarball, runs it as a full node on `knots-box`, and keeps a health check on a +systemd timer. The second play in the calling playbook publishes the P2P port +from the edge host via `socket_proxy`. + +Converted from `deploy_bitcoin_knots_playbook.yml` (892 lines) under Plan 6. The +playbook is now 40 lines. + +## The build is guarded; the chain is never touched + +`build.yml` is 32 tasks, every one carrying +`when: not bitcoind_binary_exists.stat.exists`. On a host that already has the +binary the whole download / verify / 30-60 minute compile skips — **including the +two `state: absent` deletions**, which target `/opt/bitcoin-knots/source` and the +extracted build directory. + +The chain lives elsewhere and nothing here touches it: + +| | | +|---|---| +| `bitcoin_knots_dir` | `/opt/bitcoin-knots` — build tree, safe to delete | +| `bitcoin_data_dir` | `/var/lib/bitcoin` — config, logs, wallets | +| `bitcoin_large_data_dir` | `/mnt/knots_data` — **~875 GB of blockchain** | + +The signature-verification tasks are the security control of this role. They are +copied verbatim; do not "simplify" them. + +## ⚠ This node is half of the mining setup + +`bitcoin.conf` carries a DATUM Gateway section that was hand-added on the node +and was **missing from the playbook's template**: + +```ini +blockmaxsize=3985000 +blockmaxweight=3985000 +blocknotify=killall -USR1 datum_gateway +maxmempool=1000 +blockreconstructionextratxn=1000000 +``` + +`blocknotify` is how `datum_gateway` learns a new block landed. Running the old +playbook would have deleted all of it, and solo mining would have carried on +grinding against a stale template — a silent failure that costs money rather +than raising an error. The template now carries it behind +`bitcoin_datum_gateway_enabled`. + +**bitcoin-knots and datum-gateway are one system, not two services.** Changing +either config means thinking about both. + +## The restart handler, and why exactness matters now + +The hand-written `Restart bitcoind` handler carried +`when: uptime_kuma_enabled | default(false)`, so it had been inert since the +decommissioning: `bitcoin.conf` and the systemd unit both notify it and neither +could restart anything. A config change applied to disk, reported success, and +never took effect. + +It is ungated here — which raises the bar for the template. **Any** residual +difference between the template and the live file, down to a trailing newline, +means the task reports `changed` and restarts a Bitcoin node on every run. It +took four rounds of `--check --diff` to reach `changed=0`: the DATUM section, an +explanatory comment that was rendering into the deployed file (now a `{# #}` +Jinja comment), a `# Pruning (optional)` comment the live file had, and one +trailing blank line. + +## `dbcache` + +Computed as 90% of RAM unless `bitcoin_dbcache_mb_override` is set. The live node +was hand-tuned to **200 MB**; the calculation produces 3528. As with fulcrum, +`set_fact` outranks role defaults, so the *calculation* honours the override — a +value pinned only in `defaults/` is silently ignored. + +## Monitoring: one variable, no product knowledge + +The check tests bitcoind's RPC and records the answer in its exit code, which +systemd keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. Set +`healthcheck_push_url` to report anywhere that accepts an HTTP ping. + +The timer had last fired **2026-08-09** while still reporting `active` and +`enabled` — the same `OnBootSec` + `OnUnitActiveSec` dead chain as fulcrum, where +nothing re-arms it if the service does not run in a given boot. The role runs the +check once after enabling, which both smoke-tests it and supplies the reference +the timer schedules from. diff --git a/ansible/roles/bitcoin_knots/defaults/main.yml b/ansible/roles/bitcoin_knots/defaults/main.yml new file mode 100644 index 0000000..1045e77 --- /dev/null +++ b/ansible/roles/bitcoin_knots/defaults/main.yml @@ -0,0 +1,78 @@ +# Bitcoin Knots Configuration Variables + +# Version - REQUIRED: Specify exact version/tag to build +# The only version string. There used to be a second, v-prefixed copy +# (bitcoin_knots_version) that nothing read - two hand-maintained copies of one +# fact, with nothing keeping them in step. +bitcoin_knots_version_short: "29.2.knots20251110" + +# Directories +bitcoin_knots_dir: /opt/bitcoin-knots +bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" +bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets +bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) +bitcoin_conf_dir: /etc/bitcoin + +# Network +bitcoin_rpc_port: 8332 +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +bitcoin_p2p_port: 8333 +bitcoin_rpc_bind: "0.0.0.0" + +# Build options +bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores +bitcoin_build_prefix: /usr/local + +# Configuration options +bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) +bitcoin_max_connections: 125 +# dbcache will be calculated as 90% of host RAM automatically in playbook + +# ZMQ Configuration +bitcoin_zmq_enabled: true +bitcoin_zmq_bind: "tcp://0.0.0.0" +bitcoin_zmq_port_rawblock: 28332 +bitcoin_zmq_port_rawtx: 28333 +bitcoin_zmq_port_hashblock: 28334 +bitcoin_zmq_port_hashtx: 28335 + +# Service user +bitcoin_user: bitcoin +bitcoin_group: bitcoin + +# --- Health check ---------------------------------------------------------- +# Checks bitcoind RPC and records the answer in its exit code, which systemd +# keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. +# +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping +# works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" + +# --- Logging ---------------------------------------------------------------- +# The live node logs to a file. Set to "" to use printtoconsole=1 (journald). +bitcoin_logfile: "{{ bitcoin_data_dir }}/debug.log" + +# --- dbcache ---------------------------------------------------------------- +# Computed as 90% of RAM unless this is set. The live node was hand-tuned to +# 200 MB; the calculation would have produced 3528. As with fulcrum, note that +# set_fact outranks role defaults, so the CALCULATION has to honour this - a +# value pinned only in defaults/ is silently ignored. +bitcoin_dbcache_mb_override: 200 + +# --- DATUM Gateway ---------------------------------------------------------- +# This node feeds block templates to datum_gateway on knots-box. These settings +# were hand-added to bitcoin.conf and were missing from the template, so a +# playbook run would have removed them and broken the mining setup. +bitcoin_datum_gateway_enabled: true +bitcoin_blockmaxsize: 3985000 +bitcoin_blockmaxweight: 3985000 +bitcoin_blocknotify: "killall -USR1 datum_gateway" +bitcoin_maxmempool: 1000 +bitcoin_blockreconstructionextratxn: 1000000 diff --git a/ansible/roles/bitcoin_knots/handlers/main.yml b/ansible/roles/bitcoin_knots/handlers/main.yml new file mode 100644 index 0000000..49ac219 --- /dev/null +++ b/ansible/roles/bitcoin_knots/handlers/main.yml @@ -0,0 +1,14 @@ +--- +# Ungated on purpose. The hand-written handler carried +# when: uptime_kuma_enabled | default(false) +# so it has been inert since the decommissioning. Two tasks notify it — +# bitcoin.conf and the systemd unit — and neither could actually restart +# bitcoind. A configuration change to a Bitcoin node therefore applied to disk, +# reported success, and silently never took effect. +# +# Restarting bitcoind re-opens the chainstate; it does not reindex. +- name: Restart bitcoind + systemd: + name: bitcoind + state: restarted + daemon_reload: yes diff --git a/ansible/roles/bitcoin_knots/tasks/build.yml b/ansible/roles/bitcoin_knots/tasks/build.yml new file mode 100644 index 0000000..5a1b900 --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/build.yml @@ -0,0 +1,222 @@ +--- +# Every task here is guarded by `when: not bitcoind_binary_exists.stat.exists`, +# so on a host that already has the binary the whole download / verify / build +# sequence skips — including the two `state: absent` deletions, which target +# /opt/bitcoin-knots/{source,bitcoin-} and never the chain data in +# /mnt/knots_data. +- name: Check if bitcoind binary already exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_binary_exists + changed_when: false + +- name: Install gnupg for signature verification + apt: + name: gnupg + state: present + when: not bitcoind_binary_exists.stat.exists + +- name: Import Luke Dashjr's Bitcoin Knots signing key + command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 + register: key_import + changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" + when: not bitcoind_binary_exists.stat.exists + failed_when: key_import.rc != 0 + +- name: Display imported key fingerprint + command: gpg --fingerprint 90C8019E36C2E964 + register: key_fingerprint + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Download SHA256SUMS file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + +- name: Download SHA256SUMS.asc signature file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + +- name: Verify PGP signature on SHA256SUMS file + command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS + register: sha256sums_verification + changed_when: false + failed_when: false # Don't fail here - check for 'Good signature' in next task + when: not bitcoind_binary_exists.stat.exists + + +- name: Display SHA256SUMS verification result + debug: + msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" + when: not bitcoind_binary_exists.stat.exists + +- name: Fail if SHA256SUMS signature verification failed + fail: + msg: "SHA256SUMS signature verification failed. Aborting build." + when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) + +- name: Remove any existing tarball to force fresh download + file: + path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + state: absent + when: not bitcoind_binary_exists.stat.exists + +- name: Download Bitcoin Knots source tarball + get_url: + url: "{{ bitcoin_source_tarball_url }}" + dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + mode: '0644' + validate_certs: yes + force: yes + when: not bitcoind_binary_exists.stat.exists + +- name: Calculate SHA256 checksum of downloaded tarball + command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + register: tarball_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Extract expected checksum from SHA256SUMS file + shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' + register: expected_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + failed_when: expected_checksum.stdout == "" + +- name: Display checksum comparison + debug: + msg: + - "Expected: {{ expected_checksum.stdout | trim }}" + - "Actual: {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists + +- name: Verify tarball checksum matches SHA256SUMS + fail: + msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] + +- name: Remove existing source directory if it exists (to force fresh extraction) + file: + path: "{{ bitcoin_knots_source_dir }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + +- name: Remove extracted directory if it exists (from previous runs) + file: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + +- name: Extract verified source tarball + unarchive: + src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + dest: "{{ bitcoin_knots_dir }}" + remote_src: yes + when: not bitcoind_binary_exists.stat.exists + +- name: Check if extracted directory exists + stat: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + register: extracted_dir_stat + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Rename extracted directory to expected name + command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" + when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists + +- name: Check if CMakeLists.txt exists + stat: + path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" + register: cmake_exists + changed_when: false + when: not bitcoind_binary_exists.stat.exists + +- name: Create CMake build directory + file: + path: "{{ bitcoin_knots_source_dir }}/build" + state: directory + mode: '0755' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + +- name: Configure Bitcoin Knots build with CMake + command: > + cmake + -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} + -DBUILD_BITCOIN_WALLET=OFF + -DCMAKE_BUILD_TYPE=Release + -DWITH_ZMQ=ON + .. + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: configure_result + changed_when: true + +- name: Verify CMake enabled ZMQ + shell: | + set -e + cd "{{ bitcoin_knots_source_dir }}/build" + cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: zmq_check + changed_when: false + +- name: Fail if CMakeLists.txt not found + fail: + msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." + when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) + +- name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) + command: cmake --build . -j{{ bitcoin_build_jobs }} + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + async: 3600 + poll: 0 + register: build_result + changed_when: true + +- name: Check build status + async_status: + jid: "{{ build_result.ansible_job_id }}" + register: build_job_result + until: build_job_result.finished + retries: 120 + delay: 60 + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined + +- name: Fail if build failed + fail: + msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) + +- name: Install Bitcoin Knots binaries + command: cmake --install . + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + changed_when: true + +- name: Verify bitcoind binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_installed + changed_when: false + +- name: Verify bitcoin-cli binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" + register: bitcoin_cli_installed + changed_when: false + +- name: Fail if binaries not found + fail: + msg: "Bitcoin Knots binaries not found after installation" + when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists diff --git a/ansible/roles/bitcoin_knots/tasks/configure.yml b/ansible/roles/bitcoin_knots/tasks/configure.yml new file mode 100644 index 0000000..52fd3fd --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/configure.yml @@ -0,0 +1,20 @@ +--- +# Ownership copied verbatim from the playbook this replaces; verified +# mechanically against `git show HEAD:` rather than retyped from memory. +- name: Create bitcoin.conf configuration file + ansible.builtin.template: + src: bitcoin.conf.j2 + dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0640' + notify: Restart bitcoind + +- name: Create systemd service file for bitcoind + ansible.builtin.template: + src: bitcoind.service.j2 + dest: /etc/systemd/system/bitcoind.service + owner: root + group: root + mode: '0644' + notify: Restart bitcoind diff --git a/ansible/roles/bitcoin_knots/tasks/healthcheck.yml b/ansible/roles/bitcoin_knots/tasks/healthcheck.yml new file mode 100644 index 0000000..c6c8db3 --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/healthcheck.yml @@ -0,0 +1,56 @@ +--- +# Everything here answers "is bitcoind healthy" and records the answer. The +# Uptime Kuma specifics that used to follow — an embedded Python script creating +# monitors over the API, a /tmp credentials file, push-URL extraction and a +# systemd Environment= rewrite — are gone. Where it reports is now one variable, +# healthcheck_push_url. See the role README. +- name: Install curl for health check script + apt: + name: curl + state: present + +- name: Create Bitcoin Knots health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create systemd service for Bitcoin Knots health check + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/bitcoin-knots-healthcheck.service + owner: root + group: root + mode: "0600" + +- name: Create systemd timer for Bitcoin Knots health check + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer + owner: root + group: root + mode: '0644' + +- name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + +- name: Enable and restart the Bitcoin Knots health check timer + systemd: + name: bitcoin-knots-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Runs the check once, which is both a smoke test and the thing that actually +# arms the timer. This timer is OnBootSec + OnUnitActiveSec with no OnCalendar: +# OnBootSec elapses once, and OnUnitActiveSec needs the SERVICE to have run this +# boot to have anything to schedule from. Restarting the timer does not supply +# that reference; running the service does. The live timer had last fired on +# 2026-08-09 while still reporting `active` and `enabled`. +- name: Run the Bitcoin Knots health check once to arm the timer + command: systemctl start bitcoin-knots-healthcheck.service + changed_when: false diff --git a/ansible/roles/bitcoin_knots/tasks/install.yml b/ansible/roles/bitcoin_knots/tasks/install.yml new file mode 100644 index 0000000..3003b6e --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/install.yml @@ -0,0 +1,104 @@ +--- +- name: Calculate dbcache as a share of system RAM + set_fact: + bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" + when: bitcoin_dbcache_mb_override | string | length == 0 + +- name: Use the explicit dbcache override + set_fact: + bitcoin_dbcache_mb: "{{ bitcoin_dbcache_mb_override }}" + when: bitcoin_dbcache_mb_override | string | length > 0 + changed_when: false + +- name: Display calculated dbcache value + debug: + msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" + + +- name: Install build dependencies + apt: + name: + - build-essential + - libtool + - autotools-dev + - automake + - pkg-config + - bsdmainutils + - python3 + - python3-pip + - libevent-dev + - libboost-system-dev + - libboost-filesystem-dev + - libboost-test-dev + - libboost-thread-dev + - libboost-chrono-dev + - libboost-program-options-dev + - libboost-dev + - libssl-dev + - libdb-dev + - libminiupnpc-dev + - libzmq3-dev + - libnatpmp-dev + - libsqlite3-dev + - git + - curl + - wget + - cmake + state: present + update_cache: yes + +- name: Create bitcoin group + group: + name: "{{ bitcoin_group }}" + system: yes + state: present + +- name: Create bitcoin user + user: + name: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ bitcoin_data_dir }}" + create_home: yes + state: present + +- name: Create bitcoin-knots directory + file: + path: "{{ bitcoin_knots_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Create bitcoin-knots source directory + file: + path: "{{ bitcoin_knots_source_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Create bitcoin data directory (for config, logs, wallets) + file: + path: "{{ bitcoin_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + +- name: Create bitcoin large data directory (for blockchain) + file: + path: "{{ bitcoin_large_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + +- name: Create bitcoin config directory + file: + path: "{{ bitcoin_conf_dir }}" + state: directory + owner: root + group: root + mode: '0755' diff --git a/ansible/roles/bitcoin_knots/tasks/main.yml b/ansible/roles/bitcoin_knots/tasks/main.yml new file mode 100644 index 0000000..ab869a7 --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/main.yml @@ -0,0 +1,8 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: build.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/bitcoin_knots/tasks/service.yml b/ansible/roles/bitcoin_knots/tasks/service.yml new file mode 100644 index 0000000..6f6149e --- /dev/null +++ b/ansible/roles/bitcoin_knots/tasks/service.yml @@ -0,0 +1,37 @@ +--- +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start bitcoind service + systemd: + name: bitcoind + enabled: yes + state: started + +- name: Wait for bitcoind RPC to be available + uri: + url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" + method: POST + body_format: json + body: + jsonrpc: "1.0" + id: "healthcheck" + method: "getblockchaininfo" + params: [] + user: "{{ bitcoin_rpc_user }}" + password: "{{ bitcoin_rpc_password }}" + status_code: 200 + timeout: 10 + register: rpc_check + until: rpc_check.status == 200 + retries: 30 + delay: 5 + ignore_errors: yes + +- name: Display RPC connection status + debug: + msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + +# ═════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 new file mode 100644 index 0000000..5277a3c --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 @@ -0,0 +1,67 @@ +# Bitcoin Knots Configuration +# Generated by Ansible + +# Data directory (blockchain storage) +datadir={{ bitcoin_large_data_dir }} + +# RPC Configuration +server=1 +rpcuser={{ bitcoin_rpc_user }} +rpcpassword={{ bitcoin_rpc_password }} +rpcbind={{ bitcoin_rpc_bind }} +rpcport={{ bitcoin_rpc_port }} +rpcallowip=0.0.0.0/0 + +# Network Configuration +listen=1 +port={{ bitcoin_p2p_port }} +maxconnections={{ bitcoin_max_connections }} + +# Performance +dbcache={{ bitcoin_dbcache_mb }} + +# Transaction Index (optional) +{% if bitcoin_enable_txindex %} +txindex=1 +{% endif %} + +{# The live node carries this comment and the template never produced it, so a + run would have silently deleted it. Harmless in itself, but matching it keeps + this task at `ok` - which means any future `changed` here is a real signal + rather than known noise. #} +# Pruning (optional) + +# Logging +logtimestamps=1 +{% if bitcoin_logfile %} +logfile={{ bitcoin_logfile }} +{% else %} +printtoconsole=1 +{% endif %} + +# ZMQ Configuration +{% if bitcoin_zmq_enabled | default(false) %} +zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} +zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} +zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} +zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} +{% endif %} + +# Security +disablewallet=1 +{% if bitcoin_datum_gateway_enabled %} + +{# These were hand-added on the node and were NOT in this template, so running + the playbook would have stripped them. blocknotify is how datum_gateway + learns a new block landed; without it solo mining keeps grinding on a stale + template - a silent failure that costs money rather than raising an error. + Kept as a Jinja comment so the explanation stays in the repo and out of the + deployed config. #} +# Specific for DATUM gateway +blockmaxsize={{ bitcoin_blockmaxsize }} +blockmaxweight={{ bitcoin_blockmaxweight }} +blocknotify={{ bitcoin_blocknotify }} +maxmempool={{ bitcoin_maxmempool }} +blockreconstructionextratxn={{ bitcoin_blockreconstructionextratxn }} + +{% endif %} diff --git a/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 b/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 new file mode 100644 index 0000000..ac1140f --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Bitcoin Knots daemon +After=network.target + +[Service] +Type=simple +User={{ bitcoin_user }} +Group={{ bitcoin_group }} +ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf +Restart=always +RestartSec=10 +TimeoutStopSec=600 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 new file mode 100644 index 0000000..9df056e --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Bitcoin Knots Health Check +After=network.target bitcoind.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..538b8dd --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 @@ -0,0 +1,69 @@ +#!/bin/bash +# Bitcoin Knots health check — managed by Ansible (roles/bitcoin_knots) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed bitcoin-knots-healthcheck.service +# Reporting anywhere else is optional and generic. +# +# + +RPC_HOST="{{ bitcoin_rpc_bind }}" +RPC_PORT={{ bitcoin_rpc_port }} +RPC_USER="{{ bitcoin_rpc_user }}" +RPC_PASSWORD="{{ bitcoin_rpc_password }}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +# Check if bitcoind RPC is responding +check_bitcoind() { + local response + response=$(curl -s --max-time 30 \ + --user "${RPC_USER}:${RPC_PASSWORD}" \ + --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ + --header 'Content-Type: application/json' \ + "http://${RPC_HOST}:${RPC_PORT}" 2>&1) + + if [ $? -eq 0 ]; then + # Check if response contains a non-null error + # Successful responses have "error": null, failures have "error": {...} + if echo "$response" | grep -q '"error":null\|"error": null'; then + return 0 + else + return 1 + fi + else + return 1 + fi +} + +report() { + local status=$1 + local msg=$2 + + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + # URL encode spaces in message + local encoded_msg="${msg// /%20}" + + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + if ! curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${encoded_msg}"; then + return 1 + fi +} + +# Main health check +if check_bitcoind; then + report "up" "OK" + exit 0 +else + report "down" "bitcoind RPC not responding" + exit 1 +fi diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..d5857ab --- /dev/null +++ b/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Bitcoin Knots Health Check Timer +Requires=bitcoind.service + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/caddy_site/README.md b/ansible/roles/caddy_site/README.md new file mode 100644 index 0000000..c8e32af --- /dev/null +++ b/ansible/roles/caddy_site/README.md @@ -0,0 +1,118 @@ +# `caddy_site` + +Writes one Caddy site file into `{{ caddy_sites_dir }}`, makes sure the main +Caddyfile imports that directory, validates the result, and reloads Caddy once. + +Replaces the four-task block that was copy-pasted into 10 playbooks. + +Runs on any host in the `[caddy]` group — `edge` (vipy), `monitoring` +(watchtower) and `vpn_control` (spacey). + +## Usage + +```yaml +- ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: forgejo # -> forgejo.conf + caddy_site_domain: "{{ forgejo_domain }}" + caddy_site_upstream: "localhost:{{ forgejo_port }}" +``` + +Use `include_role`, not a `roles:` block, so the call stays in task order next +to the tasks it depends on. Variables passed this way are scoped to the include +and do not leak into later calls — so **every call must pass everything it +needs**; nothing carries over. + +## Shapes + +Pick exactly one of `caddy_site_upstream`, `caddy_site_root`, `caddy_site_body`. + +| Want | Set | +|---|---| +| `reverse_proxy host:port` | `caddy_site_upstream` | +| static `root *` + `file_server` | `caddy_site_root` | +| anything else | `caddy_site_body` (raw, indented 4 for you) | + +`caddy_site_upstream` accepts two modifiers, which add a block to the +`reverse_proxy`: + +- `caddy_site_headers_up: {"X-Forwarded-Host": "..."}` +- `caddy_site_resolvers: "100.100.100.100"` — Tailscale MagicDNS + +and `caddy_site_basic_auth` wraps the site in a `basic_auth` block. + +## `caddy_site_basic_auth` is a LIST, not a dict + +```yaml +caddy_site_basic_auth: + - user: "{{ datum_dashboard_username }}" + hash: "{{ datum_dashboard_password_hash }}" +``` + +**Ansible does not template dictionary keys.** With `{ "{{ user }}": "hash" }` +the value is rendered and the key is not, so the literal string +`{{ datum_dashboard_username }}` lands in the config file. Found while building +this role; the `assert` refuses a mapping so it cannot happen again. + +## Secrets and `--diff` + +Rendered site files can carry credentials — `datum-gateway.conf` holds a bcrypt +hash — and `--diff` prints rendered content. The template task therefore sets +`diff: "{{ caddy_site_reveal | bool }}"`, default `false`, so `--diff` runs are +safe everywhere. Pass `-e caddy_site_reveal=true` to see what moved on a site +you know is not secret. + +## Validation + +`validate: "caddy validate --adapter caddyfile --config %s"` runs against the +rendered temp file before it is moved into place. Verified on vipy that a single +site fragment validates cleanly (rc=0, `Valid configuration`) and that a +malformed one is rejected (rc=1, with the syntax error and line number). A +failed validate leaves the live file untouched, so a broken config can no longer +reach a running Caddy. + +What it cannot catch is a conflict with the global `/etc/caddy/Caddyfile`. + +## The reload is a handler + +`Reload caddy` fires **once, at the end of the play**, however many sites +notified it. The code this replaced ran `command: systemctl reload caddy` +immediately, mid-play. If a later task in the same play needs the new config to +be live, flush first: + +```yaml +- ansible.builtin.meta: flush_handlers +``` + +## Known intentional difference + +The `resolvers` block is commented `# Use Tailscale MagicDNS to resolve the +upstream hostname` in every case. `datum-gateway` previously said `# Resolve via +Tailscale MagicDNS`. Migrating it therefore rewrites one comment line, which +Caddy ignores. Every other site renders byte-identical to what its playbook +produced. + +## Sites on the hosts that this role does NOT manage + +Four vhosts exist in `/etc/caddy/sites-enabled/` that no playbook writes. They +were made by hand. The role only ever writes the one file it is told to, so it +leaves them alone — but nothing in the repo records them, and that is why they +are listed here. Checked 2026-09-11: + +| File | Host | Serves | State | +|---|---|---|---| +| `uptime-kuma.conf` | watchtower | `localhost:3001` | **HTTP 302 — still live**, see below | +| `arbretstaging.conf` | vipy | `arbret-staging-box:80` via MagicDNS | HTTP 200 | +| `bitcoininfra.conf` | vipy | static `file_server` from `/var/www/bitcoin-services-home` | HTTP 200 | +| `scriberr.conf` | vipy | `scriberr-box:8080` via MagicDNS | HTTP 502 — upstream down | + +**`uptime-kuma.conf` must not be deleted as dead config.** Uptime Kuma was +"decommissioned" in the repo — its playbooks archived and its credentials pulled +from the vault — but the container is **still running** on watchtower +(`louislam/uptime-kuma:latest`, created 2026-02-07, `restart=unless-stopped`) +and is still reachable at its public subdomain. Only the Ansible code was +retired; the service was not. See `archive/uptime_kuma/`. + +`scriberr` returning 502 is the one that looks like genuine rot: it proxies to a +`scriberr-box` that is not answering, and `scriberr-box` is not in the inventory. diff --git a/ansible/roles/caddy_site/defaults/main.yml b/ansible/roles/caddy_site/defaults/main.yml new file mode 100644 index 0000000..0a59906 --- /dev/null +++ b/ansible/roles/caddy_site/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Required +caddy_site_name: "" # file basename -> .conf +caddy_site_domain: "" # site address line; may hold several, comma separated + +# Pick exactly one shape +caddy_site_upstream: "" # "localhost:3000" -> reverse_proxy +caddy_site_root: "" # filesystem path -> root * + file_server +caddy_site_body: "" # raw escape hatch for one-off sites; wins over both + +# reverse_proxy modifiers +caddy_site_resolvers: "" # "100.100.100.100" for Tailscale MagicDNS +caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"} +# A LIST, not a dict: Ansible does not template dict *keys*, so a Jinja +# expression for the username silently passes through as literal text. +caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}] + +# Placement. This is now the only definition of caddy_sites_dir - services_config.yml +# used to carry an identical copy, which was removed as redundant. +caddy_sites_dir: /etc/caddy/sites-enabled + +# Rendered site files can carry credentials (basic_auth hashes), so --diff is +# suppressed by default. Pass -e caddy_site_reveal=true to see what moved. +caddy_site_reveal: false diff --git a/ansible/roles/caddy_site/handlers/main.yml b/ansible/roles/caddy_site/handlers/main.yml new file mode 100644 index 0000000..4bfa7ff --- /dev/null +++ b/ansible/roles/caddy_site/handlers/main.yml @@ -0,0 +1,12 @@ +--- +# Fires once at the end of the play, however many sites notified it. +# Anything later in the same play that needs the new config live must be +# preceded by `- ansible.builtin.meta: flush_handlers`. +# become is explicit because handlers do not inherit it from the task that +# notified them. headscale's play runs become: no and elevates per task, so +# without this the reload would run unprivileged and fail. +- name: Reload caddy + become: true + ansible.builtin.systemd: + name: caddy + state: reloaded diff --git a/ansible/roles/caddy_site/tasks/main.yml b/ansible/roles/caddy_site/tasks/main.yml new file mode 100644 index 0000000..5ad3976 --- /dev/null +++ b/ansible/roles/caddy_site/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: Assert caddy_site parameters are sane + ansible.builtin.assert: + that: + - caddy_site_name | length > 0 + - caddy_site_domain | length > 0 + - (caddy_site_upstream | length > 0) or (caddy_site_root | length > 0) or (caddy_site_body | length > 0) + - caddy_site_basic_auth is not mapping + fail_msg: >- + caddy_site: '{{ caddy_site_name | default("") }}' needs a name, a domain and + one of caddy_site_upstream / caddy_site_root / caddy_site_body. + caddy_site_basic_auth must be a LIST of {user, hash} — Ansible does not template dict keys. + quiet: true + +- name: Ensure Caddy sites-enabled directory exists + ansible.builtin.file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Ensure Caddyfile imports sites-enabled + ansible.builtin.lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + create: yes + mode: '0644' + backup: yes + +# `validate` runs `caddy validate` against the rendered temp file before it is +# moved into place: verified on vipy that a single site fragment validates +# cleanly (rc=0, "Valid configuration") and that a malformed one is rejected +# (rc=1). A failed validate leaves the live file untouched. +- name: "Write Caddy site '{{ caddy_site_name }}'" + ansible.builtin.template: + src: site.conf.j2 + dest: "{{ caddy_sites_dir }}/{{ caddy_site_name }}.conf" + owner: root + group: root + mode: '0644' + validate: "caddy validate --adapter caddyfile --config %s" + diff: "{{ caddy_site_reveal | bool }}" + notify: Reload caddy diff --git a/ansible/roles/caddy_site/templates/site.conf.j2 b/ansible/roles/caddy_site/templates/site.conf.j2 new file mode 100644 index 0000000..5d34c01 --- /dev/null +++ b/ansible/roles/caddy_site/templates/site.conf.j2 @@ -0,0 +1,34 @@ +{{ caddy_site_domain }} { +{% if caddy_site_body %} +{{ caddy_site_body | trim | indent(4, first=True) }} +{% else %} +{% if caddy_site_basic_auth %} + basic_auth { +{% for cred in caddy_site_basic_auth %} + {{ cred.user }} {{ cred.hash }} +{% endfor %} + } +{% endif %} +{% if caddy_site_root %} + root * {{ caddy_site_root }} + file_server +{% endif %} +{% if caddy_site_upstream %} +{% if caddy_site_headers_up or caddy_site_resolvers %} + reverse_proxy {{ caddy_site_upstream }} { +{% for key, value in caddy_site_headers_up.items() %} + header_up {{ key }} {{ value }} +{% endfor %} +{% if caddy_site_resolvers %} + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers {{ caddy_site_resolvers }} + } +{% endif %} + } +{% else %} + reverse_proxy {{ caddy_site_upstream }} +{% endif %} +{% endif %} +{% endif %} +} diff --git a/ansible/roles/datum_gateway/README.md b/ansible/roles/datum_gateway/README.md new file mode 100644 index 0000000..cdf35e5 --- /dev/null +++ b/ansible/roles/datum_gateway/README.md @@ -0,0 +1,73 @@ +# `datum_gateway` + +Builds and runs [DATUM Gateway](https://github.com/OCEAN-xyz/datum_gateway), the +solo/pooled mining gateway, on `knots-box`. The calling playbook adds two more +plays on the edge host: the dashboard via `caddy_site`, and the public Stratum +port via `socket_proxy`. + +Converted from `deploy_datum_gateway_playbook.yml` (802 lines) under Plan 6. The +playbook is now 68 lines and keeps all three plays. + +## ⚠ This is half of a system + +The Bitcoin Knots node on the same host feeds this gateway through +`blocknotify=killall -USR1 datum_gateway` in `bitcoin.conf` — see +`roles/bitcoin_knots/README.md`, where that line was found to be missing from the +template entirely. **Changing either config means thinking about both.** + +Interrupting Stratum costs mining shares. Check before any run that restarts it: + +```bash +ss -tn state established '( sport = :23334 )' +``` + +## Two pieces of drift where the node was right + +The repo and the node had diverged on values that matter, and the deployment +would have applied the repo's: + +| | node (correct) | repo said | +|---|---|---| +| `datum_mining_address` | `bc1qvrj3g84…` | `bc1qdse9dsg…` | +| `pool_pass_workers` / `_full_users` | `false` | `true` | + +The address is the one that would have hurt: **it is where block rewards are +paid**, and unlike fulcrum and bitcoin-knots the `Restart datum-gateway` handler +here was *never* gated, so the change would have applied immediately rather than +sitting inert. Both corrected in the vault and defaults, with notes. + +Verify semantics rather than text when touching `config.json` — render it and +compare parsed JSON, because the live file is single-line and the template is +pretty-printed, so a textual diff is all noise: + +```python +json.load(open('live.json')) == json.load(open('rendered.json')) +``` + +## `config.json` holds real secrets — diff is suppressed + +The file carries `bitcoind.rpcpassword` and `api.admin_password`. `--diff` +prints rendered content, so the task sets `diff: false` by default; pass +`-e datum_reveal_config=true` to opt in. + +Note `pool_pass_workers` / `pool_pass_full_users` are **booleans**, not +passwords, despite the names — they control DATUM's pool-password passthrough. +`mining.pool_address` is a Bitcoin address and public by nature. + +## Expect `changed` on the compile every run + +`Configure cmake build` and `Compile datum_gateway` are bare `command:` tasks +with no `changed_when`, so they always report changed and always re-run. The +build is reproducible — `Install datum_gateway binary` sees identical content and +does not replace it, so the installed binary keeps its original timestamp — but +the compile itself is wasted work on every run. That is the idempotent floor, not +drift. + +## Monitoring: one variable, no product knowledge + +The check tests the gateway API and records the answer in its exit code, which +systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. Set +`healthcheck_push_url` to report anywhere accepting an HTTP ping. + +Unlike the other services here, only the health-check *timer* handler was gated +by `uptime_kuma_enabled`; the main deployment restart worked throughout. diff --git a/ansible/roles/datum_gateway/defaults/main.yml b/ansible/roles/datum_gateway/defaults/main.yml new file mode 100644 index 0000000..0c30aa7 --- /dev/null +++ b/ansible/roles/datum_gateway/defaults/main.yml @@ -0,0 +1,61 @@ +# DATUM Gateway Configuration Variables +# https://github.com/OCEAN-xyz/datum_gateway + +# Version - pin to a specific tag +datum_gateway_version: "v0.4.1beta" + +# Directories +datum_gateway_dir: /opt/datum-gateway +datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" +datum_gateway_config_dir: /etc/datum-gateway +datum_gateway_log_dir: /var/log/datum-gateway + +# Binary +datum_gateway_bin_path: /usr/local/bin/datum_gateway + +# Ports +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +datum_gateway_stratum_port: 23334 +datum_gateway_api_port: 7152 + +# Stratum settings +datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) + +# Service user +datum_gateway_user: datum +datum_gateway_group: datum + +# Build options +datum_gateway_build_jobs: 4 + +# Bitcoin node connection +# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. +# datum_bitcoin_rpc_url should include http:// and port. +datum_bitcoin_rpc_url: "http://127.0.0.1:8332" +# Note: bitcoin_rpc_user and bitcoin_rpc_password come from group_vars/all/vault.yml + +# Mining config +datum_coinbase_tag_primary: "DATUM" +datum_coinbase_tag_secondary: "BY ORDER OF BIP110" + +# Both false on the node; the vars file said true. Corrected 2026-09-13 to +# match reality, on the same basis as datum_mining_address: the running node +# is authoritative. These control DATUM's pool-password passthrough. +datum_pool_pass_workers: false +datum_pool_pass_full_users: false +datum_pooled_mining_only: true + + +# --- Health check ----------------------------------------------------------- +# Checks the DATUM Gateway API and records the answer in its exit code, which +# systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. +# +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/datum_gateway/handlers/main.yml b/ansible/roles/datum_gateway/handlers/main.yml new file mode 100644 index 0000000..b7cfe25 --- /dev/null +++ b/ansible/roles/datum_gateway/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: Restart datum-gateway + systemd: + name: datum-gateway + state: restarted + daemon_reload: yes + +# Ungated. This one carried `when: uptime_kuma_enabled | default(false)` while +# the main Restart datum-gateway handler above did not — so on this service the +# deployment restart worked and only the health-check timer restart was dead. +- name: Restart datum-gateway health check timer + systemd: + name: datum-gateway-healthcheck.timer + state: restarted + daemon_reload: yes diff --git a/ansible/roles/datum_gateway/tasks/configure.yml b/ansible/roles/datum_gateway/tasks/configure.yml new file mode 100644 index 0000000..463dec0 --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/configure.yml @@ -0,0 +1,25 @@ +--- +# Ownership copied verbatim from the playbook this replaces and verified +# mechanically against `git show HEAD:`. +- name: Write DATUM Gateway config.json + ansible.builtin.template: + src: config.json.j2 + dest: "{{ datum_gateway_config_dir }}/config.json" + owner: "{{ datum_gateway_user }}" + group: "{{ datum_gateway_group }}" + mode: '0640' + # config.json carries the bitcoind RPC password, the API admin password and + # the pool passwords. `--diff` prints rendered content, so running with --diff + # put all of them on the terminal and into any log capturing it. Suppressed by + # default; pass -e datum_reveal_config=true when you genuinely need the diff. + diff: "{{ datum_reveal_config | default(false) | bool }}" + notify: Restart datum-gateway + +- name: Create datum-gateway systemd service + ansible.builtin.template: + src: datum-gateway.service.j2 + dest: /etc/systemd/system/datum-gateway.service + owner: root + group: root + mode: '0644' + notify: Restart datum-gateway diff --git a/ansible/roles/datum_gateway/tasks/healthcheck.yml b/ansible/roles/datum_gateway/tasks/healthcheck.yml new file mode 100644 index 0000000..b5ccb90 --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/healthcheck.yml @@ -0,0 +1,50 @@ +--- +# Everything here answers "is DATUM Gateway healthy" and records the answer. The +# Uptime Kuma specifics that used to follow — an embedded Python script creating +# monitors over the API, a /tmp credentials file, a push-URL file read back and +# parsed, and a systemd Environment= rewrite — are gone. Where it reports is now +# one variable, healthcheck_push_url. +- name: Create DATUM Gateway health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/datum-gateway-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create datum-gateway health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/datum-gateway-healthcheck.service + owner: root + group: root + mode: "0600" + notify: Restart datum-gateway health check timer + +- name: Create datum-gateway health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/datum-gateway-healthcheck.timer + owner: root + group: root + mode: '0644' + notify: Restart datum-gateway health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated: enabling a timer is deployment, not monitoring. +- name: Enable and restart the datum-gateway health check timer + systemd: + name: datum-gateway-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Arms the timer and smoke-tests the check. See roles/bitcoin_knots/README.md for +# why restarting the timer alone is not enough with OnBootSec + OnUnitActiveSec. +- name: Run the DATUM Gateway health check once to arm the timer + command: systemctl start datum-gateway-healthcheck.service + changed_when: false diff --git a/ansible/roles/datum_gateway/tasks/install.yml b/ansible/roles/datum_gateway/tasks/install.yml new file mode 100644 index 0000000..61dcadb --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/install.yml @@ -0,0 +1,74 @@ +--- +- name: Install DATUM Gateway build dependencies + apt: + name: + - cmake + - build-essential + - git + - libjansson-dev + - libmicrohttpd-dev + - libsodium-dev + - libcurl4-openssl-dev + # Runtime-only (netcat for health check) + - netcat-openbsd + state: present + update_cache: yes + +# =========================================== +# System User and Directories +# =========================================== +- name: Create datum system user + user: + name: "{{ datum_gateway_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ datum_gateway_dir }}" + create_home: no + comment: "DATUM Gateway" + +- name: Create DATUM Gateway directories + file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ datum_gateway_group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + +# =========================================== +# Build from Source +# =========================================== +- name: Clone DATUM Gateway repository at {{ datum_gateway_version }} + git: + repo: https://github.com/OCEAN-xyz/datum_gateway.git + dest: "{{ datum_gateway_source_dir }}" + version: "{{ datum_gateway_version }}" + force: yes + register: git_clone + +- name: Configure cmake build + command: cmake . -DCMAKE_BUILD_TYPE=Release + args: + chdir: "{{ datum_gateway_source_dir }}" + +- name: Compile datum_gateway + command: make -j{{ datum_gateway_build_jobs }} + args: + chdir: "{{ datum_gateway_source_dir }}" + +- name: Install datum_gateway binary + copy: + src: "{{ datum_gateway_source_dir }}/datum_gateway" + dest: "{{ datum_gateway_bin_path }}" + remote_src: yes + owner: root + group: root + mode: "0755" + notify: Restart datum-gateway + +# =========================================== +# Configuration diff --git a/ansible/roles/datum_gateway/tasks/main.yml b/ansible/roles/datum_gateway/tasks/main.yml new file mode 100644 index 0000000..57d9a45 --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/datum_gateway/tasks/service.yml b/ansible/roles/datum_gateway/tasks/service.yml new file mode 100644 index 0000000..c8581fd --- /dev/null +++ b/ansible/roles/datum_gateway/tasks/service.yml @@ -0,0 +1,16 @@ +--- +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start datum-gateway + systemd: + name: datum-gateway + enabled: yes + state: started + +# =========================================== +# Health Check Script + Systemd Timer +# =========================================== +# ═════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/datum_gateway/templates/config.json.j2 b/ansible/roles/datum_gateway/templates/config.json.j2 new file mode 100644 index 0000000..e17a386 --- /dev/null +++ b/ansible/roles/datum_gateway/templates/config.json.j2 @@ -0,0 +1,35 @@ +{ + "bitcoind": { + "rpcuser": "{{ bitcoin_rpc_user }}", + "rpcpassword": "{{ bitcoin_rpc_password }}", + "rpcurl": "{{ datum_bitcoin_rpc_url }}", + "notify_fallback": true + }, + "stratum": { + "listen_port": {{ datum_gateway_stratum_port }}, + "vardiff_min": {{ datum_vardiff_min }} + }, + "mining": { + "pool_address": "{{ datum_mining_address }}", + "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", + "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" + }, + "api": { + "admin_password": "{{ datum_gateway_admin_password }}", + "listen_port": {{ datum_gateway_api_port }}, + "modify_conf": false + }, + "logger": { + "log_to_console": true, + "log_to_file": true, + "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", + "log_rotate_daily": true, + "log_level_console": 2, + "log_level_file": 1 + }, + "datum": { + "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, + "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, + "pooled_mining_only": {{ datum_pooled_mining_only | lower }} + } +} diff --git a/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 b/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 new file mode 100644 index 0000000..18f7a3c --- /dev/null +++ b/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 @@ -0,0 +1,22 @@ +[Unit] +Description=DATUM Gateway - Bitcoin Mining Gateway +Documentation=https://github.com/OCEAN-xyz/datum_gateway +After=network.target bitcoind.service +Wants=bitcoind.service + +[Service] +User={{ datum_gateway_user }} +Group={{ datum_gateway_group }} +Type=simple +ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +# Prevent config from being read by other users +ReadWritePaths={{ datum_gateway_log_dir }} +ReadOnlyPaths={{ datum_gateway_config_dir }} + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.service.j2 b/ansible/roles/datum_gateway/templates/healthcheck.service.j2 new file mode 100644 index 0000000..e21672e --- /dev/null +++ b/ansible/roles/datum_gateway/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=DATUM Gateway Health Check +After=network.target datum-gateway.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 b/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..43ec6b2 --- /dev/null +++ b/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 @@ -0,0 +1,39 @@ +#!/bin/bash +# DATUM Gateway health check — managed by Ansible (roles/datum_gateway) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed datum-gateway-healthcheck.service +# Reporting anywhere else is optional and generic. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +STRATUM_PORT={{ datum_gateway_stratum_port }} + +check_datum() { + # Service must be active and stratum port must be listening + systemctl is-active --quiet datum-gateway && \ + nc -z 127.0.0.1 "${STRATUM_PORT}" +} + +report() { + local status=$1 + local msg=$2 + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true +} + +if check_datum; then + report "up" "OK" + exit 0 +else + report "down" "DATUM Gateway not responding" + exit 1 +fi diff --git a/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 b/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..c14d1d2 --- /dev/null +++ b/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=DATUM Gateway Health Check Timer + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/forgejo_runner/README.md b/ansible/roles/forgejo_runner/README.md new file mode 100644 index 0000000..e7f761b --- /dev/null +++ b/ansible/roles/forgejo_runner/README.md @@ -0,0 +1,59 @@ +# `forgejo_runner` + +Installs and runs a Forgejo Actions runner, registers it with the Forgejo +instance, and keeps a health check on a systemd timer. + +Converted from `deploy_forgejo_runner_playbook.yml` (409 lines) under Plan 6. +The playbook is now 16 lines. + +## Phases + +`tasks/main.yml` imports five files in order: + +| | | +|---|---| +| `prerequisites.yml` | Docker must be present | +| `install.yml` | binary, system user, working directory | +| `configure.yml` | config file, registration with the instance | +| `service.yml` | systemd unit, start, assert it came up | +| `healthcheck.yml` | check script, unit, timer | + +`import_tasks`, not `include_tasks` — static imports are visible to +`--list-tasks`, which is how the conversion was verified against the playbook it +replaced. + +## Monitoring: one variable, no product knowledge + +This role contains **nothing specific to any monitoring system**. What used to +be here — an ~80-line embedded Python script creating monitors over the Uptime +Kuma API, a `/tmp` credentials file, token extraction, a systemd `Environment=` +rewrite, and 8 `when: uptime_kuma_enabled` guards — is gone. + +What remains answers the actual question, *is this service healthy*, and records +it two ways: + +- **the exit code**, which systemd keeps: `systemctl is-failed + forgejo-runner-healthcheck.service` is a complete answer with no monitoring + system involved at all; +- **a log file** at `{{ healthcheck_log_file }}`. + +To report health somewhere, set one variable: + +```yaml +healthcheck_push_url: "https://example/api/push/TOKEN" +``` + +Any endpoint accepting an HTTP ping works. Empty (the default) means check, log, +exit honestly, report nowhere — which is also the right setting for a *pull*-based +monitor like Prometheus' textfile collector, since that reads unit state instead. + +The push URL is a credential (anyone holding it can forge an "up"), so callers +pass it from the vault rather than committing it. + +## One behaviour change, deliberate + +`Assert runner is running` used to be guarded by `uptime_kuma_enabled`, so it +never ran. It is not a monitoring task — it is the deployment checking its own +work — and the deprecation banner swept it up by mistake. It is ungated here, +which means a runner that fails to start now fails the play instead of +deploying "successfully" in silence. diff --git a/ansible/roles/forgejo_runner/defaults/main.yml b/ansible/roles/forgejo_runner/defaults/main.yml new file mode 100644 index 0000000..aba6b61 --- /dev/null +++ b/ansible/roles/forgejo_runner/defaults/main.yml @@ -0,0 +1,41 @@ +--- +# Binary +forgejo_runner_version: "6.3.1" +forgejo_runner_arch: "linux-amd64" +forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" +forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" + +# Runtime +forgejo_runner_user: "runner" +forgejo_runner_dir: "/opt/forgejo-runner" +forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" +forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" + +# The Forgejo instance this runner registers with. +forgejo_instance_url: "https://forgejo.contrapeso.xyz" +# forgejo_runner_registration_token comes from the vault. + +# --- Health check ----------------------------------------------------------- +# The check answers "is this service healthy" and records the answer two ways: +# a log file, and its own exit code. The exit code is the durable artefact — +# systemd stores it, so `systemctl is-failed forgejo-runner-healthcheck.service` +# answers the question with no monitoring system involved at all. +healthcheck_interval_seconds: 60 +healthcheck_script_dir: /opt/forgejo-runner-healthcheck +healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" +healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" +healthcheck_service_name: forgejo-runner-healthcheck + +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. +# +# Empty means "check, log, exit honestly, report nowhere". Set it to any URL +# that accepts an HTTP ping and the check will report there. Nothing in this +# role is specific to a particular monitoring product: the Uptime Kuma API +# calls, monitor creation and token handling that used to live here are gone. +# +# A pull-based monitor (Prometheus node_exporter textfile, say) needs this left +# empty — it reads the systemd unit state instead. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/forgejo_runner/tasks/configure.yml b/ansible/roles/forgejo_runner/tasks/configure.yml new file mode 100644 index 0000000..ab96d52 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/configure.yml @@ -0,0 +1,43 @@ +--- +- name: Check if config already exists + stat: + path: "{{ forgejo_runner_config_path }}" + register: config_stat + +- name: Generate default config + shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not config_stat.stat.exists + +- name: Set config file ownership + file: + path: "{{ forgejo_runner_config_path }}" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not config_stat.stat.exists + +# ── 6. Register runner ───────────────────────────────────────────── +- name: Check if runner is already registered + stat: + path: "{{ forgejo_runner_dir }}/.runner" + register: runner_stat + +- name: Register runner with Forgejo instance + command: > + {{ forgejo_runner_bin_path }} register --no-interactive + --instance {{ forgejo_instance_url }} + --token {{ forgejo_runner_registration_token }} + --name forgejo-runner-box + --labels "{{ forgejo_runner_labels }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not runner_stat.stat.exists + +- name: Set runner registration file ownership + file: + path: "{{ forgejo_runner_dir }}/.runner" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not runner_stat.stat.exists + diff --git a/ansible/roles/forgejo_runner/tasks/healthcheck.yml b/ansible/roles/forgejo_runner/tasks/healthcheck.yml new file mode 100644 index 0000000..42fbaf7 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/healthcheck.yml @@ -0,0 +1,73 @@ +--- +# Everything here answers "is the service healthy" and records the answer. +# The Uptime Kuma specifics that used to surround it — an embedded Python script +# that created monitors over the API, a /tmp credentials file, token extraction, +# and a systemd Environment= rewrite — are gone. What reports where is now one +# variable, healthcheck_push_url. See the role README. +- name: Create healthcheck script directory + ansible.builtin.file: + path: "{{ healthcheck_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Create forgejo-runner healthcheck script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ healthcheck_script_path }}" + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create healthcheck systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" + owner: root + group: root + mode: "0600" + +- name: Create healthcheck systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" + owner: root + group: root + mode: '0644' + +- name: Reload systemd for healthcheck units + systemd: + daemon_reload: yes + +- name: Enable and start healthcheck timer + systemd: + name: "{{ healthcheck_service_name }}.timer" + enabled: yes + state: started + +- name: Test healthcheck script + command: "{{ healthcheck_script_path }}" + register: healthcheck_test + changed_when: false + +- name: Verify healthcheck script works + assert: + that: + - healthcheck_test.rc == 0 + fail_msg: "Healthcheck script failed to execute properly" + +- name: Display deployment summary + debug: + msg: | + Forgejo Runner deployed successfully! + + Runner Name: forgejo-runner-box + Instance: {{ forgejo_instance_url }} + Working Directory: {{ forgejo_runner_dir }} + Service: forgejo-runner.service ({{ runner_active.stdout }}) + + Healthcheck Monitor: {{ healthcheck_service_name }} + Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s + Reporting to: {{ healthcheck_push_url | default('', true) | regex_replace('/api/push/.*', '/api/push/***') | default('(nowhere - set healthcheck_push_url)', true) }} diff --git a/ansible/roles/forgejo_runner/tasks/install.yml b/ansible/roles/forgejo_runner/tasks/install.yml new file mode 100644 index 0000000..b95ab4c --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/install.yml @@ -0,0 +1,27 @@ +--- +- name: Download forgejo-runner binary + get_url: + url: "{{ forgejo_runner_url }}" + dest: "{{ forgejo_runner_bin_path }}" + mode: '0755' + +# ── 3. Create runner system user ─────────────────────────────────── +- name: Create runner system user + user: + name: "{{ forgejo_runner_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ forgejo_runner_dir }}" + create_home: no + groups: docker + append: yes + comment: 'Forgejo Runner' + +# ── 4. Create working directory ──────────────────────────────────── +- name: Create forgejo-runner working directory + file: + path: "{{ forgejo_runner_dir }}" + state: directory + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + mode: '0750' diff --git a/ansible/roles/forgejo_runner/tasks/main.yml b/ansible/roles/forgejo_runner/tasks/main.yml new file mode 100644 index 0000000..aa71001 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/main.yml @@ -0,0 +1,9 @@ +--- +# import_tasks, not include_tasks: these are unconditional phases, and static +# imports are visible to `--list-tasks`. That matters because the task list is +# how this refactor was verified against the playbook it replaced. +- ansible.builtin.import_tasks: prerequisites.yml +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/forgejo_runner/tasks/prerequisites.yml b/ansible/roles/forgejo_runner/tasks/prerequisites.yml new file mode 100644 index 0000000..d441709 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/prerequisites.yml @@ -0,0 +1,12 @@ +--- +- name: Check if Docker is installed + command: docker --version + register: docker_check + changed_when: false + failed_when: docker_check.rc != 0 + +- name: Fail if Docker is not available + assert: + that: + - docker_check.rc == 0 + fail_msg: "Docker is required for forgejo-runner but is not installed" diff --git a/ansible/roles/forgejo_runner/tasks/service.yml b/ansible/roles/forgejo_runner/tasks/service.yml new file mode 100644 index 0000000..f971e52 --- /dev/null +++ b/ansible/roles/forgejo_runner/tasks/service.yml @@ -0,0 +1,33 @@ +--- +- name: Create forgejo-runner systemd service + ansible.builtin.template: + src: forgejo-runner.service.j2 + dest: /etc/systemd/system/forgejo-runner.service + owner: root + group: root + mode: '0644' + +- name: Reload systemd + systemd: + daemon_reload: yes + +- name: Enable and start forgejo-runner service + systemd: + name: forgejo-runner + enabled: yes + state: started + +- name: Verify forgejo-runner is active + command: systemctl is-active forgejo-runner + register: runner_active + changed_when: false + +# Ungated on purpose. This was previously guarded by `uptime_kuma_enabled`, but +# it is not a monitoring task — it is the deployment asserting its own success. +# The deprecation banner swept it up along with the Kuma plumbing, which meant a +# broken runner deployed "successfully" and silently. +- name: Assert runner is running + assert: + that: + - runner_active.stdout == "active" + fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" diff --git a/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 b/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 new file mode 100644 index 0000000..d3db25d --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Forgejo Runner +Documentation=https://forgejo.org/docs/latest/admin/actions/ +After=docker.service +Requires=docker.service + +[Service] +Type=simple +User={{ forgejo_runner_user }} +Group={{ forgejo_runner_user }} +WorkingDirectory={{ forgejo_runner_dir }} +ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 new file mode 100644 index 0000000..173f42b --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Forgejo Runner Healthcheck +After=network.target + +[Service] +Type=oneshot +ExecStart={{ healthcheck_script_path }} +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +User=root +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..9aa5ac1 --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 @@ -0,0 +1,47 @@ +#!/bin/bash +# Forgejo Runner healthcheck — managed by Ansible (roles/forgejo_runner) +# +# Answers "is forgejo-runner healthy" and records it two ways: this log, and the +# exit code. The exit code is the durable artefact — systemd keeps it, so +# systemctl is-failed {{ healthcheck_service_name }}.service +# answers the question with no monitoring system involved. +# +# Reporting is optional and generic: if a push URL is configured it also pings +# it. Nothing here knows or cares which monitoring product is on the other end. + +LOG_FILE="{{ healthcheck_log_file }}" +# Read from the environment rather than templated in, so the unit file is the +# only place the token lives and the script is not secret. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Gatus external endpoint: a POST with a bearer token and success=true|false. +# +# This used to report ONLY success - it exited before pushing when the runner +# was down - so a failure was invisible until the heartbeat window expired. +# Reporting the failure is the whole point of having a check. +report() { + local ok="$1" msg="$2" + [ -n "$PUSH_URL" ] || return 0 + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${ok}&error=${msg// /%20}" || true +} + +main() { + if ! systemctl is-active --quiet forgejo-runner; then + log_message "ERROR: forgejo-runner is not active" + report false "forgejo-runner is not active" + exit 1 + fi + + log_message "forgejo-runner is active" + report true "active" + exit 0 +} + +main diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..4cba51a --- /dev/null +++ b/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run Forgejo Runner Healthcheck every minute +Requires={{ healthcheck_service_name }}.service + +[Timer] +OnBootSec=30sec +OnUnitActiveSec={{ healthcheck_interval_seconds }}sec +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/fulcrum/README.md b/ansible/roles/fulcrum/README.md new file mode 100644 index 0000000..ddefabb --- /dev/null +++ b/ansible/roles/fulcrum/README.md @@ -0,0 +1,65 @@ +# `fulcrum` + +Deploys [Fulcrum](https://github.com/cculianu/Fulcrum), an Electrum server +indexing the Bitcoin Knots node, on `fulcrum-box`. The second play in the +calling playbook publishes its SSL port from the edge host via `socket_proxy`. + +Converted from `deploy_fulcrum_playbook.yml` (685 lines) under Plan 6. The +playbook is now 33 lines. + +## The index is the expensive thing + +`{{ fulcrum_db_dir }}` is ~192 GB and takes days to rebuild. Nothing in this role +touches it beyond `state: directory` with the ownership it already has +(`fulcrum:fulcrum 0755`). Restarting Fulcrum re-opens the database; it does not +reindex. + +## Three things this conversion fixed, all pre-existing + +**`bitcoind` pointed at the wrong machine.** The vars file carried +`bitcoin_rpc_host: "192.168.1.140"` commented "IP of knots_box_local", but `.140` +is **fulcrum-box itself** — knots-box is `.135`. The DHCP leases had reshuffled. +The live config had already been hand-corrected to `knots-box`; running the +playbook would have reverted it and broken indexing. Now addressed by Tailscale +name, like everything else in this repo. + +**The restart handler was inert.** It carried +`when: uptime_kuma_enabled | default(false)`, so the three tasks that notify it +(SSL certificate, `fulcrum.conf`, systemd unit) could not restart anything. A +configuration change would write to disk, report success, and silently never take +effect. Ungated. + +**`db_mem` was about to quadruple.** The role computes a share of RAM; on this +5931 MB host 75% is 4448 MB, leaving ~1.4 GB for the OS and Fulcrum's non-cache +memory. The live value had been hand-tuned to 2048. `fulcrum_db_mem_mb_override` +pins it. Note `set_fact` outranks role defaults, so the *calculation* has to +honour the override — pinning it in `defaults/` alone is silently ignored. + +## The health check timer, and how to read it + +The timer is `OnBootSec` + `OnUnitActiveSec` with no `OnCalendar`. That +combination has a failure mode worth knowing: `OnBootSec` is monotonic and +elapses once; `OnUnitActiveSec` schedules relative to the **service** last being +active. If the service does not run in a given boot, there is no reference to +schedule from and the timer sits `active` and `enabled` doing nothing. That is +exactly what had happened here — last trigger **2026-02-17**, seven months of no +health check, with every surface-level indicator green. + +Restarting the timer does not supply that reference; running the service does. +So the role runs the check once after enabling the timer, which is both the fix +and a smoke test. + +**Diagnosing this is easy to get wrong**: `NextElapseUSecRealtime` is always +empty for a monotonic timer, so it looks broken even when it is fine. Read +`NextElapseUSecMonotonic`, or just use `systemctl list-timers`. + +The timer also no longer carries `Requires=fulcrum.service`. On a timer that +means "stop watching when the watched thing stops", which is backwards for a +health check. + +## Monitoring: one variable, no product knowledge + +The check tests the Electrum TCP port and records the answer in its exit code, +which systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. To report +elsewhere set `healthcheck_push_url` to any endpoint accepting an HTTP ping. The +Uptime Kuma API calls, monitor creation and token handling are gone. diff --git a/ansible/roles/fulcrum/defaults/main.yml b/ansible/roles/fulcrum/defaults/main.yml new file mode 100644 index 0000000..3e97fa0 --- /dev/null +++ b/ansible/roles/fulcrum/defaults/main.yml @@ -0,0 +1,81 @@ +# Fulcrum Configuration Variables + +# Version - Pinned to specific release +fulcrum_version: "2.1.0" # Fulcrum version to install + +# Directories +fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) +fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) +fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk +fulcrum_binary_path: /usr/local/bin/Fulcrum + +# Network - Bitcoin RPC connection +# Bitcoin Knots is on a different host (knots_box_local) +# Using RPC user/password authentication (credentials from group_vars/all/vault.yml) +# Addressed by Tailscale name, never a LAN IP. This was +# bitcoin_rpc_host: "192.168.1.140" # IP of knots_box_local +# but .140 is fulcrum-box ITSELF - knots-box is .135. The DHCP leases had +# reshuffled (the same drift that transposed the inventory), so running this +# playbook would have pointed Fulcrum at itself and broken indexing. The live +# config had already been hand-corrected to knots-box; this makes the repo +# agree with it. +bitcoin_rpc_host: "knots-box" +bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml + +# Network - Fulcrum server +fulcrum_tcp_port: 50001 +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/fulcrum_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +fulcrum_ssl_port: 50002 +# Binding address for Fulcrum TCP/SSL server: +# - "127.0.0.1" = localhost only (use when Caddy is on the same box) +# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) +# - Specific IP = bind to specific network interface +fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) +fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port +# If Caddy is on a different box, set this to the IP address that Caddy will use to connect + +# SSL/TLS Configuration +fulcrum_ssl_enabled: true +fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" +fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" +fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert + + +# Performance +# db_mem will be calculated as 75% of available RAM automatically in playbook +# db_mem is computed as this share of RAM unless fulcrum_db_mem_mb is set +# explicitly. On a 5931 MB host 75% is 4448 MB, which leaves ~1.4 GB for the +# OS and for Fulcrum's non-cache memory; the live config had been hand-tuned +# down to 2048 and that setting is respected below. +fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache + +# Configuration options +fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs +fulcrum_peering: false # Disable peering with other Fulcrum servers +fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications + +# Service user +fulcrum_user: fulcrum +fulcrum_group: fulcrum + + +# --- Health check ----------------------------------------------------------- +# Checks the Electrum TCP port and records the answer in its exit code, which +# systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. +# +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping +# works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" + +# Explicit db_mem in MB. When set it wins over fulcrum_db_mem_percent; empty +# means compute from RAM. Set here because the live host had been hand-tuned to +# 2048 and a silent jump to 4448 is not something a refactor should do. +fulcrum_db_mem_mb_override: 2048 diff --git a/ansible/roles/fulcrum/handlers/main.yml b/ansible/roles/fulcrum/handlers/main.yml new file mode 100644 index 0000000..c0d9cc4 --- /dev/null +++ b/ansible/roles/fulcrum/handlers/main.yml @@ -0,0 +1,15 @@ +--- +# Ungated on purpose. The hand-written handler carried +# when: uptime_kuma_enabled | default(false) +# so it has been inert since the decommissioning: three tasks notify it (the SSL +# certificate, fulcrum.conf and the systemd unit), and none of them could +# actually restart Fulcrum. A configuration change therefore applied to disk and +# silently never took effect — the worst kind of quiet failure, because the +# playbook reports success and the running service keeps its old settings. +# +# Restarting Fulcrum re-opens its database; it does not reindex. +- name: Restart fulcrum + systemd: + name: fulcrum + state: restarted + daemon_reload: yes diff --git a/ansible/roles/fulcrum/tasks/healthcheck.yml b/ansible/roles/fulcrum/tasks/healthcheck.yml new file mode 100644 index 0000000..983bffb --- /dev/null +++ b/ansible/roles/fulcrum/tasks/healthcheck.yml @@ -0,0 +1,62 @@ +--- +# Everything here answers "is Fulcrum healthy" and records the answer. The +# Uptime Kuma specifics that used to follow — an embedded Python script creating +# monitors over the API, a /tmp credentials file, push-URL extraction and a +# systemd Environment= rewrite — are gone. Where it reports is now one variable, +# healthcheck_push_url. See the role README. +- name: Create Fulcrum health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/fulcrum-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +- name: Create systemd service for Fulcrum health check + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/fulcrum-healthcheck.service + owner: root + group: root + mode: "0600" + +- name: Create systemd timer for Fulcrum health check + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/fulcrum-healthcheck.timer + owner: root + group: root + mode: '0644' + +- name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + +# state: restarted, not started. The hand-written timer had got itself stuck +# `active` with no next elapse and had not fired since 2026-02-17; `started` on +# an already-active timer is a no-op and would have left it stuck. Restarting +# re-arms it. See the note in healthcheck.timer.j2. +- name: Enable and restart the Fulcrum health check timer + systemd: + name: fulcrum-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Run the check once, which is both a smoke test and the thing that actually +# arms the timer. +# +# This timer is OnBootSec + OnUnitActiveSec with no OnCalendar. OnBootSec is +# monotonic and had long since elapsed; OnUnitActiveSec schedules relative to the +# SERVICE last being active, and the service had not run since 2026-02-17 — so +# there was no reference to schedule from and the timer sat `active` and +# `enabled` with NextElapseUSecMonotonic=infinity. Restarting the timer alone +# does not supply that reference; running the service does. +# +# (Diagnosing this is easy to get wrong: NextElapseUSecRealtime is always empty +# for a monotonic timer, so it looks broken even when it is fine. Read +# NextElapseUSecMonotonic, or just use `systemctl list-timers`.) +- name: Run the Fulcrum health check once to arm the timer + command: systemctl start fulcrum-healthcheck.service + changed_when: false diff --git a/ansible/roles/fulcrum/tasks/install.yml b/ansible/roles/fulcrum/tasks/install.yml new file mode 100644 index 0000000..1216a96 --- /dev/null +++ b/ansible/roles/fulcrum/tasks/install.yml @@ -0,0 +1,146 @@ +--- +- name: Calculate db_mem as a share of system RAM + set_fact: + fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" + when: fulcrum_db_mem_mb_override | string | length == 0 + +- name: Use the explicit db_mem override + set_fact: + fulcrum_db_mem_mb: "{{ fulcrum_db_mem_mb_override }}" + when: fulcrum_db_mem_mb_override | string | length > 0 + changed_when: false + +- name: Display calculated db_mem value + debug: + msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" + +- name: Display Fulcrum version to install + debug: + msg: "Installing Fulcrum version {{ fulcrum_version }}" + +- name: Install required packages + apt: + name: + - curl + - wget + - openssl + state: present + update_cache: yes + +- name: Create fulcrum group + group: + name: "{{ fulcrum_group }}" + system: yes + state: present + +- name: Create fulcrum user + user: + name: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + system: yes + shell: /usr/sbin/nologin + home: /home/{{ fulcrum_user }} + create_home: yes + state: present + +- name: Create Fulcrum database directory (heavy data on special mount) + file: + path: "{{ fulcrum_db_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + +- name: Create Fulcrum config directory + file: + path: "{{ fulcrum_config_dir }}" + state: directory + owner: root + group: "{{ fulcrum_group }}" + mode: '0755' + +- name: Create Fulcrum lib directory (for banner and other data files) + file: + path: "{{ fulcrum_lib_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + +# =========================================== +# SSL Certificate Generation +# =========================================== +- name: Check if SSL certificate already exists + stat: + path: "{{ fulcrum_ssl_cert_path }}" + register: fulcrum_ssl_cert_exists + when: fulcrum_ssl_enabled | default(false) + +- name: Generate self-signed SSL certificate for Fulcrum + command: > + openssl req -x509 -newkey rsa:4096 + -keyout {{ fulcrum_ssl_key_path }} + -out {{ fulcrum_ssl_cert_path }} + -sha256 -days {{ fulcrum_ssl_cert_days }} + -nodes + -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" + args: + creates: "{{ fulcrum_ssl_cert_path }}" + when: fulcrum_ssl_enabled | default(false) + notify: Restart fulcrum + +- name: Set SSL certificate permissions + file: + path: "{{ fulcrum_ssl_cert_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) + +- name: Set SSL key permissions + file: + path: "{{ fulcrum_ssl_key_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0600' + when: fulcrum_ssl_enabled | default(false) + +- name: Check if Fulcrum binary already exists + stat: + path: "{{ fulcrum_binary_path }}" + register: fulcrum_binary_exists + changed_when: false + +- name: Download Fulcrum binary tarball + get_url: + url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + mode: '0644' + when: not fulcrum_binary_exists.stat.exists + +- name: Extract Fulcrum binary + unarchive: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp" + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + +- name: Install Fulcrum binary + copy: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" + dest: "{{ fulcrum_binary_path }}" + owner: root + group: root + mode: '0755' + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + +- name: Verify Fulcrum binary installation + command: "{{ fulcrum_binary_path }} --version" + register: fulcrum_version_check + changed_when: false + +- name: Display Fulcrum version + debug: + msg: "{{ fulcrum_version_check.stdout_lines }}" + diff --git a/ansible/roles/fulcrum/tasks/main.yml b/ansible/roles/fulcrum/tasks/main.yml new file mode 100644 index 0000000..bc7ff05 --- /dev/null +++ b/ansible/roles/fulcrum/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/fulcrum/tasks/service.yml b/ansible/roles/fulcrum/tasks/service.yml new file mode 100644 index 0000000..716b44d --- /dev/null +++ b/ansible/roles/fulcrum/tasks/service.yml @@ -0,0 +1,54 @@ +--- +- name: Create Fulcrum banner file + ansible.builtin.template: + src: banner.txt.j2 + dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + +- name: Create Fulcrum configuration file + ansible.builtin.template: + src: fulcrum.conf.j2 + dest: "{{ fulcrum_config_dir }}/fulcrum.conf" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0640' + notify: Restart fulcrum + +- name: Create systemd service file for Fulcrum + ansible.builtin.template: + src: fulcrum.service.j2 + dest: /etc/systemd/system/fulcrum.service + owner: root + group: root + mode: '0644' + notify: Restart fulcrum + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start Fulcrum service + systemd: + name: fulcrum + enabled: yes + state: started + +- name: Wait for Fulcrum to start + wait_for: + port: "{{ fulcrum_tcp_port }}" + host: "{{ fulcrum_tcp_bind }}" + delay: 5 + timeout: 30 + ignore_errors: yes + +- name: Check Fulcrum service status + systemd: + name: fulcrum + register: fulcrum_service_status + changed_when: false + +- name: Display Fulcrum service status + debug: + msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" diff --git a/ansible/roles/fulcrum/templates/banner.txt.j2 b/ansible/roles/fulcrum/templates/banner.txt.j2 new file mode 100644 index 0000000..1ce4ced --- /dev/null +++ b/ansible/roles/fulcrum/templates/banner.txt.j2 @@ -0,0 +1,3 @@ +counterinfra + +PER ASPERA AD ASTRA diff --git a/ansible/roles/fulcrum/templates/fulcrum.conf.j2 b/ansible/roles/fulcrum/templates/fulcrum.conf.j2 new file mode 100644 index 0000000..69f072c --- /dev/null +++ b/ansible/roles/fulcrum/templates/fulcrum.conf.j2 @@ -0,0 +1,29 @@ +# Fulcrum Configuration +# Generated by Ansible + +# Bitcoin Core/Knots RPC settings +bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} +rpcuser = {{ bitcoin_rpc_user }} +rpcpassword = {{ bitcoin_rpc_password }} + +# Fulcrum server general settings +datadir = {{ fulcrum_db_dir }} +tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} +peering = {{ 'true' if fulcrum_peering else 'false' }} +zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} + +# SSL/TLS Configuration +{% if fulcrum_ssl_enabled | default(false) %} +ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} +cert = {{ fulcrum_ssl_cert_path }} +key = {{ fulcrum_ssl_key_path }} +{% endif %} + +# Anonymize client IP addresses and TxIDs in logs +anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} + +# Max RocksDB Memory in MiB +db_mem = {{ fulcrum_db_mem_mb }}.0 + +# Banner +banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt diff --git a/ansible/roles/fulcrum/templates/fulcrum.service.j2 b/ansible/roles/fulcrum/templates/fulcrum.service.j2 new file mode 100644 index 0000000..02f4aa5 --- /dev/null +++ b/ansible/roles/fulcrum/templates/fulcrum.service.j2 @@ -0,0 +1,24 @@ +# MiniBolt: systemd unit for Fulcrum +# /etc/systemd/system/fulcrum.service + +[Unit] +Description=Fulcrum +After=network.target + +StartLimitBurst=2 +StartLimitIntervalSec=20 + +[Service] +ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf + +User={{ fulcrum_user }} +Group={{ fulcrum_group }} + +# Process management +#################### +Type=simple +KillSignal=SIGINT +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.service.j2 b/ansible/roles/fulcrum/templates/healthcheck.service.j2 new file mode 100644 index 0000000..b519808 --- /dev/null +++ b/ansible/roles/fulcrum/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Fulcrum Health Check +After=network.target fulcrum.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.sh.j2 b/ansible/roles/fulcrum/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..4449fed --- /dev/null +++ b/ansible/roles/fulcrum/templates/healthcheck.sh.j2 @@ -0,0 +1,41 @@ +#!/bin/bash +# Fulcrum health check — managed by Ansible (roles/fulcrum) +# +# Checks that Fulcrum's Electrum TCP port is accepting connections, and records +# the answer in the exit code, which systemd keeps: +# systemctl is-failed fulcrum-healthcheck.service +# That is a complete answer with no monitoring system involved. Reporting +# elsewhere is optional and generic — set healthcheck_push_url. + +FULCRUM_HOST="{{ fulcrum_tcp_bind }}" +FULCRUM_PORT={{ fulcrum_tcp_port }} +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +check_fulcrum() { + timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null +} + +report() { + local status=$1 msg=$2 + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true +} + +if check_fulcrum; then + report "up" "OK" + exit 0 +else + echo "Fulcrum TCP port ${FULCRUM_PORT} not responding" + report "down" "Fulcrum TCP port not responding" + exit 1 +fi diff --git a/ansible/roles/fulcrum/templates/healthcheck.timer.j2 b/ansible/roles/fulcrum/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..df16ff3 --- /dev/null +++ b/ansible/roles/fulcrum/templates/healthcheck.timer.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Fulcrum Health Check Timer +# NOTE: this deliberately does NOT carry `Requires=fulcrum.service`, which the +# hand-written unit had. Requires on a timer means the timer is stopped when the +# required unit stops — i.e. "if the thing I am watching goes down, stop +# watching it", which is backwards for a health check and leaves nothing to +# re-arm the timer when the service returns. The live timer had been `active` +# and `enabled` with NextElapseUSecMonotonic=infinity and a last trigger of +# 2026-02-17: seven months with no health check and no outward sign of it. + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/gatus/README.md b/ansible/roles/gatus/README.md new file mode 100644 index 0000000..129057c --- /dev/null +++ b/ansible/roles/gatus/README.md @@ -0,0 +1,110 @@ +# gatus + +Deploys [Gatus](https://github.com/TwiN/gatus) — health checks, a status page, +and alerting — **as the upstream container image**, on the +`observability` group. + +## Why the container + +Upstream publishes **no binary release assets**. The image is the only artefact +they ship and therefore the only one they test, so it is what `docker run` in +their README gets you, and it is what this role deploys. + +Building from source is entirely possible — their Dockerfile is a bare +`CGO_ENABLED=0 go build`, the Vue dashboard is compiled in via `//go:embed +static` in `web/static.go`, and the sqlite driver is pure-Go +`modernc.org/sqlite` so nothing needs linking. This role did that at first. The +reasons it doesn't now: + +- It produces a binary upstream never ran. +- It means compiling the AWS SDK, gRPC and the Google API libraries on the + smallest box in the estate. On this VPS that load was heavy enough that + unrelated Ansible tasks timed out while it ran. + +The cost of the container is a daemon on the machine whose job is to notice +when everything else breaks. That is a real trade, made deliberately. + +## Pinned by digest, not by tag + +```yaml +gatus_image_digest: "sha256:c5f210d0…" +gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" +``` + +A tag is mutable — `v5.36.0` can be repushed — so pinning the tag alone is a +weaker promise than it looks. The digest is a content address: if it resolves, +it is byte-for-byte the image reviewed here, and `docker compose pull` either +fetches exactly that or fails. `gatus_version` is kept alongside it purely so a +human can read which release it is; **the two must be updated together.** + +## What `FROM scratch` means for running it + +The image has no `/etc/passwd`, so there is no user to drop to by name and the +default is root. The compose file runs it by numeric id (`10001:10001`) and the +data directory on the host is owned to match. Everything else is locked down to +approximate what the systemd unit used to do natively: + +| systemd | compose | +|---|---| +| `ProtectSystem=strict` | `read_only: true` | +| `NoNewPrivileges=true` | `security_opt: [no-new-privileges:true]` | +| `CapabilityBoundingSet=` | `cap_drop: [ALL]` | +| `AmbientCapabilities=CAP_NET_RAW` | `cap_add: [NET_RAW]` (for `icmp://`) | + +## Configuration is a directory, not a file + +`GATUS_CONFIG_PATH` points at `/opt/gatus/config`, and Gatus merges every `*.yaml` +underneath it — maps deep-merge, lists append. This role owns exactly one file: + +``` +/opt/gatus/config/00-base.yaml web, storage, ui, alerting, security (this role) +/opt/gatus/config/endpoints/*.yaml one file per service (gatus_endpoint) +``` + +**A primitive defined in two files is ambiguous and upstream refuses it.** So +anything that is not a list belongs in `00-base.yaml` and nowhere else. Endpoints +are lists, so each service's file appends cleanly — the same shape as +`caddy_site`, where each service contributes its own vhost. + +## Pull and push + +Gatus polls. For anything with a reachable HTTP or TCP surface that is the +better check, because it tests the path a user actually takes. The monitoring +host joins the headscale mesh via `infra/920`, so internal boxes are reachable +by MagicDNS name and can be polled directly rather than having to report in. + +For state with no pollable surface — ZFS pool health, UPS mains status, disk +usage, backup freshness — Gatus has **external endpoints**, a push API: + +``` +POST /api/v1/endpoints/{group}_{name}/external?success=true&error=&duration= +Authorization: Bearer +``` + +with `heartbeat.interval` to alert when nothing reports in. That is the same +shape as the generic `healthcheck_push_url` already wired into every service +role, so those scripts need a URL, a POST, and an auth header — not a rewrite. + +## Variables + +See `defaults/main.yml`. The ones that matter: + +| Variable | Default | Note | +|---|---|---| +| `gatus_version` / `gatus_image_digest` | `v5.36.0` / `sha256:c5f210d0…` | must move together | +| `gatus_bind_address` | `127.0.0.1` | **never bind publicly** — the push API shares this listener | +| `gatus_storage_type` | `sqlite` | `memory` loses all history on restart | +| `gatus_alerting` | `{}` | pass-through; any provider Gatus supports | +| `gatus_allow_icmp` | `true` | adds back `NET_RAW` for `icmp://` checks | + +`gatus_alerting` empty is valid and is the current state: every condition is +still evaluated and recorded, there is just nowhere to shout yet. + +## Verifying + +```bash +docker ps --filter name=gatus +docker logs gatus --tail 50 +curl -s localhost:8080/health +ls /opt/gatus/config/endpoints/ +``` diff --git a/ansible/roles/gatus/defaults/main.yml b/ansible/roles/gatus/defaults/main.yml new file mode 100644 index 0000000..6f6ec09 --- /dev/null +++ b/ansible/roles/gatus/defaults/main.yml @@ -0,0 +1,118 @@ +--- +# Gatus, deployed the way upstream distributes it: the container image. +# +# Upstream publishes NO binary release assets - the image is the only artefact +# they ship, and therefore the only artefact they test. Building from source is +# possible (`go build` alone is enough; the Vue dashboard is compiled in via +# `//go:embed static`, and CGO_ENABLED=0 works because the sqlite driver is +# pure-Go modernc.org/sqlite) but it produces a binary upstream never ran, and +# it means a full compile of the AWS SDK and gRPC on the smallest box in the +# estate. + +# ── Image ──────────────────────────────────────────────────────────────────── +gatus_version: "v5.36.0" +# Pinned by DIGEST, not by tag. A tag is mutable - `v5.36.0` can be repushed - +# so pinning the tag alone is a weaker promise than it looks. The digest is the +# content address: if it resolves, it is byte-for-byte the image reviewed here. +# Both must be updated together; the tag is kept only so humans can read it. +gatus_image_digest: "sha256:c5f210d095fa78e6efaa20ffeb14803f2ba4f10615e16a6d12087697149617f0" +gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" + +# ── Paths (host side) ──────────────────────────────────────────────────────── +gatus_dir: /opt/gatus +gatus_config_dir: "{{ gatus_dir }}/config" +gatus_data_dir: "{{ gatus_dir }}/data" + +# Gatus merges every *.yaml under GATUS_CONFIG_PATH and its subdirectories: +# maps deep-merge, lists append. That is why this role ships a config DIRECTORY +# rather than one file - each service contributes its own endpoint file, the +# same way each service contributes a vhost through `caddy_site`. +# +# Primitives must be defined exactly once across all files or the merge is +# ambiguous, so everything that is not a list lives in the base file and +# nowhere else. +gatus_base_config_file: "00-base.yaml" +gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" + +# ── Identity ───────────────────────────────────────────────────────────────── +# The image is FROM scratch, so it has no /etc/passwd and no user to drop to by +# name. Run it by numeric uid/gid instead, and own the data volume to match. +gatus_uid: 10001 +gatus_gid: 10001 + +# ── Web ────────────────────────────────────────────────────────────────────── +gatus_port: 8080 +# HOST-side address the container's port is published on. Gatus itself always +# binds 0.0.0.0 inside the container - see the note in config.yaml.j2. Never +# publish this on 0.0.0.0: the external-endpoint push API shares the dashboard's +# listener, and Caddy is what should be in front of both. +gatus_bind_address: "127.0.0.1" +# Pass-through, the same shape as gatus_alerting: whatever is set here is +# rendered verbatim under `ui:`, so every option upstream supports is reachable +# without adding a variable per key. See config/ui/ui.go for the full set - +# title, description, header, dashboard-heading/subheading, logo, link, favicon, +# buttons, custom-css, dark-mode, default-sort-by, default-filter-by. +gatus_ui: + title: "Status" + header: "Status" + # Group by default. The dashboard's own groupByGroup toggle starts OFF and + # remembers per browser in localStorage, so without this the ten groups render + # as one flat list for anyone who has not clicked it. + default-sort-by: group + +# ── Storage ────────────────────────────────────────────────────────────────── +# sqlite, not memory: history has to survive a restart, or the dashboard lies +# about uptime after every deploy. Path is INSIDE the container. +gatus_storage_type: sqlite +gatus_storage_path: "/data/gatus.db" +gatus_storage_caching: true + +# Per-endpoint row caps. Gatus bounds the database by COUNT, not by time, and +# trims inline on insert (storage/store/sql/sql.go, InsertEndpointResult) - so +# there is no retention job to write and no way for this to fill a disk. +# +# History depth is therefore a function of check frequency, not of days: +# 900 results is ~3 days of a 5-minute liveness check, and ~2.5 years of a daily +# disk check. Upstream's default is 100, which would have been 8 hours of +# liveness - not enough to still see a weekend incident on Monday. +# +# Note the uptime table is separate and its 30-day retention is hard-coded +# upstream (uptimeRetention), so uptime percentages top out at 30 days whatever +# this is set to. +gatus_storage_max_results: 900 +gatus_storage_max_events: 50 + +# ── Alerting ───────────────────────────────────────────────────────────────── +# Pass-through: rendered verbatim under `alerting:`, so any provider Gatus +# supports works without touching this role. Empty means "check and record, +# alert nowhere" - valid, and the default until a provider is chosen. +gatus_alerting: {} +gatus_default_alerts: [] + +# ── Security ───────────────────────────────────────────────────────────────── +# gatus_basic_auth: {username: admin, password-bcrypt-base64: "..."} +gatus_basic_auth: {} + +gatus_maintenance: {} + +# Keep serving the previous config if a contributed endpoint file is malformed, +# instead of panicking. See the note in config.yaml.j2. +gatus_skip_invalid_config_update: true +gatus_log_level: INFO + +# ── Self-check ─────────────────────────────────────────────────────────────── +# Gatus panics on a config with no endpoints, so the role always ships one. +# Turning this off is only safe once another file in endpoints/ provides one. +gatus_self_check: true + +# ── ICMP ───────────────────────────────────────────────────────────────────── +# Gatus supports icmp:// endpoints. Raw ICMP needs CAP_NET_RAW, which the +# container would get free only if it ran as root; it does not. Set false if +# you never use icmp:// checks and want the capability dropped entirely. +gatus_allow_icmp: true + +# ── Shared network ─────────────────────────────────────────────────────────── +# Gatus runs in a container, so the HOST's loopback is not reachable from it. +# Anything Gatus must talk to locally - the Signal API that sends its alerts - +# has to be on a shared docker network and addressed by service name. +gatus_network: monitoring diff --git a/ansible/roles/gatus/handlers/main.yml b/ansible/roles/gatus/handlers/main.yml new file mode 100644 index 0000000..25ad865 --- /dev/null +++ b/ansible/roles/gatus/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart gatus + ansible.builtin.command: + cmd: docker compose up -d --force-recreate + chdir: "{{ gatus_dir }}" diff --git a/ansible/roles/gatus/tasks/configure.yml b/ansible/roles/gatus/tasks/configure.yml new file mode 100644 index 0000000..ef5d7e7 --- /dev/null +++ b/ansible/roles/gatus/tasks/configure.yml @@ -0,0 +1,64 @@ +--- +- name: Assert Docker is available + ansible.builtin.command: docker --version + register: gatus_docker_check + changed_when: false + failed_when: gatus_docker_check.rc != 0 + +# Created explicitly rather than by either compose file, so neither the gatus +# stack nor the signal-api stack has to be deployed before the other. +- name: Ensure the shared monitoring network exists + ansible.builtin.command: "docker network create {{ gatus_network }}" + register: gatus_net + changed_when: "'already exists' not in gatus_net.stderr" + failed_when: + - gatus_net.rc != 0 + - "'already exists' not in gatus_net.stderr" + +- name: Create the gatus directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: + - {path: "{{ gatus_dir }}", owner: root, group: root, mode: "0755"} + # Config is root-owned and world-unreadable: it holds external-endpoint + # tokens. The container mounts it read-only and reads it as gatus_uid, so + # that id needs group access - hence the group ownership below. + - {path: "{{ gatus_config_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} + - {path: "{{ gatus_endpoints_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} + # Data is the one path the container writes to, so it must be owned by the + # numeric id the container runs as. `read_only: true` makes everything else + # in the container immutable. + - {path: "{{ gatus_data_dir }}", owner: "{{ gatus_uid }}", group: "{{ gatus_gid }}", mode: "0750"} + +- name: Write the base gatus configuration + ansible.builtin.template: + src: config.yaml.j2 + dest: "{{ gatus_config_dir }}/{{ gatus_base_config_file }}" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + notify: Restart gatus + +# Without this the container crash-loops on an empty config. See the template. +- name: Write the gatus self-check endpoint + ansible.builtin.template: + src: endpoint-self.yaml.j2 + dest: "{{ gatus_endpoints_dir }}/00-self.yaml" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + when: gatus_self_check | bool + notify: Restart gatus + +- name: Write the docker compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ gatus_dir }}/docker-compose.yml" + owner: root + group: root + mode: "0644" + notify: Restart gatus diff --git a/ansible/roles/gatus/tasks/main.yml b/ansible/roles/gatus/tasks/main.yml new file mode 100644 index 0000000..e1b67bf --- /dev/null +++ b/ansible/roles/gatus/tasks/main.yml @@ -0,0 +1,5 @@ +--- +# import, never include: dynamic includes are opaque to --list-tasks, which is +# the primary verification tool in this repo. +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml diff --git a/ansible/roles/gatus/tasks/service.yml b/ansible/roles/gatus/tasks/service.yml new file mode 100644 index 0000000..bfcca3b --- /dev/null +++ b/ansible/roles/gatus/tasks/service.yml @@ -0,0 +1,35 @@ +--- +# Pulling by digest means this either fetches exactly the reviewed image or +# fails. There is no "latest wins" path. +- name: Pull the pinned gatus image + ansible.builtin.command: + cmd: docker compose pull + chdir: "{{ gatus_dir }}" + register: gatus_pull + changed_when: "'Downloaded newer image' in gatus_pull.stderr or 'Pull complete' in gatus_pull.stderr" + +- name: Start gatus + ansible.builtin.command: + cmd: docker compose up -d --remove-orphans + chdir: "{{ gatus_dir }}" + register: gatus_up + changed_when: "'Started' in gatus_up.stderr or 'Created' in gatus_up.stderr or 'Recreated' in gatus_up.stderr" + +- name: Flush handlers so a config change is live before it is verified + ansible.builtin.meta: flush_handlers + +- name: Wait for gatus to answer + ansible.builtin.uri: + url: "http://{{ gatus_bind_address }}:{{ gatus_port }}/health" + status_code: [200, 404] + register: gatus_health + until: gatus_health.status in [200, 404] + retries: 12 + delay: 5 + +- name: Assert gatus is running + ansible.builtin.assert: + that: + - gatus_health.status in [200, 404] + fail_msg: "gatus did not come up on {{ gatus_bind_address }}:{{ gatus_port }} - check `docker logs gatus`" + success_msg: "gatus is answering on {{ gatus_bind_address }}:{{ gatus_port }}" diff --git a/ansible/roles/gatus/templates/config.yaml.j2 b/ansible/roles/gatus/templates/config.yaml.j2 new file mode 100644 index 0000000..faf61fb --- /dev/null +++ b/ansible/roles/gatus/templates/config.yaml.j2 @@ -0,0 +1,64 @@ +# {{ gatus_base_config_file }} — managed by Ansible (roles/gatus) +# +# BASE CONFIGURATION ONLY. +# +# Gatus merges every *.yaml under GATUS_CONFIG_PATH: maps are deep-merged and +# lists are appended, but a primitive defined in two files is ambiguous and +# upstream refuses it. So `web`, `storage`, `ui`, `alerting` and `security` are +# set here and MUST NOT appear in any other file in this directory. +# +# Endpoints are lists, so they append cleanly. Each service drops its own file +# into endpoints/ via the gatus_endpoint role - the same shape as caddy_site. + +web: + # 0.0.0.0 is the CONTAINER's interface, not the host's. This must not be + # 127.0.0.1: that is the container's own loopback, which docker-proxy cannot + # reach, and gatus comes up healthy while the published port refuses every + # connection. + # + # The isolation comes from the port mapping in docker-compose.yml, which + # publishes to {{ gatus_bind_address }} on the host. Caddy fronts that, and it + # matters because the external-endpoint push API shares this listener with the + # dashboard. + address: 0.0.0.0 + port: {{ gatus_port }} + +# Gatus reloads when its config changes. If the NEW config fails to parse it +# calls panic() - unless this is set, in which case it logs the error and keeps +# running on the old config. Endpoint files are contributed by other playbooks, +# so one malformed file would otherwise take the monitor down, which is the +# worst possible time to lose it. +skip-invalid-config-update: {{ gatus_skip_invalid_config_update | bool | lower }} + +storage: + type: {{ gatus_storage_type }} +{% if gatus_storage_type != 'memory' %} + path: {{ gatus_storage_path }} +{% endif %} + caching: {{ gatus_storage_caching | bool | lower }} + maximum-number-of-results: {{ gatus_storage_max_results }} + maximum-number-of-events: {{ gatus_storage_max_events }} + +ui: +{{ gatus_ui | to_nice_yaml(indent=2) | indent(2, true) }} +{% if gatus_alerting %} + +alerting: +{{ gatus_alerting | to_nice_yaml(indent=2) | indent(2, true) }} +{% else %} + +# No alerting provider is configured yet. Gatus still evaluates every condition +# and records every result; it simply has nowhere to shout. Setting +# `gatus_alerting` is the single change needed to wire one up. +{% endif %} +{% if gatus_basic_auth %} + +security: + basic: +{{ gatus_basic_auth | to_nice_yaml(indent=2) | indent(4, true) }} +{% endif %} +{% if gatus_maintenance %} + +maintenance: +{{ gatus_maintenance | to_nice_yaml(indent=2) | indent(2, true) }} +{% endif %} diff --git a/ansible/roles/gatus/templates/docker-compose.yml.j2 b/ansible/roles/gatus/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..512c9c5 --- /dev/null +++ b/ansible/roles/gatus/templates/docker-compose.yml.j2 @@ -0,0 +1,53 @@ +# Managed by Ansible (roles/gatus) +services: + gatus: + image: {{ gatus_image }} + container_name: gatus + restart: unless-stopped + + # The image is FROM scratch: no /etc/passwd, so there is no user to drop to + # by name and the default is root. Run it by numeric id instead. + user: "{{ gatus_uid }}:{{ gatus_gid }}" + + ports: + # Loopback on purpose. Caddy fronts this, and the external-endpoint push + # API is served from the same listener as the dashboard - publishing + # 0.0.0.0 would put both straight on the public internet. + - "{{ gatus_bind_address }}:{{ gatus_port }}:{{ gatus_port }}" + + environment: + GATUS_CONFIG_PATH: /config + GATUS_LOG_LEVEL: "{{ gatus_log_level }}" + + volumes: + - {{ gatus_config_dir }}:/config:ro + - {{ gatus_data_dir }}:/data + + # Hardening. The systemd unit this replaced got most of it from + # ProtectSystem/NoNewPrivileges/etc; these are the container equivalents. + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL +{% if gatus_allow_icmp %} + cap_add: + # icmp:// endpoints need raw sockets. Dropped above with ALL, added back + # explicitly so the grant is visible rather than inherited from root. + - NET_RAW +{% endif %} + + networks: + # Shared with signal-api, so alerts can be delivered by service name. + # 127.0.0.1 inside this container is the container, not the host. + - {{ gatus_network }} + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +networks: + {{ gatus_network }}: + external: true diff --git a/ansible/roles/gatus/templates/endpoint-self.yaml.j2 b/ansible/roles/gatus/templates/endpoint-self.yaml.j2 new file mode 100644 index 0000000..197ae56 --- /dev/null +++ b/ansible/roles/gatus/templates/endpoint-self.yaml.j2 @@ -0,0 +1,25 @@ +# 00-self.yaml — managed by Ansible (roles/gatus) +# +# Gatus refuses to start with no endpoints at all: +# panic: error parsing config: configuration should contain at least one +# endpoint or suite +# +# So the role ships one. Checking its own listener is not as circular as it +# looks: it proves the config directory parsed, the container is serving, and +# the storage backend accepted a write. If this row is missing from the +# dashboard, the dashboard is not telling you the truth about anything else. +# +# It is also what keeps `gatus` deployable before any service has contributed +# an endpoint file of its own. + +endpoints: + - name: gatus + group: infrastructure + url: "http://localhost:{{ gatus_port }}/health" + interval: 60s + conditions: + - "[STATUS] == 200" +{% if gatus_default_alerts %} + alerts: +{{ gatus_default_alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} diff --git a/ansible/roles/gatus_endpoint/defaults/main.yml b/ansible/roles/gatus_endpoint/defaults/main.yml new file mode 100644 index 0000000..90a2c19 --- /dev/null +++ b/ansible/roles/gatus_endpoint/defaults/main.yml @@ -0,0 +1,58 @@ +--- +# One invocation writes ONE file into Gatus's endpoints directory. Gatus merges +# every *.yaml under GATUS_CONFIG_PATH and appends lists, so each caller owns +# its own file and they compose without coordinating - the same shape as +# caddy_site, where each service contributes its own vhost. + +# Filename stem: .yaml +gatus_endpoint_name: "" + +# PULLED endpoints - Gatus makes the request and evaluates conditions. +# - {name, group, url, interval, conditions: [...], alerts: [...]} +# +# A DNS check adds `dns: {query-type, query-name}` - and note that for those, +# `url` is the RESOLVER to ask, not the name being looked up. +# A domain-expiry check is just `url: ` with a [DOMAIN_EXPIRATION] +# condition; it uses WHOIS/RDAP and needs no scheme. +gatus_endpoint_pulled: [] + +# EXTERNAL endpoints - the host pushes its own result. Gatus never reaches out, +# which is what makes this work for machines behind NAT and for state that has +# no pollable surface at all (disk usage, ZFS health, UPS mains). +# +# - {name, group, token, heartbeat, alerts: [...]} +# +# `heartbeat` is the important one: if nothing reports within that window Gatus +# alerts. That is what makes a push check detect its own failure - a dead timer +# looks exactly like a dead host, which is the correct reading. +gatus_endpoint_external: [] + +# Where the files live. Matches roles/gatus. +gatus_config_dir: /opt/gatus/config +gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" +gatus_gid: 10001 + +# Alerts attached to every endpoint in this file that does not specify its own. +# +# Gatus's provider-level `default-alert` only supplies DEFAULTS - an endpoint +# still has to opt in with `alerts: - type: signal` or it alerts on nothing at +# all. With ~90 endpoints that cannot be written by hand, so it is applied here. +# +# failure-threshold is set by the CALLER, because the right value depends on the +# check's cadence and there is no single correct default. See the note in +# infra/400_host_monitoring.yml. +gatus_endpoint_default_alerts: [] + +# UI options applied to every PULLED endpoint that does not set its own. +# +# Only pulled endpoints can carry this - an external (push) endpoint has no `ui` +# field at all, because it has no conditions to display. +# +# Note that every hide-* option already defaults to false upstream, so there is +# nothing to un-hide. The one setting that genuinely shows MORE is +# resolve-successful-conditions: by default a failing check displays the real +# value - "[STATUS] (502) == 200" - while a passing one drops it and shows only +# "[STATUS] == 200". Turning it on resolves both, so a healthy DNS check shows +# the IP it actually resolved rather than just the assertion. +gatus_endpoint_default_ui: + resolve-successful-conditions: true diff --git a/ansible/roles/gatus_endpoint/tasks/main.yml b/ansible/roles/gatus_endpoint/tasks/main.yml new file mode 100644 index 0000000..40f3448 --- /dev/null +++ b/ansible/roles/gatus_endpoint/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Assert gatus_endpoint parameters are sane + ansible.builtin.assert: + that: + - gatus_endpoint_name | length > 0 + - gatus_endpoint_pulled | length > 0 or gatus_endpoint_external | length > 0 + fail_msg: >- + gatus_endpoint needs a name and at least one of gatus_endpoint_pulled or + gatus_endpoint_external. An empty file would contribute nothing, and a + config with no endpoints at all makes Gatus refuse to start. + +- name: "Write Gatus endpoints '{{ gatus_endpoint_name }}'" + ansible.builtin.template: + src: endpoints.yaml.j2 + dest: "{{ gatus_endpoints_dir }}/{{ gatus_endpoint_name }}.yaml" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + # No handler. Gatus polls its own config every 30s + # (main.listenToConfigurationFileChanges) and reloads itself, so writing the + # file IS the deploy. A handler here would also fail whenever this role is + # used without roles/gatus loaded. diff --git a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 new file mode 100644 index 0000000..680691e --- /dev/null +++ b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 @@ -0,0 +1,60 @@ +# {{ gatus_endpoint_name }}.yaml — managed by Ansible (roles/gatus_endpoint) +# +# Contributed by a playbook, not hand-edited. Gatus appends the lists in every +# *.yaml under its config directory, so this file adds to whatever else is +# registered without knowing about it. +{% if gatus_endpoint_external %} + +external-endpoints: +{% for e in gatus_endpoint_external %} + - name: {{ e.name }} + group: {{ e.group }} + token: "{{ e.token }}" +{% if e.heartbeat is defined %} + heartbeat: + interval: {{ e.heartbeat }} +{% endif %} +{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} +{% if _alerts %} + alerts: +{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{# external endpoints have no `ui` field upstream - they carry no conditions #} +{% endfor %} +{% endif %} +{% if gatus_endpoint_pulled %} + +endpoints: +{% for e in gatus_endpoint_pulled %} + - name: {{ e.name }} + group: {{ e.group }} + url: "{{ e.url }}" + interval: {{ e.interval | default('60s') }} +{% if e.dns is defined %} + # A DNS endpoint: `url` is the RESOLVER to ask, not the thing being asked + # about. The name being queried lives in query-name, and [BODY] holds the + # resolved record for the conditions below. + dns: + query-type: {{ e.dns['query-type'] }} + query-name: {{ e.dns['query-name'] }} +{% endif %} +{% if e.client is defined %} + client: +{{ e.client | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} + conditions: +{% for c in e.conditions %} + - "{{ c }}" +{% endfor %} +{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} +{% if _alerts %} + alerts: +{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{% set _ui = e.ui | default(gatus_endpoint_default_ui) %} +{% if _ui %} + ui: +{{ _ui | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{% endfor %} +{% endif %} diff --git a/ansible/roles/healthcheck/defaults/main.yml b/ansible/roles/healthcheck/defaults/main.yml new file mode 100644 index 0000000..51534c2 --- /dev/null +++ b/ansible/roles/healthcheck/defaults/main.yml @@ -0,0 +1,54 @@ +--- +# One health check: a script, a systemd service, a timer, and an optional push. +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed -healthcheck.service +# Reporting anywhere else is optional and generic. Point healthcheck_push_url at +# Gatus, or at whatever replaces it, or at nothing. + +healthcheck_name: "" # e.g. disk-usage -> disk-usage-healthcheck +healthcheck_description: "" + +# The check itself. Pick ONE: +# healthcheck_check: a template under templates/checks/ (without .sh.j2) +# healthcheck_command: a shell one-liner that exits 0 for healthy +healthcheck_check: "" +healthcheck_command: "" + +# systemd timer. OnUnitActiveSec unless healthcheck_on_calendar is set. +healthcheck_interval: "5min" +healthcheck_on_calendar: "" +healthcheck_boot_delay: "2min" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Gatus external endpoints: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty url = check and log only, which is a valid state and not an error. +healthcheck_push_url: "" +healthcheck_push_token: "" + +# Some checks report MORE THAN ONE result - a host with four systemd services +# needs four endpoints, or a single red light cannot tell you which one died. +# Those set healthcheck_push_base to the endpoints COLLECTION and the check body +# appends each key itself, the same way check-backups.sh reports per source. +healthcheck_push_base: "" + +# Units for the systemd-units check. Each becomes its own Gatus endpoint. +healthcheck_units: [] +# Prefix for the per-unit endpoint keys, e.g. "services_vipy" -> services_vipy-caddy. +healthcheck_units_key_prefix: "" + +healthcheck_script_dir: /usr/local/bin +healthcheck_log_dir: /var/log/healthchecks + +# Per-check knobs, consumed by the templates under checks/ +healthcheck_disk_threshold: 85 # percent +healthcheck_cpu_temp_threshold: 80 # celsius +healthcheck_zfs_pool: "" +# ZFS degrades badly once a pool passes roughly 80% - allocation gets slow and +# fragmentation becomes hard to undo, and unlike a normal filesystem you cannot +# simply delete your way back to good performance. So this alarms well before +# the pool is actually out of space. +healthcheck_zfs_capacity_threshold: 80 +healthcheck_ups_name: "" diff --git a/ansible/roles/healthcheck/tasks/main.yml b/ansible/roles/healthcheck/tasks/main.yml new file mode 100644 index 0000000..f37a0d5 --- /dev/null +++ b/ansible/roles/healthcheck/tasks/main.yml @@ -0,0 +1,90 @@ +--- +- name: "Assert healthcheck '{{ healthcheck_name }}' is fully specified" + ansible.builtin.assert: + that: + - healthcheck_name | length > 0 + - healthcheck_description | length > 0 + - (healthcheck_check | length > 0) != (healthcheck_command | length > 0) + - not (healthcheck_push_url | length > 0) or (healthcheck_push_token | length > 0) + fail_msg: >- + healthcheck needs a name, a description, exactly one of healthcheck_check + or healthcheck_command, and a token whenever a push URL is set. A push URL + with no token would report to Gatus and be rejected 401 on every run. + +# Deduplicated across the whole play run. This role is included once PER CHECK, +# and a host with several checks was otherwise running apt several times to +# install a curl that was already there - 29 apt transactions estate-wide, and +# the slowest thing in the deploy by a wide margin. The fact below remembers +# what has already been ensured on this host. +- name: Install healthcheck dependencies + ansible.builtin.package: + name: "{{ healthcheck_wanted_packages }}" + state: present + vars: + healthcheck_wanted_packages: >- + {{ (healthcheck_packages | default(['curl'])) + | difference(healthcheck_installed_packages | default([])) }} + when: healthcheck_wanted_packages | length > 0 + +- name: Remember which dependencies this host already has + ansible.builtin.set_fact: + healthcheck_installed_packages: >- + {{ (healthcheck_installed_packages | default([])) + | union(healthcheck_packages | default(['curl'])) }} + +- name: Create the healthcheck log directory + ansible.builtin.file: + path: "{{ healthcheck_log_dir }}" + state: directory + owner: root + group: root + mode: "0750" + +- name: "Install the {{ healthcheck_name }} check script" + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" + owner: root + group: root + mode: "0755" + +# The token is in this unit file, so it must not be world-readable. +- name: "Install the {{ healthcheck_name }} systemd service" + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.service" + owner: root + group: root + mode: "0600" + +- name: "Install the {{ healthcheck_name }} systemd timer" + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.timer" + owner: root + group: root + mode: "0644" + +# `restarted`, not `started`: started is a no-op on an already-active timer, so +# a changed interval or a stuck timer would never be picked up. +- name: "Enable and start the {{ healthcheck_name }} timer" + ansible.builtin.systemd: + name: "{{ healthcheck_name }}-healthcheck.timer" + enabled: yes + state: restarted + daemon_reload: yes + +- name: "Run the {{ healthcheck_name }} check once now" + ansible.builtin.command: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" + environment: + HEALTHCHECK_PUSH_URL: "{{ healthcheck_push_url }}" + HEALTHCHECK_PUSH_TOKEN: "{{ healthcheck_push_token }}" + register: healthcheck_first_run + changed_when: false + failed_when: false + +- name: "Report the first {{ healthcheck_name }} result" + ansible.builtin.debug: + msg: >- + {{ healthcheck_name }}: {{ 'HEALTHY' if healthcheck_first_run.rc == 0 + else 'UNHEALTHY (rc=' ~ healthcheck_first_run.rc ~ ')' }} diff --git a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 new file mode 100644 index 0000000..cf57509 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 @@ -0,0 +1,28 @@ + # Hottest core across every thermal zone and hwmon sensor lm-sensors knows + # about. Reading the hottest rather than an average is deliberate: one core + # throttling is a real problem that an average hides. + local threshold={{ healthcheck_cpu_temp_threshold }} + local hottest=0 label="" + + while read -r t; do + [ -z "$t" ] && continue + t=${t%.*} + if [ "$t" -gt "$hottest" ]; then hottest=$t; fi + done < <(sensors -u 2>/dev/null | awk '/_input:/ && /temp/ {print $2}') + + # Fall back to the kernel thermal zones if lm-sensors reports nothing. + if [ "$hottest" -eq 0 ]; then + for z in /sys/class/thermal/thermal_zone*/temp; do + [ -r "$z" ] || continue + local milli; milli=$(cat "$z" 2>/dev/null) || continue + local c=$((milli / 1000)) + if [ "$c" -gt "$hottest" ]; then hottest=$c; label=$(cat "${z%/temp}/type" 2>/dev/null); fi + done + fi + + if [ "$hottest" -eq 0 ]; then + MESSAGE="no temperature sensors readable" + return 1 + fi + MESSAGE="${hottest}C${label:+ (${label})}" + [ "$hottest" -lt "$threshold" ] diff --git a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 new file mode 100644 index 0000000..b0e76ce --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 @@ -0,0 +1,19 @@ + # Every real filesystem must be under the threshold. tmpfs, devtmpfs, + # squashfs and overlay are excluded: they are either RAM, read-only, or + # container layers, and none of them fills up in a way an operator can act on. + local threshold={{ healthcheck_disk_threshold }} + local worst=0 worst_mount="" over="" + + while read -r pct mount; do + pct=${pct%\%} + [ -z "$pct" ] && continue + if [ "$pct" -gt "$worst" ]; then worst=$pct; worst_mount=$mount; fi + if [ "$pct" -ge "$threshold" ]; then over="${over}${over:+, }${mount} ${pct}%"; fi + done < <(df -P -x tmpfs -x devtmpfs -x squashfs -x overlay --output=pcent,target 2>/dev/null | tail -n +2) + + if [ -n "$over" ]; then + MESSAGE="over ${threshold}%: ${over}" + return 1 + fi + MESSAGE="max ${worst}% on ${worst_mount:-/}" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 new file mode 100644 index 0000000..141e478 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 @@ -0,0 +1,6 @@ + # Liveness has no test to run: the fact that this script executed at all is + # the signal. What proves the host is alive is the PUSH arriving at Gatus, + # and what detects the host being dead is the heartbeat window expiring with + # no push. So this always succeeds - the reporting is the check. + MESSAGE="up since $(uptime -p 2>/dev/null || echo unknown)" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 b/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 new file mode 100644 index 0000000..90c6193 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 @@ -0,0 +1,33 @@ + # One result PER UNIT, not one for the host. A host running four services + # needs four endpoints: a single red light would tell you "something on vipy + # is down" without saying which, and that is the question you actually have. + # + # Keys are host-qualified because unit names collide - caddy runs on four + # machines. Gatus builds the key as sanitize(group)_sanitize(name), so + # group "services" + name "vipy/caddy" becomes services_vipy-caddy. + local prefix="{{ healthcheck_units_key_prefix }}" + local down="" + + for unit in {{ healthcheck_units | join(' ') }}; do + local state substate + state=$(systemctl is-active "$unit" 2>/dev/null || true) + substate=$(systemctl show -p SubState --value "$unit" 2>/dev/null || true) + + if [ "$state" = "active" ]; then + report_key "${prefix}-${unit}" "true" "${unit} active (${substate:-running})" + else + # `failed` and `inactive` are different stories: one crashed, one was + # stopped. Both are down, but the message should say which. + local detail="${unit} is ${state:-unknown}" + [ "$state" = "failed" ] && detail="${detail} (${substate:-failed})" + report_key "${prefix}-${unit}" "false" "$detail" + down="${down}${down:+, }${unit}=${state:-unknown}" + fi + done + + if [ -n "$down" ]; then + MESSAGE="down: ${down}" + return 1 + fi + MESSAGE="all {{ healthcheck_units | length }} units active" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 new file mode 100644 index 0000000..508c860 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 @@ -0,0 +1,18 @@ + # OL means on line power. Anything else - OB (on battery), LB (low battery), + # or no answer at all - is a failure worth waking up for, because the + # hypervisor has a finite number of minutes left. + local ups="{{ healthcheck_ups_name }}" + local status charge runtime load + + status=$(upsc "${ups}@localhost" ups.status 2>/dev/null) + if [ -z "$status" ]; then + MESSAGE="cannot reach UPS ${ups} via upsd" + return 1 + fi + + charge=$(upsc "${ups}@localhost" battery.charge 2>/dev/null) + runtime=$(upsc "${ups}@localhost" battery.runtime 2>/dev/null) + load=$(upsc "${ups}@localhost" ups.load 2>/dev/null) + + MESSAGE="status=${status} charge=${charge}% runtime=${runtime}s load=${load}%" + [[ "$status" == *"OL"* ]] diff --git a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 new file mode 100644 index 0000000..ebe88b9 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 @@ -0,0 +1,63 @@ + # Five conditions, all of which have to hold. Ported from the check that + # infra/nodito/32_zfs_pool_setup_playbook.yml deployed, which was correct - + # only its reporting was tied to Uptime Kuma. + local pool="{{ healthcheck_zfs_pool }}" + local json issues="" + + json=$(zpool status -j "$pool" 2>&1) || { MESSAGE="zpool status failed: ${json}"; return 1; } + + # 1. pool state + local state + state=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].state') + [ "$state" = "ONLINE" ] || issues="${issues}${issues:+; }pool ${state}" + + # 2. every vdev and device ONLINE + local bad + bad=$(echo "$json" | jq -r --arg p "$pool" ' + .pools[$p].vdevs[] | .. | objects + | select(.state? and .state != "ONLINE") + | "\(.name // "unknown"):\(.state)"' 2>/dev/null | paste -sd, -) + [ -z "$bad" ] || issues="${issues}${issues:+; }devices ${bad}" + + # 3. resilver in progress + local fn st + fn=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.function // "NONE"') + st=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.state // "NONE"') + if [ "$fn" = "RESILVER" ] && [ "$st" = "SCANNING" ]; then + issues="${issues}${issues:+; }resilvering" + fi + + # 4. read/write/checksum errors. ZFS reports these as strings. + local errs + errs=$(echo "$json" | jq -r --arg p "$pool" ' + .pools[$p].vdevs[] | .. | objects + | select(.name? and ((.read_errors // "0" | tonumber) > 0 + or (.write_errors // "0" | tonumber) > 0 + or (.checksum_errors // "0" | tonumber) > 0)) + | "\(.name) r=\(.read_errors) w=\(.write_errors) c=\(.checksum_errors)"' 2>/dev/null | paste -sd, -) + [ -z "$errs" ] || issues="${issues}${issues:+; }errors ${errs}" + + # 5. errors from the last scrub + local scan_err + scan_err=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.errors // "0"') + if [ -n "$scan_err" ] && [ "$scan_err" != "0" ] && [ "$scan_err" != "null" ]; then + issues="${issues}${issues:+; }scan errors ${scan_err}" + fi + + # 6. Capacity. Not an error condition in `zpool status` - a 95% full pool is + # reported perfectly ONLINE - so it has to be read separately, and it is + # the failure you get warning of rather than the one you discover. + local capacity + capacity=$(zpool list -H -o capacity "$pool" 2>/dev/null | tr -dc '0-9') + if [ -z "$capacity" ]; then + issues="${issues}${issues:+; }cannot read capacity" + elif [ "$capacity" -ge {{ healthcheck_zfs_capacity_threshold }} ]; then + issues="${issues}${issues:+; }pool ${capacity}% full (>={{ healthcheck_zfs_capacity_threshold }}%)" + fi + + if [ -n "$issues" ]; then MESSAGE="$issues"; return 1; fi + + local scrub + scrub=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.start_time // "never"') + MESSAGE="${pool} ONLINE, ${capacity}% full, last scrub ${scrub}" + return 0 diff --git a/ansible/roles/healthcheck/templates/healthcheck.service.j2 b/ansible/roles/healthcheck/templates/healthcheck.service.j2 new file mode 100644 index 0000000..3ddd8aa --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description={{ healthcheck_description }} +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart={{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +Environment=HEALTHCHECK_PUSH_BASE={{ healthcheck_push_base }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..6667878 --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 @@ -0,0 +1,82 @@ +#!/bin/bash +# {{ healthcheck_name }} — {{ healthcheck_description }} +# Managed by Ansible (roles/healthcheck). Do not edit on the host. +# +# The exit code is the real answer; systemd stores it: +# systemctl is-failed {{ healthcheck_name }}-healthcheck.service +# The push below is an optional extra, and having no URL is normal. + +set -uo pipefail + +LOG_FILE="{{ healthcheck_log_dir }}/{{ healthcheck_name }}.log" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +# Set only by checks that report several results; see healthcheck_push_base. +PUSH_BASE="${HEALTHCHECK_PUSH_BASE:-}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $*" >> "$LOG_FILE"; } + +# Report to Gatus as an external endpoint. Note this is a POST with a bearer +# token, not a GET with a query string - it is not the shape Uptime Kuma used. +report() { + local success="$1" message="$2" + + # No push URL is normal, not an error: the exit code below is a complete + # answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + + local code + code=$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + --max-time 15 --retry 2 --retry-delay 3 \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null) + + if [ "$code" = "200" ]; then + log "reported success=${success}" + else + log "ERROR: report failed (HTTP ${code})" + return 1 + fi +} + +# Report to an arbitrary endpoint key under PUSH_BASE. Used by checks that +# produce one result per item rather than a single verdict. +report_key() { + local key="$1" success="$2" message="$3" + [ -n "$PUSH_BASE" ] || return 0 + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true +} + +# ── the check ──────────────────────────────────────────────────────────────── +# +# The blank line before the closing brace below is load-bearing. Jinja strips an +# included template's trailing newline, and trim_blocks (on by default in +# Ansible) then eats the newline after {% raw %}{% endif %}{% endraw %} - so without it the brace lands +# on the same line as the check body's last statement, producing `return 0}` and +# a script that dies with "syntax error: unexpected end of file". +check() { +{% if healthcheck_check %} +{% include 'checks/' ~ healthcheck_check ~ '.sh.j2' %} +{% else %} + {{ healthcheck_command }} +{% endif %} + +} + +MESSAGE="" +if check; then + log "OK${MESSAGE:+ - $MESSAGE}" + report "true" "${MESSAGE:-ok}" + exit 0 +else + log "FAILED${MESSAGE:+ - $MESSAGE}" + report "false" "${MESSAGE:-check failed}" + exit 1 +fi diff --git a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..0271735 --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Run {{ healthcheck_description }} +Requires={{ healthcheck_name }}-healthcheck.service + +[Timer] +OnBootSec={{ healthcheck_boot_delay }} +{% if healthcheck_on_calendar %} +OnCalendar={{ healthcheck_on_calendar }} +{% else %} +OnUnitActiveSec={{ healthcheck_interval }} +{% endif %} +# Run a missed occurrence on the next boot rather than silently skipping it. +Persistent=true +# Spread the pushes out so twelve hosts do not all report in the same second. +RandomizedDelaySec={{ healthcheck_randomized_delay | default('30') }} + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/mempool/README.md b/ansible/roles/mempool/README.md new file mode 100644 index 0000000..d8282cc --- /dev/null +++ b/ansible/roles/mempool/README.md @@ -0,0 +1,69 @@ +# `mempool` + +Deploys the [Mempool](https://mempool.space) block explorer as a three-container +Docker Compose stack — MariaDB, backend, frontend — on `mempool-box`, and keeps a +health check on each. + +Converted from `deploy_mempool_playbook.yml` (745 lines) under Plan 6. The +playbook is now 37 lines: this role, plus a second play that publishes the +frontend through Caddy on the edge host. + +## Phases + +| | | +|---|---| +| `docker.yml` | Docker engine: repo, key, packages, service | +| `deploy.yml` | directories, `docker-compose.yml`, pull, up, wait-for-healthy | +| `healthcheck.yml` | three check scripts, three services, three timers | + +## Three health checks, not one + +Mempool is three moving parts and knowing *which* one is down is the point, so +each gets its own check, unit and timer, driven by the `mempool_healthchecks` +list: + +| | checks | +|---|---| +| `mariadb` | `docker inspect` health status of `mempool-db` | +| `backend` | `GET /api/v1/backend-info` | +| `frontend` | `GET /` | + +Each records its answer in its exit code, which systemd keeps: +`systemctl is-failed mempool-backend-healthcheck.service`. Reporting elsewhere +is one field per check, `push_url`, and is the plug-in point for whatever +monitoring exists. Empty means check, exit honestly, report nowhere. The URLs +are credentials, so callers pass them from the vault. + +Nothing here is specific to a monitoring product. The embedded Python that +created monitors over the Uptime Kuma API, the `/tmp` credentials file, the +push-URL file read back and parsed, and three systemd `Environment=` rewrites +are gone. + +## MariaDB owns its own data directory + +`{{ mempool_mysql_dir }}` is bind-mounted into the container, which runs as uid +**999** and must create files there. The playbook this replaced declared +`owner: "{{ ansible_user }}"` (1000) on it, which had drifted from reality ever +since the containers were created — unnoticed, because the playbook had not been +run since. + +That was not academic. The first real run of this role pulled a newer +`mariadb:10.11` and recreated `mempool-db`; had the chown still been in place, +MariaDB would have come back to a directory it could not write. The role now +ensures the directory exists and leaves ownership to the container. + +## `mempool_frontend_port` lives in `services_config.yml` + +Two hosts need it: this role deploys the frontend on `mempool-box`, and the Caddy +play proxies to it from the edge host. A role default is invisible to the second +play, so the value lives in `service_settings.mempool.frontend_port` and the role +default derives from it. + +## Expect `changed=2` on a converged host + +`Pull Mempool images` and `Deploy Mempool containers with docker compose` are +bare `command:` tasks with no `changed_when`, so they always report changed. +That is the idempotent floor, not drift. Everything else reports `ok`. + +**`mariadb:10.11` is a moving tag**, so a run can pull a newer patch release and +recreate the database container. Pin it if that is not what you want. diff --git a/ansible/roles/mempool/defaults/main.yml b/ansible/roles/mempool/defaults/main.yml new file mode 100644 index 0000000..9ce8f6a --- /dev/null +++ b/ansible/roles/mempool/defaults/main.yml @@ -0,0 +1,59 @@ +# Mempool Configuration Variables + +# Version - Pinned to specific release +mempool_version: "v3.2.1" + +# Directories +mempool_dir: /opt/mempool +mempool_data_dir: "{{ mempool_dir }}/data" +mempool_mysql_dir: "{{ mempool_dir }}/mysql" + +# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) +bitcoin_host: "knots-box" +bitcoin_rpc_port: 8332 +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml + +# Network - Fulcrum Electrum server (via Tailnet Magic DNS) +fulcrum_host: "fulcrum-box" +fulcrum_port: 50001 +fulcrum_tls: "false" + +# Mempool network mode +mempool_network: "mainnet" + +# Container ports (internal) +# The edge host's socket-proxy/Caddy play needs this too, and a role default is +# invisible outside this role. The authoritative value for the live deployment is +# in host_vars/mempool_box_local/main.yml, which outranks this; the value here is the +# protocol standard, so the role still works standalone. +mempool_frontend_port: 8080 +mempool_backend_port: 8999 + +# MariaDB settings +mariadb_database: "mempool" +mariadb_user: "mempool" +# Note: mariadb_mempool_password is loaded from group_vars/all/vault.yml + + + +# --- Health checks ---------------------------------------------------------- +# Three independent checks, because Mempool is three moving parts and knowing +# WHICH one is down is the whole point. Each records its answer in its exit +# code, which systemd keeps: +# systemctl is-failed mempool-backend-healthcheck.service +# +# push_url is where to report, and is the single plug-in point for whatever +# monitoring exists. Empty means check, exit honestly, report nowhere. +# The URLs are credentials, so callers pass them from the vault. +healthcheck_push_token: "" + +mempool_healthchecks: + - name: mariadb + label: MariaDB + push_url: "" + - name: backend + label: Backend + push_url: "" + - name: frontend + label: Frontend + push_url: "" diff --git a/ansible/roles/mempool/tasks/deploy.yml b/ansible/roles/mempool/tasks/deploy.yml new file mode 100644 index 0000000..d98df7e --- /dev/null +++ b/ansible/roles/mempool/tasks/deploy.yml @@ -0,0 +1,75 @@ +--- +- name: Create mempool directories + file: + path: "{{ item }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + loop: + - "{{ mempool_dir }}" + - "{{ mempool_data_dir }}" + +# MariaDB owns its own data directory. The container runs as uid 999 and has to +# create files in there; this playbook declared owner: {{ ansible_user }} (1000), +# which had been drifting from reality ever since the containers were created and +# would have broken MariaDB the first time it needed a new file. It went +# unnoticed only because the playbook had not been run since. +# +# So: ensure the directory exists, and let the container own it. On a fresh +# install the mariadb image's entrypoint sets ownership itself. +- name: Ensure the MariaDB data directory exists + file: + path: "{{ mempool_mysql_dir }}" + state: directory + +- name: Create docker-compose.yml for Mempool + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ mempool_dir }}/docker-compose.yml" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' +- name: Pull Mempool images + command: docker compose pull + args: + chdir: "{{ mempool_dir }}" + +- name: Deploy Mempool containers with docker compose + command: docker compose up -d + args: + chdir: "{{ mempool_dir }}" + +- name: Wait for MariaDB to be healthy + command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db + register: mariadb_health + until: mariadb_health.stdout == 'healthy' + retries: 30 + delay: 10 + changed_when: false + +- name: Wait for Mempool backend to start + uri: + url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + method: GET + status_code: 200 + timeout: 10 + register: backend_check + until: backend_check.status == 200 + retries: 30 + delay: 10 + ignore_errors: yes + +- name: Wait for Mempool frontend to be available + uri: + url: "http://localhost:{{ mempool_frontend_port }}" + method: GET + status_code: 200 + timeout: 10 + register: frontend_check + until: frontend_check.status == 200 + retries: 20 + delay: 5 + ignore_errors: yes + +# ═════════════════════════════════════════════════════════════════════════ diff --git a/ansible/roles/mempool/tasks/docker.yml b/ansible/roles/mempool/tasks/docker.yml new file mode 100644 index 0000000..b97e63e --- /dev/null +++ b/ansible/roles/mempool/tasks/docker.yml @@ -0,0 +1,72 @@ +--- +- name: Remove old Docker-related packages + apt: + name: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + purge: yes + autoremove: yes + +- name: Update apt cache + apt: + update_cache: yes + +- name: Install prerequisites + apt: + name: + - ca-certificates + - curl + state: present + +- name: Create directory for Docker GPG key + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + +- name: Download Docker GPG key + get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + +- name: Get Debian architecture + command: dpkg --print-architecture + register: deb_arch + changed_when: false + +- name: Add Docker repository + apt_repository: + repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + filename: docker + state: present + update_cache: yes + +- name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + +- name: Ensure Docker is started and enabled + systemd: + name: docker + enabled: yes + state: started + +- name: Add user to docker group + user: + name: "{{ ansible_user }}" + groups: docker + append: yes + diff --git a/ansible/roles/mempool/tasks/healthcheck.yml b/ansible/roles/mempool/tasks/healthcheck.yml new file mode 100644 index 0000000..1e1ac28 --- /dev/null +++ b/ansible/roles/mempool/tasks/healthcheck.yml @@ -0,0 +1,58 @@ +--- +# Three checks, one per moving part. The Uptime Kuma specifics that used to +# follow — an embedded Python script creating monitors over the API, a /tmp +# credentials file, a push-URL file read back and parsed, and three systemd +# Environment= rewrites — are gone. Where each reports is now hc.push_url. +- name: Create Mempool health check scripts + ansible.builtin.template: + src: "healthcheck-{{ hc.name }}.sh.j2" + dest: "/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh" + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" + +- name: Create systemd services for health checks + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.service" + owner: root + group: root + mode: "0600" + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" + +- name: Create systemd timers for health checks + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.timer" + owner: root + group: root + mode: '0644' + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +# Ungated on purpose: enabling a timer is deployment, not monitoring. The +# deprecation banner swept this up with the push plumbing, so Ansible stopped +# managing three timers that are in fact running on the host. +- name: Enable and start health check timers + systemd: + name: "mempool-{{ hc.name }}-healthcheck.timer" + enabled: yes + state: started + loop: "{{ mempool_healthchecks }}" + loop_control: + loop_var: hc + label: "{{ hc.name }}" diff --git a/ansible/roles/mempool/tasks/main.yml b/ansible/roles/mempool/tasks/main.yml new file mode 100644 index 0000000..9feb652 --- /dev/null +++ b/ansible/roles/mempool/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: docker.yml +- ansible.builtin.import_tasks: deploy.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/mempool/templates/docker-compose.yml.j2 b/ansible/roles/mempool/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..fda2ad1 --- /dev/null +++ b/ansible/roles/mempool/templates/docker-compose.yml.j2 @@ -0,0 +1,75 @@ +# All containers use host network for Tailscale MagicDNS resolution +services: + mariadb: + image: mariadb:10.11 + container_name: mempool-db + restart: unless-stopped + network_mode: host + environment: + MYSQL_DATABASE: "{{ mariadb_database }}" + MYSQL_USER: "{{ mariadb_user }}" + MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" + MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" + volumes: + - {{ mempool_mysql_dir }}:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + mempool-backend: + image: mempool/backend:{{ mempool_version }} + container_name: mempool-backend + restart: unless-stopped + network_mode: host + environment: + # Database (localhost since all containers share host network) + DATABASE_ENABLED: "true" + DATABASE_HOST: "127.0.0.1" + DATABASE_DATABASE: "{{ mariadb_database }}" + DATABASE_USERNAME: "{{ mariadb_user }}" + DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" + # Bitcoin Core/Knots (via Tailnet MagicDNS) + CORE_RPC_HOST: "{{ bitcoin_host }}" + CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" + CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" + CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" + # Electrum (Fulcrum via Tailnet MagicDNS) + ELECTRUM_HOST: "{{ fulcrum_host }}" + ELECTRUM_PORT: "{{ fulcrum_port }}" + ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" + # Mempool settings + MEMPOOL_NETWORK: "{{ mempool_network }}" + MEMPOOL_BACKEND: "electrum" + MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" + volumes: + - {{ mempool_data_dir }}:/backend/cache + depends_on: + mariadb: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + mempool-frontend: + image: mempool/frontend:{{ mempool_version }} + container_name: mempool-frontend + restart: unless-stopped + network_mode: host + environment: + FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" + BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" + depends_on: + - mempool-backend + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s diff --git a/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 new file mode 100644 index 0000000..fe0f57c --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash +# Mempool backend health check — managed by Ansible (roles/mempool) +# The exit code is the answer; systemd keeps it. Reporting is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +BACKEND_PORT="{{ mempool_backend_port }}" + +check() { + curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 +} + +report() { + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "$1" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true +} + +if check; then report up "OK"; exit 0 +else echo "Mempool backend not responding"; report down "Mempool backend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 new file mode 100644 index 0000000..8452202 --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash +# Mempool frontend health check — managed by Ansible (roles/mempool) +# The exit code is the answer; systemd keeps it. Reporting is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +FRONTEND_PORT="{{ mempool_frontend_port }}" + +check() { + curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 +} + +report() { + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "$1" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true +} + +if check; then report up "OK"; exit 0 +else echo "Mempool frontend not responding"; report down "Mempool frontend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 b/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 new file mode 100644 index 0000000..922adab --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash +# Mempool MariaDB health check — managed by Ansible (roles/mempool) +# The exit code is the answer; systemd keeps it. Reporting is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +check() { +{% raw %} + [ "$(docker inspect --format='{{.State.Health.Status}}' mempool-db 2>/dev/null)" = "healthy" ] +{% endraw %} +} + +report() { + # No push URL is normal, not an error. The previous version logged + # "ERROR: UPTIME_KUMA_PUSH_URL not set" on every fire, once a minute. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "$1" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true +} + +if check; then report up "OK"; exit 0 +else echo "MariaDB container unhealthy"; report down "MariaDB container unhealthy"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck.service.j2 b/ansible/roles/mempool/templates/healthcheck.service.j2 new file mode 100644 index 0000000..76fa4e9 --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Mempool {{ hc.label }} Health Check +After=network.target docker.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ hc.push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/mempool/templates/healthcheck.timer.j2 b/ansible/roles/mempool/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..868cb7c --- /dev/null +++ b/ansible/roles/mempool/templates/healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Mempool {{ hc.name }} Health Check Timer + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/phoenixd/README.md b/ansible/roles/phoenixd/README.md new file mode 100644 index 0000000..3c147e3 --- /dev/null +++ b/ansible/roles/phoenixd/README.md @@ -0,0 +1,56 @@ +# `phoenixd` + +Deploys and runs [phoenixd](https://phoenix.acinq.co/server), an ACINQ Lightning +node, on the edge host. LNBits uses it as a wallet backend. The HTTP API stays on +loopback — phoenixd is never published through Caddy. + +Converted from `deploy_phoenixd_playbook.yml` (552 lines) under Plan 6. The +playbook is now 18 lines. + +## Phases + +| | | +|---|---| +| `install.yml` | packages, system user, directories, versioned download and install | +| `service.yml` | systemd unit, start, then first-boot checks (config written, seed created) | +| `healthcheck.yml` | check script, unit, timer | + +## The seed + +`{{ phoenixd_data_dir }}/seed.dat` **is** the funds. phoenixd is deliberately +excluded from the automated backups (Plan 5, Model C): the seed is twelve fixed +words that never change, so an automated job would only manufacture more copies +of a static secret on more machines. Write them down offline, once. + +Note the live file is mode `0644`. That is phoenixd's own doing, not this role's, +and it is worth tightening. + +## Monitoring: one variable, no product knowledge + +The check asks the node itself — the service must be active **and** +`phoenix-cli getinfo` must return a `nodeId` — and records the answer in its exit +code, which systemd keeps: + +```bash +systemctl is-failed phoenixd-healthcheck.service +``` + +That is a complete answer with no monitoring system involved. To report +elsewhere, set `healthcheck_push_url` to anything accepting an HTTP ping. Gone +from this role: the embedded Python that created monitors over the Uptime Kuma +API, the `/tmp` credentials file, the push-URL file written and parsed back, and +the systemd `Environment=` rewrite. + +### Two things the conversion fixed + +**The check used to log an error once a minute.** Its `Environment=` push URL had +been empty since the decommissioning, and the script printed +`ERROR: UPTIME_KUMA_PUSH_URL not set` on every fire — roughly 1,400 times a day. +The exit code was still correct, so nothing was broken; it was pure noise, and +noise that trains you to ignore the log. An unset push URL is now normal and +silent. + +**`Enable and start phoenixd health check timer` was guarded by +`uptime_kuma_enabled`** and so had not run since the decommissioning — while the +timer itself was still live on the host from before. Ansible had quietly stopped +managing something that was still running. Ungated. diff --git a/ansible/services/phoenixd/phoenixd_vars.yml b/ansible/roles/phoenixd/defaults/main.yml similarity index 63% rename from ansible/services/phoenixd/phoenixd_vars.yml rename to ansible/roles/phoenixd/defaults/main.yml index 57218dc..2fcf1ea 100644 --- a/ansible/services/phoenixd/phoenixd_vars.yml +++ b/ansible/roles/phoenixd/defaults/main.yml @@ -35,18 +35,22 @@ phoenixd_http_bind_port: 9740 # Optional webhook for payment events. Leave empty to disable. phoenixd_webhook_url: "" -# Monitoring + + + +# --- Health check ----------------------------------------------------------- +# The check asks phoenixd itself whether it is healthy (service active AND the +# node answers getinfo with a nodeId) and records the answer in its exit code, +# which systemd keeps: +# systemctl is-failed phoenixd-healthcheck.service +# That is a complete answer with no monitoring system involved. phoenixd_healthcheck_script_path: /usr/local/bin/phoenixd-healthcheck-push.sh phoenixd_healthcheck_service_name: phoenixd-healthcheck -phoenixd_monitor_name: "Phoenixd" -# Remote access -remote_host_name: "vipy" -remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" -remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" -remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" -remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" - -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/phoenixd-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/phoenixd_backup.sh" +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. +# Empty means check, log, exit honestly, report nowhere. Any endpoint that +# accepts an HTTP ping works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/phoenixd/handlers/main.yml b/ansible/roles/phoenixd/handlers/main.yml new file mode 100644 index 0000000..7779684 --- /dev/null +++ b/ansible/roles/phoenixd/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: Restart phoenixd + systemd: + name: phoenixd + state: restarted + daemon_reload: yes + +- name: Restart phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/phoenixd/tasks/healthcheck.yml b/ansible/roles/phoenixd/tasks/healthcheck.yml new file mode 100644 index 0000000..d439047 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/healthcheck.yml @@ -0,0 +1,68 @@ +--- +# Everything here answers "is phoenixd healthy" and records the answer. The +# Uptime Kuma specifics that used to follow it — an embedded Python script +# creating monitors over the API, a /tmp credentials file, a push-URL file read +# back and parsed, and a systemd Environment= rewrite — are gone. What reports +# where is now one variable, healthcheck_push_url. See the role README. +- name: Create phoenixd health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ phoenixd_healthcheck_script_path }}" + owner: root + group: root + mode: "0755" + validate: "bash -n %s" + +- name: Create phoenixd health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" + owner: root + group: root + mode: "0600" + notify: Restart phoenixd health check timer + +- name: Create phoenixd health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" + owner: root + group: root + mode: "0644" + notify: Restart phoenixd health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated on purpose. This was guarded by `uptime_kuma_enabled`, but enabling a +# timer is deployment, not monitoring — the deprecation banner swept it up with +# the push plumbing. The timer is in fact running on the host, from before the +# decommissioning, so the guard meant Ansible had stopped managing something +# that was still live. +- name: Enable and start phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + enabled: yes + state: started + +- name: Display post-install information + debug: + msg: | + ✓ phoenixd {{ phoenixd_version }} deployed + + Status: systemctl status phoenixd + Logs: journalctl -u phoenixd -f + CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo + HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) + Data dir: {{ phoenixd_data_dir }} + Health: systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service + + API password (needed to wire LNBits up to this node): + sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf + + ⚠️ BACK UP THE SEED: {{ phoenixd_data_dir }}/seed.dat + Losing it means losing the funds. phoenixd is deliberately excluded + from the automated backups (Plan 5, Model C) because the seed is 12 + fixed words — write them down offline, once: + sudo cat {{ phoenixd_data_dir }}/seed.dat diff --git a/ansible/roles/phoenixd/tasks/install.yml b/ansible/roles/phoenixd/tasks/install.yml new file mode 100644 index 0000000..89a4fa0 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/install.yml @@ -0,0 +1,125 @@ +--- +- name: Install phoenixd runtime dependencies + apt: + name: + - unzip + - curl + state: present + update_cache: yes + +# System User and Directories +- name: Create phoenixd system group + group: + name: "{{ phoenixd_group }}" + system: yes + +- name: Create phoenixd system user + user: + name: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ phoenixd_home }}" + create_home: yes + comment: "phoenixd Lightning node" + +- name: Create phoenixd home directory + file: + path: "{{ phoenixd_home }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0750" + +- name: Create phoenixd data directory + file: + path: "{{ phoenixd_data_dir }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0700" + +# Download and Install +- name: Check if phoenixd is already installed + stat: + path: "{{ phoenixd_bin_dir }}/phoenixd" + register: phoenixd_binary + +- name: Check installed phoenixd version + command: "{{ phoenixd_bin_dir }}/phoenixd --version" + register: phoenixd_installed_version + changed_when: false + failed_when: false + when: phoenixd_binary.stat.exists + +- name: Decide whether phoenixd needs installing + set_fact: + phoenixd_needs_install: >- + {{ not phoenixd_binary.stat.exists + or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} + +- name: Download phoenixd {{ phoenixd_version }} + get_url: + url: "{{ phoenixd_url }}" + dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + mode: "0644" + when: phoenixd_needs_install | bool + +- name: Create temporary extraction directory + file: + path: /tmp/phoenixd-extract + state: directory + mode: "0755" + when: phoenixd_needs_install | bool + +- name: Extract phoenixd archive + unarchive: + src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + dest: /tmp/phoenixd-extract + remote_src: yes + when: phoenixd_needs_install | bool + +- name: Locate extracted binaries + find: + paths: /tmp/phoenixd-extract + patterns: "{{ item }}" + recurse: yes + file_type: file + register: phoenixd_extracted + loop: + - phoenixd + - phoenix-cli + when: phoenixd_needs_install | bool + +- name: Fail if the archive did not contain the expected binaries + assert: + that: + - item.files | length > 0 + fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + +- name: Install phoenixd and phoenix-cli binaries + copy: + src: "{{ item.files[0].path }}" + dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" + remote_src: yes + owner: root + group: root + mode: "0755" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + notify: Restart phoenixd + +- name: Clean up phoenixd download artifacts + file: + path: "{{ item }}" + state: absent + loop: + - "/tmp/phoenixd-{{ phoenixd_version }}.zip" + - /tmp/phoenixd-extract + diff --git a/ansible/roles/phoenixd/tasks/main.yml b/ansible/roles/phoenixd/tasks/main.yml new file mode 100644 index 0000000..bc7ff05 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is how this conversion was verified against the playbook it replaced. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/phoenixd/tasks/service.yml b/ansible/roles/phoenixd/tasks/service.yml new file mode 100644 index 0000000..79bf6f5 --- /dev/null +++ b/ansible/roles/phoenixd/tasks/service.yml @@ -0,0 +1,52 @@ +--- +- name: Build phoenixd command line arguments + set_fact: + phoenixd_args: >- + {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) + + ['--chain', phoenixd_chain] + + ['--auto-liquidity', phoenixd_auto_liquidity] + + ['--http-bind-ip', phoenixd_http_bind_ip] + + ['--http-bind-port', phoenixd_http_bind_port | string] + + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) + + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) + + ['--silent'] }} + +- name: Create phoenixd systemd service + ansible.builtin.template: + src: phoenixd.service.j2 + dest: /etc/systemd/system/phoenixd.service + owner: root + group: root + mode: "0644" + notify: Restart phoenixd + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start phoenixd + systemd: + name: phoenixd + enabled: yes + state: started + +- name: Flush handlers so phoenixd is running before we inspect its data dir + meta: flush_handlers + +# --- First boot checks --- +- name: Wait for phoenixd to write its config file + wait_for: + path: "{{ phoenixd_data_dir }}/phoenix.conf" + state: present + timeout: 120 + +- name: Check that the seed file exists + stat: + path: "{{ phoenixd_data_dir }}/seed.dat" + register: phoenixd_seed_file + +- name: Fail if phoenixd did not create a seed + assert: + that: + - phoenixd_seed_file.stat.exists + fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" diff --git a/ansible/roles/phoenixd/templates/healthcheck.service.j2 b/ansible/roles/phoenixd/templates/healthcheck.service.j2 new file mode 100644 index 0000000..6ff21ab --- /dev/null +++ b/ansible/roles/phoenixd/templates/healthcheck.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=phoenixd Health Check +After=network.target phoenixd.service + +[Service] +Type=oneshot +User=root +ExecStart={{ phoenixd_healthcheck_script_path }} +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/phoenixd/templates/healthcheck.sh.j2 b/ansible/roles/phoenixd/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..a1717c5 --- /dev/null +++ b/ansible/roles/phoenixd/templates/healthcheck.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash +# phoenixd health check — managed by Ansible (roles/phoenixd) +# +# Asks the node whether it is healthy and records the answer in the exit code, +# which systemd keeps: +# systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service +# That is a complete answer on its own. Reporting anywhere else is optional. +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" +export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" + +check_phoenixd() { + # Service must be active and the node must answer getinfo. + # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, + # but not the bind address, so pass it explicitly. + systemctl is-active --quiet phoenixd && \ + {{ phoenixd_bin_dir }}/phoenix-cli \ + --http-bind-ip {{ phoenixd_http_bind_ip }} \ + --http-bind-port {{ phoenixd_http_bind_port }} \ + getinfo 2>/dev/null | grep -q '"nodeId"' +} + +report() { + local status=$1 msg=$2 + # No push URL configured is NORMAL, not an error: the exit code below still + # answers the question. The previous version logged ERROR here on every + # single fire, once a minute, which is noise that trains you to ignore it. + [ -n "$PUSH_URL" ] || return 0 + # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's + # GET with ?status=up. The callers still pass up/down, so the mapping is + # done here rather than at every call site. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true +} + +if check_phoenixd; then + report "up" "OK" + exit 0 +else + echo "phoenixd is not responding" + report "down" "phoenixd not responding" + exit 1 +fi diff --git a/ansible/roles/phoenixd/templates/healthcheck.timer.j2 b/ansible/roles/phoenixd/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..ee07257 --- /dev/null +++ b/ansible/roles/phoenixd/templates/healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=phoenixd Health Check Timer + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/phoenixd/templates/phoenixd.service.j2 b/ansible/roles/phoenixd/templates/phoenixd.service.j2 new file mode 100644 index 0000000..44fdb94 --- /dev/null +++ b/ansible/roles/phoenixd/templates/phoenixd.service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=phoenixd - Lightning Network Node +Documentation=https://phoenix.acinq.co/server +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ phoenixd_user }} +Group={{ phoenixd_group }} +WorkingDirectory={{ phoenixd_home }} +Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} +ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} +Restart=always +RestartSec=30 +TimeoutStartSec=120 +TimeoutStopSec=120 +StandardOutput=journal +StandardError=journal + +# Hardening: the node only ever writes to its own data directory +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={{ phoenixd_data_dir }} + +LimitNOFILE=65535 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/signal_api/README.md b/ansible/roles/signal_api/README.md new file mode 100644 index 0000000..eadcfb7 --- /dev/null +++ b/ansible/roles/signal_api/README.md @@ -0,0 +1,133 @@ +# signal_api + +Runs [signal-cli-rest-api](https://github.com/bbernhard/signal-cli-rest-api) on +the `observability` host. Gatus uses it to deliver alerts over Signal. + +Gatus does not speak Signal — it POSTs JSON to this service, which holds the +Signal identity and does the protocol work. + +## It is never published, and that is not optional + +**This API has no authentication of any kind.** No key, no token, no basic auth. +Anything that can reach the port can send messages as your identity and read +your Signal. So the compose file publishes **no ports at all** and there is no +Caddy vhost. + +Gatus reaches it over a shared docker network (`monitoring`) by service name: +`http://signal-api:8080`. That is also *why* a shared network is needed rather +than a published port — Gatus runs in a container, so `127.0.0.1` for Gatus is +the Gatus container, not the host. + +The network is created by an explicit Ansible task in both this role and +`gatus`, so neither stack has to be deployed before the other. + +## MODE, and why `native` + +Upstream offers `normal`, `native`, `json-rpc` and `json-rpc-native`. The +json-rpc modes keep a resident JVM daemon and upstream describes them as +"increased memory". + +**This VPS has 464 MB of RAM**, already running Gatus and Caddy. A resident JVM +is not affordable. `native` runs a precompiled GraalVM binary per request — no +daemon, no resident cost — and alerts are rare enough that paying startup cost +per alert is the right trade. + +## Linking the device — a one-time manual step + +Ansible cannot scan a QR code, so this is manual. **Do not use +`/v1/qrcodelink`** — it is broken in `native` mode. + +### The trap + +`GET /v1/qrcodelink?device_name=...` returns: + +```json +{"error":"Couldn't create QR code: no data to encode"} +``` + +The linking itself is fine: running the binary directly inside the container +emits a perfectly good provisioning URI. + +``` +$ docker exec signal-api signal-cli-native link -n gatus +sgnl://linkdevice?uuid=...&pub_key=... +``` + +It is the REST wrapper that fails to capture that output in `native` mode. + +**Do not "fix" this by switching MODE to `normal` or `json-rpc`.** That puts a +JVM in the path of *every alert* on a 464 MB host, permanently degrading the +running system to work around a step performed once. Generate the QR yourself +instead. + +### The procedure + +**`docker exec` runs as root, but the service runs as uid 1000.** Without +`--config`, signal-cli writes the linked account to `/root/.local/share/signal-cli` +— the container's ephemeral layer, NOT the mounted volume. It looks like it +worked (`Associated with: +34…`), `/v1/accounts` keeps returning `[]`, and the +account is destroyed on the next `docker compose up`. Always pass `--config`. + +1. Start the link and capture the URI. It must keep running while you scan: + + docker exec signal-api sh -c "rm -f /tmp/link.uri; \ + nohup signal-cli-native --config /home/.local/share/signal-cli \ + link -n gatus > /tmp/link.uri 2>/tmp/link.log & echo started" + sleep 10 + docker exec signal-api cat /tmp/link.uri + + Do **not** add `setsid`, and do **not** background `docker exec` itself from + the host — the first stops the URI appearing, the second is killed when the + Ansible task returns. The output is block-buffered because stdout is a file, + so the URI appears only after several seconds; `stdbuf` does not help, as the + buffering is GraalVM's, not libc's. + +2. Render the QR on your own machine and scan it: + + qrencode -o /tmp/qr.png -s 12 -m 4 "sgnl://linkdevice?uuid=...&pub_key=..." + +3. Phone: Signal → Settings → Linked devices → **+** → scan. Provisioning links + expire in a couple of minutes, so generate and scan in one sitting. + +4. Confirm — this must list the number, not `[]`: + + docker exec signal-api curl -s http://localhost:8080/v1/accounts + +5. Send a test message: + + docker exec signal-api curl -s -X POST -H "Content-Type: application/json" \ + -d '{"message":"test","number":"+34…","recipients":["+34…"]}' \ + http://localhost:8080/v2/send + +Alerts are sent **from your own number**, so sending to yourself lands in Note +to Self. If the device is ever unlinked from the phone, alerts stop silently — +which is why this service is itself monitored. + +### If the phone says "network error" + +The phone is not the problem. `chat.signal.org` resolves to AWS Global +Accelerator **dualstack** addresses with the AAAA records first, this container +has no IPv6 address at all, and this host's IPv6 path is broken — the same edge +that returned a bogus 404 for the Go tarball. signal-cli reaches for an +unreachable IPv6 address and dies with `Link request error: Connection closed!`, +while the phone can only report a failed handshake. + +That is what `gai.conf` (mounted at `/etc/gai.conf`) fixes. If linking starts +failing again, check it is still mounted and that `getent ahosts chat.signal.org` +returns an IPv4 address first. + +## Backups + +Deliberately **not** backed up. The data directory holds Signal private keys, +and the recovery path is to link again from the phone — which takes a minute and +does not depend on any stored artefact. Backing it up would copy a credential +off the host to buy nothing. + +## Verifying + +```bash +docker ps --filter name=signal-api +docker exec signal-api curl -fsS http://localhost:8080/v1/health +docker exec signal-api curl -fsS http://localhost:8080/v1/accounts +docker logs signal-api --tail 50 +``` diff --git a/ansible/roles/signal_api/defaults/main.yml b/ansible/roles/signal_api/defaults/main.yml new file mode 100644 index 0000000..6bb7d4f --- /dev/null +++ b/ansible/roles/signal_api/defaults/main.yml @@ -0,0 +1,39 @@ +--- +# signal-cli-rest-api: the transport Gatus uses to send Signal messages. +# +# Gatus does not speak Signal. It POSTs JSON to this service, which holds the +# actual Signal identity and does the protocol work. + +# Pinned by digest for the same reason as Gatus: a tag is mutable. +# Upstream publishes no versioned tags worth pinning to, so this pins the +# DIGEST that `latest` resolved to when this was reviewed. `latest` is a moving +# target; a digest is a content address, and `docker compose pull` either +# fetches exactly this image or fails. +signal_api_image_digest: "sha256:2399d449123cdad56c4d859277e3b9127e1a00c4d2ab4601c239882609286cf8" +signal_api_image: "bbernhard/signal-cli-rest-api@{{ signal_api_image_digest }}" + +signal_api_dir: /opt/signal-api +signal_api_data_dir: "{{ signal_api_dir }}/data" + +# MODE matters on this host. Upstream offers normal / native / json-rpc / +# json-rpc-native. json-rpc keeps a resident JVM daemon and upstream describes it +# as "increased memory" - this VPS has 464MB total and already runs Gatus and +# Caddy, so a resident JVM is not affordable. `native` runs a precompiled +# GraalVM binary per request: no daemon, no resident cost, and alerts are rare +# enough that paying startup per alert is the right trade. +signal_api_mode: native + +# Port INSIDE the shared docker network. Never published to the host: this API +# has NO AUTHENTICATION of any kind. Anyone who can reach it can send messages +# as you and read your Signal. +signal_api_port: 8080 + +# Both this and Gatus join this network so Gatus can reach the API by service +# name. Gatus runs in a container, so the host's loopback is NOT reachable from +# it - this is why a shared network is required rather than a published port. +signal_api_network: monitoring +signal_api_service_name: signal-api + +# The uid the upstream image drops to (`setpriv --reuid=1000`). The data +# directory must be owned by it or signal-cli cannot write the account. +signal_api_uid: 1000 diff --git a/ansible/roles/signal_api/tasks/main.yml b/ansible/roles/signal_api/tasks/main.yml new file mode 100644 index 0000000..1aa92b4 --- /dev/null +++ b/ansible/roles/signal_api/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Assert Docker is available + ansible.builtin.command: docker --version + register: signal_docker_check + changed_when: false + +# Created explicitly rather than by either compose file, so neither stack has to +# be deployed before the other and neither owns it. +- name: Ensure the shared monitoring network exists + ansible.builtin.command: "docker network create {{ signal_api_network }}" + register: signal_net + changed_when: "'already exists' not in signal_net.stderr" + failed_when: + - signal_net.rc != 0 + - "'already exists' not in signal_net.stderr" + +- name: Create the signal-api directory + ansible.builtin.file: + path: "{{ signal_api_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +# Owned by the container's uid, NOT root. +# +# The image drops to uid 1000 (`setpriv --reuid=1000`), and a root-owned 0700 +# directory cannot be traversed by uid 1000 - signal-cli then fails to write the +# account and linking silently never completes, leaving a 39-byte accounts.json +# with no accounts and the API returning "Failed to read local accounts list". +# +# 0700 on uid 1000 is still private: only that uid and root can read the Signal +# private keys, which is the property actually wanted. +- name: Create the signal-api data directory owned by the container user + ansible.builtin.file: + path: "{{ signal_api_data_dir }}" + state: directory + owner: "{{ signal_api_uid }}" + group: "{{ signal_api_uid }}" + mode: "0700" + +- name: Write the IPv4-preference resolver config + ansible.builtin.template: + src: gai.conf.j2 + dest: "{{ signal_api_dir }}/gai.conf" + owner: root + group: root + mode: "0644" + +- name: Write the docker compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ signal_api_dir }}/docker-compose.yml" + owner: root + group: root + mode: "0644" + +- name: Pull the pinned signal-api image + ansible.builtin.command: + cmd: docker compose pull + chdir: "{{ signal_api_dir }}" + register: signal_pull + changed_when: "'Downloaded newer image' in signal_pull.stderr or 'Pull complete' in signal_pull.stderr" + +- name: Start signal-api + ansible.builtin.command: + cmd: docker compose up -d --remove-orphans + chdir: "{{ signal_api_dir }}" + register: signal_up + changed_when: "'Started' in signal_up.stderr or 'Created' in signal_up.stderr or 'Recreated' in signal_up.stderr" + +- name: Wait for the API to answer + ansible.builtin.command: + cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/health" + register: signal_health + until: signal_health.rc == 0 + retries: 12 + delay: 5 + changed_when: false + +- name: Report whether an account is linked yet + ansible.builtin.command: + cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/accounts" + register: signal_accounts + changed_when: false + failed_when: false + +- name: Show the linking status + ansible.builtin.debug: + msg: >- + {{ 'Linked account(s): ' ~ signal_accounts.stdout + if (signal_accounts.stdout | default('[]') | trim) not in ['[]', '', 'null'] + else 'NO ACCOUNT LINKED YET - this is a one-time manual step, see the role README.' }} diff --git a/ansible/roles/signal_api/templates/docker-compose.yml.j2 b/ansible/roles/signal_api/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..9825299 --- /dev/null +++ b/ansible/roles/signal_api/templates/docker-compose.yml.j2 @@ -0,0 +1,47 @@ +# Managed by Ansible (roles/signal_api) +services: + {{ signal_api_service_name }}: + image: {{ signal_api_image }} + container_name: {{ signal_api_service_name }} + restart: unless-stopped + + environment: + MODE: "{{ signal_api_mode }}" + + volumes: + # Prefer IPv4. See gai.conf.j2 - without this, signal-cli reaches for + # chat.signal.org's IPv6 address, which is unreachable from here, and + # linking fails with an opaque "network error" on the phone. + - {{ signal_api_dir }}/gai.conf:/etc/gai.conf:ro + # Holds the Signal identity: the linked-device keys and registration + # state. Lose this and the device must be linked again by scanning a new + # QR code from the phone. It is also the most sensitive thing on this + # host - anyone with these keys can send and read Signal as you. + - {{ signal_api_data_dir }}:/home/.local/share/signal-cli + + networks: + - {{ signal_api_network }} + + # NO PORTS. Deliberately. + # + # This API has no authentication whatsoever - no key, no token, nothing. + # Publishing it, even on 127.0.0.1, would expose "send a Signal message as + # this identity" to anything that can reach the host. Gatus talks to it over + # the shared docker network by service name instead, which is why no port is + # published and why there is no Caddy vhost. + + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8080/v1/health"] + interval: 60s + timeout: 5s + retries: 3 + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +networks: + {{ signal_api_network }}: + external: true diff --git a/ansible/roles/signal_api/templates/gai.conf.j2 b/ansible/roles/signal_api/templates/gai.conf.j2 new file mode 100644 index 0000000..9b2cca4 --- /dev/null +++ b/ansible/roles/signal_api/templates/gai.conf.j2 @@ -0,0 +1,17 @@ +# Managed by Ansible (roles/signal_api) +# +# Prefer IPv4 over IPv6 in getaddrinfo. +# +# chat.signal.org resolves to AWS Global Accelerator dualstack addresses, and +# DNS returns the AAAA records first. This container has NO IPv6 address at all, +# and this host's IPv6 path is unreliable anyway - the same edge that made +# Google's IPv6 endpoint return a confident 404 for the Go tarball. +# +# signal-cli would connect to the AAAA address, fail, and report +# Link request error: Connection closed! +# while the phone showed a bare "network error" - a failure with no obvious +# cause on either end. +# +# This line flips the precedence so IPv4-mapped addresses sort first, which is +# the standard glibc fix. It does NOT disable IPv6; it only changes the order. +precedence ::ffff:0:0/96 100 diff --git a/ansible/roles/socket_proxy/README.md b/ansible/roles/socket_proxy/README.md new file mode 100644 index 0000000..cebb95f --- /dev/null +++ b/ansible/roles/socket_proxy/README.md @@ -0,0 +1,62 @@ +# `socket_proxy` + +Exposes a service running on a private Tailscale host through a public TCP port +on an edge machine, using `systemd-socket-proxyd`. Writes a `.socket` and a +`.service` unit, enables the socket, opens the UFW port, and checks the upstream +is reachable. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: fulcrum-ssl # -> fulcrum-ssl-proxy.{socket,service} + socket_proxy_description: "Fulcrum SSL" # -> "Fulcrum SSL Proxy Socket" + socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" +``` + +`socket_proxy_upstream_port` defaults to `socket_proxy_listen_port`, which is +what all three current callers want. + +## Optional unit settings + +These exist because the **live** `bitcoin-p2p-proxy` units on vipy carried +settings the playbook never wrote. Somebody added them by hand, so running +`deploy_bitcoin_knots_playbook.yml` would have silently removed them: + +| Variable | Emits | Why it matters | +|---|---|---| +| `socket_proxy_free_bind` | `FreeBind=true` in `[Socket]` | Lets the socket bind to an address that is not up yet. Without it the socket can fail to start on boot. | +| `socket_proxy_documentation` | `Documentation=` in both units | Cosmetic. | +| `socket_proxy_timeout_stop_sec` | `TimeoutStopSec=` | Bounds how long a stop can hang. | +| `socket_proxy_log_to_journal` | `StandardOutput=journal` + `StandardError=journal` | Cosmetic on modern systemd, which defaults to the journal anyway. | + +Only `bitcoin-p2p` passes any of them. + +## `socket_proxy_ufw_comment` + +Defaults to `" public access"`, which reproduces the live rule +comment for bitcoin-p2p and fulcrum-ssl. **datum-stratum must pass it +explicitly** — its live comment is `DATUM Gateway Stratum public access` while +the derived default would be `DATUM Stratum public access`, and UFW treats the +comment as part of the rule, so the mismatch rewrites the rule on every run. + +## The upstream check never fails the play + +`wait_for` on the upstream carries `failed_when: false`. The proxy is correctly +configured whether or not the backend happens to be up, and this is the one task +that depends on another machine. The original plays used `ignore_errors: yes`, +which prints a red "ignoring" line; `failed_when: false` is the quieter +equivalent. + +## Restarts + +The handler restarts the `.socket`, not the `.service` — that is what picks up a +changed unit; the service is started by the socket on the next connection. + +**Restarting a socket drops connections that are currently open through it.** +For bitcoin-p2p that means peers reconnect; for datum-stratum it means a mining +client has to reconnect and may lose in-flight shares. The handler only fires +when a unit file actually changes. diff --git a/ansible/roles/socket_proxy/defaults/main.yml b/ansible/roles/socket_proxy/defaults/main.yml new file mode 100644 index 0000000..6a512ea --- /dev/null +++ b/ansible/roles/socket_proxy/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# Required +socket_proxy_name: "" # "bitcoin-p2p" -> bitcoin-p2p-proxy.{socket,service} +socket_proxy_description: "" # "Bitcoin P2P" -> "Bitcoin P2P Proxy Socket" +socket_proxy_listen_port: 0 # public port on the edge host +socket_proxy_upstream_host: "" # Tailscale hostname, e.g. "knots-box" + +# Optional +socket_proxy_upstream_port: "" # defaults to socket_proxy_listen_port +socket_proxy_documentation: "" # Documentation= in both units +socket_proxy_free_bind: false # FreeBind=true: bind before the address is up +socket_proxy_timeout_stop_sec: "" # TimeoutStopSec= +socket_proxy_log_to_journal: false # StandardOutput/StandardError=journal + +# Firewall +socket_proxy_ufw_proto: tcp +socket_proxy_ufw_comment: "" # defaults to " public access" diff --git a/ansible/roles/socket_proxy/handlers/main.yml b/ansible/roles/socket_proxy/handlers/main.yml new file mode 100644 index 0000000..7fdf421 --- /dev/null +++ b/ansible/roles/socket_proxy/handlers/main.yml @@ -0,0 +1,8 @@ +--- +# Restarting the .socket is what picks up a changed unit; the .service is +# started by the socket on the next connection. +- name: Restart socket proxy + ansible.builtin.systemd: + name: "{{ socket_proxy_name }}-proxy.socket" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/socket_proxy/tasks/main.yml b/ansible/roles/socket_proxy/tasks/main.yml new file mode 100644 index 0000000..4315555 --- /dev/null +++ b/ansible/roles/socket_proxy/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Assert socket_proxy parameters are sane + ansible.builtin.assert: + that: + - socket_proxy_name | length > 0 + - socket_proxy_description | length > 0 + - socket_proxy_listen_port | int > 0 + - socket_proxy_upstream_host | length > 0 + fail_msg: >- + socket_proxy: '{{ socket_proxy_name | default("") }}' needs a name, + a description, a listen port and an upstream host. + quiet: true + +- name: "Create the {{ socket_proxy_name }}-proxy socket unit" + ansible.builtin.template: + src: proxy.socket.j2 + dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.socket" + owner: root + group: root + mode: '0644' + notify: Restart socket proxy + +- name: "Create the {{ socket_proxy_name }}-proxy service unit" + ansible.builtin.template: + src: proxy.service.j2 + dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.service" + owner: root + group: root + mode: '0644' + notify: Restart socket proxy + +- name: "Enable and start the {{ socket_proxy_name }}-proxy socket" + ansible.builtin.systemd: + name: "{{ socket_proxy_name }}-proxy.socket" + enabled: yes + state: started + daemon_reload: yes + +- name: "Allow the {{ socket_proxy_name }} port through UFW" + community.general.ufw: + rule: allow + port: "{{ socket_proxy_listen_port | string }}" + proto: "{{ socket_proxy_ufw_proto }}" + comment: "{{ socket_proxy_ufw_comment | default(socket_proxy_description ~ ' public access', true) }}" + +# Reachability of the upstream over Tailscale. Deliberately non-fatal: the proxy +# is still correctly configured if the backend happens to be down, and this is +# the one check that depends on another machine being up. +- name: "Verify {{ socket_proxy_upstream_host }} is reachable over Tailscale" + ansible.builtin.wait_for: + host: "{{ socket_proxy_upstream_host }}" + port: "{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }}" + timeout: 10 + failed_when: false diff --git a/ansible/roles/socket_proxy/templates/proxy.service.j2 b/ansible/roles/socket_proxy/templates/proxy.service.j2 new file mode 100644 index 0000000..e59d251 --- /dev/null +++ b/ansible/roles/socket_proxy/templates/proxy.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description={{ socket_proxy_description }} Proxy to {{ socket_proxy_upstream_host }} +{% if socket_proxy_documentation %} +Documentation={{ socket_proxy_documentation }} +{% endif %} +Requires={{ socket_proxy_name }}-proxy.socket +After=network.target + +[Service] +Type=notify +ExecStart=/lib/systemd/systemd-socket-proxyd {{ socket_proxy_upstream_host }}:{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }} +{% if socket_proxy_timeout_stop_sec %} +TimeoutStopSec={{ socket_proxy_timeout_stop_sec }} +{% endif %} +{% if socket_proxy_log_to_journal %} +StandardOutput=journal +StandardError=journal +{% endif %} diff --git a/ansible/roles/socket_proxy/templates/proxy.socket.j2 b/ansible/roles/socket_proxy/templates/proxy.socket.j2 new file mode 100644 index 0000000..0dc1721 --- /dev/null +++ b/ansible/roles/socket_proxy/templates/proxy.socket.j2 @@ -0,0 +1,14 @@ +[Unit] +Description={{ socket_proxy_description }} Proxy Socket +{% if socket_proxy_documentation %} +Documentation={{ socket_proxy_documentation }} +{% endif %} + +[Socket] +ListenStream={{ socket_proxy_listen_port }} +{% if socket_proxy_free_bind %} +FreeBind=true +{% endif %} + +[Install] +WantedBy=sockets.target diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml deleted file mode 100644 index c9bd7ca..0000000 --- a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Bitcoin Knots Configuration Variables - -# Version - REQUIRED: Specify exact version/tag to build -bitcoin_knots_version: "v29.2.knots20251110" # Must specify exact version/tag -bitcoin_knots_version_short: "29.2.knots20251110" # Version without 'v' prefix (for tarball URLs) - -# Directories -bitcoin_knots_dir: /opt/bitcoin-knots -bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" -bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets -bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) -bitcoin_conf_dir: /etc/bitcoin - -# Network -bitcoin_rpc_port: 8332 -bitcoin_p2p_port: 8333 -bitcoin_rpc_bind: "0.0.0.0" - -# Build options -bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores -bitcoin_build_prefix: /usr/local - -# Configuration options -bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) -bitcoin_max_connections: 125 -# dbcache will be calculated as 90% of host RAM automatically in playbook - -# ZMQ Configuration -bitcoin_zmq_enabled: true -bitcoin_zmq_bind: "tcp://0.0.0.0" -bitcoin_zmq_port_rawblock: 28332 -bitcoin_zmq_port_rawtx: 28333 -bitcoin_zmq_port_hashblock: 28334 -bitcoin_zmq_port_hashtx: 28335 - -# Service user -bitcoin_user: bitcoin -bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index d818945..d67006b 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -1,916 +1,35 @@ +--- +# Bitcoin Knots: full node built from source, with PGP signature and SHA256 +# verification of the release tarball. The build is guarded by a binary-exists +# check, so a converged host skips the whole 30-60 minute compile. +# +# The chain lives in bitcoin_large_data_dir (/mnt/knots_data, ~875 GB). Nothing +# here touches it; the only `state: absent` tasks target the build tree under +# /opt/bitcoin-knots and run only when the binary is missing. - name: Build and Deploy Bitcoin Knots from Source - hosts: knots_box_local + hosts: bitcoin become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./bitcoin_knots_vars.yml vars: - bitcoin_repo_url: "https://github.com/bitcoinknots/bitcoin.git" - bitcoin_sigs_base_url: "https://raw.githubusercontent.com/bitcoinknots/guix.sigs/knots" - bitcoin_version_major: "{{ bitcoin_knots_version_short | regex_replace('^(\\d+)\\..*', '\\1') }}" - bitcoin_source_tarball_url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Calculate 90% of system RAM for dbcache - set_fact: - bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" - changed_when: false - - - name: Display calculated dbcache value - debug: - msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" - - - - name: Install build dependencies - apt: - name: - - build-essential - - libtool - - autotools-dev - - automake - - pkg-config - - bsdmainutils - - python3 - - python3-pip - - libevent-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-test-dev - - libboost-thread-dev - - libboost-chrono-dev - - libboost-program-options-dev - - libboost-dev - - libssl-dev - - libdb-dev - - libminiupnpc-dev - - libzmq3-dev - - libnatpmp-dev - - libsqlite3-dev - - git - - curl - - wget - - cmake - state: present - update_cache: yes - - - name: Create bitcoin group - group: - name: "{{ bitcoin_group }}" - system: yes - state: present - - - name: Create bitcoin user - user: - name: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ bitcoin_data_dir }}" - create_home: yes - state: present - - - name: Create bitcoin-knots directory - file: - path: "{{ bitcoin_knots_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create bitcoin-knots source directory - file: - path: "{{ bitcoin_knots_source_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create bitcoin data directory (for config, logs, wallets) - file: - path: "{{ bitcoin_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - - - name: Create bitcoin large data directory (for blockchain) - file: - path: "{{ bitcoin_large_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - - - name: Create bitcoin config directory - file: - path: "{{ bitcoin_conf_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Check if bitcoind binary already exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_binary_exists - changed_when: false - - - name: Install gnupg for signature verification - apt: - name: gnupg - state: present - when: not bitcoind_binary_exists.stat.exists - - - name: Import Luke Dashjr's Bitcoin Knots signing key - command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 - register: key_import - changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" - when: not bitcoind_binary_exists.stat.exists - failed_when: key_import.rc != 0 - - - name: Display imported key fingerprint - command: gpg --fingerprint 90C8019E36C2E964 - register: key_fingerprint - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Download SHA256SUMS file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - - - name: Download SHA256SUMS.asc signature file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - - - name: Verify PGP signature on SHA256SUMS file - command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS - register: sha256sums_verification - changed_when: false - failed_when: false # Don't fail here - check for 'Good signature' in next task - when: not bitcoind_binary_exists.stat.exists - - - - name: Display SHA256SUMS verification result - debug: - msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" - when: not bitcoind_binary_exists.stat.exists - - - name: Fail if SHA256SUMS signature verification failed - fail: - msg: "SHA256SUMS signature verification failed. Aborting build." - when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) - - - name: Remove any existing tarball to force fresh download - file: - path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Download Bitcoin Knots source tarball - get_url: - url: "{{ bitcoin_source_tarball_url }}" - dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - mode: '0644' - validate_certs: yes - force: yes - when: not bitcoind_binary_exists.stat.exists - - - name: Calculate SHA256 checksum of downloaded tarball - command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - register: tarball_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Extract expected checksum from SHA256SUMS file - shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' - register: expected_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - failed_when: expected_checksum.stdout == "" - - - name: Display checksum comparison - debug: - msg: - - "Expected: {{ expected_checksum.stdout | trim }}" - - "Actual: {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists - - - name: Verify tarball checksum matches SHA256SUMS - fail: - msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] - - - name: Remove existing source directory if it exists (to force fresh extraction) - file: - path: "{{ bitcoin_knots_source_dir }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Remove extracted directory if it exists (from previous runs) - file: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Extract verified source tarball - unarchive: - src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - dest: "{{ bitcoin_knots_dir }}" - remote_src: yes - when: not bitcoind_binary_exists.stat.exists - - - name: Check if extracted directory exists - stat: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - register: extracted_dir_stat - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Rename extracted directory to expected name - command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" - when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists - - - name: Check if CMakeLists.txt exists - stat: - path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" - register: cmake_exists - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Create CMake build directory - file: - path: "{{ bitcoin_knots_source_dir }}/build" - state: directory - mode: '0755' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - - - name: Configure Bitcoin Knots build with CMake - command: > - cmake - -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} - -DBUILD_BITCOIN_WALLET=OFF - -DCMAKE_BUILD_TYPE=Release - -DWITH_ZMQ=ON - .. - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: configure_result - changed_when: true - - - name: Verify CMake enabled ZMQ - shell: | - set -e - cd "{{ bitcoin_knots_source_dir }}/build" - cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: zmq_check - changed_when: false - - - name: Fail if CMakeLists.txt not found - fail: - msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." - when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) - - - name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) - command: cmake --build . -j{{ bitcoin_build_jobs }} - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - async: 3600 - poll: 0 - register: build_result - changed_when: true - - - name: Check build status - async_status: - jid: "{{ build_result.ansible_job_id }}" - register: build_job_result - until: build_job_result.finished - retries: 120 - delay: 60 - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined - - - name: Fail if build failed - fail: - msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) - - - name: Install Bitcoin Knots binaries - command: cmake --install . - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - changed_when: true - - - name: Verify bitcoind binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_installed - changed_when: false - - - name: Verify bitcoin-cli binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" - register: bitcoin_cli_installed - changed_when: false - - - name: Fail if binaries not found - fail: - msg: "Bitcoin Knots binaries not found after installation" - when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists - - - name: Create bitcoin.conf configuration file - copy: - dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" - content: | - # Bitcoin Knots Configuration - # Generated by Ansible - - # Data directory (blockchain storage) - datadir={{ bitcoin_large_data_dir }} - - # RPC Configuration - server=1 - rpcuser={{ bitcoin_rpc_user }} - rpcpassword={{ bitcoin_rpc_password }} - rpcbind={{ bitcoin_rpc_bind }} - rpcport={{ bitcoin_rpc_port }} - rpcallowip=0.0.0.0/0 - - # Network Configuration - listen=1 - port={{ bitcoin_p2p_port }} - maxconnections={{ bitcoin_max_connections }} - - # Performance - dbcache={{ bitcoin_dbcache_mb }} - - # Transaction Index (optional) - {% if bitcoin_enable_txindex %} - txindex=1 - {% endif %} - - # Logging (to journald via systemd) - logtimestamps=1 - printtoconsole=1 - - # ZMQ Configuration - {% if bitcoin_zmq_enabled | default(false) %} - zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} - zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} - zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} - zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} - {% endif %} - - # Security - disablewallet=1 - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0640' - notify: Restart bitcoind - - - name: Create systemd service file for bitcoind - copy: - dest: /etc/systemd/system/bitcoind.service - content: | - [Unit] - Description=Bitcoin Knots daemon - After=network.target - - [Service] - Type=simple - User={{ bitcoin_user }} - Group={{ bitcoin_group }} - ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf - Restart=always - RestartSec=10 - TimeoutStopSec=600 - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoind - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start bitcoind service - systemd: - name: bitcoind - enabled: yes - state: started - - - name: Wait for bitcoind RPC to be available - uri: - url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" - method: POST - body_format: json - body: - jsonrpc: "1.0" - id: "healthcheck" - method: "getblockchaininfo" - params: [] - user: "{{ bitcoin_rpc_user }}" - password: "{{ bitcoin_rpc_password }}" - status_code: 200 - timeout: 10 - register: rpc_check - until: rpc_check.status == 200 - retries: 30 - delay: 5 - ignore_errors: yes - - - name: Display RPC connection status - debug: - msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" - - - name: Create Bitcoin Knots health check and push script - copy: - dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh - content: | - #!/bin/bash - # - # Bitcoin Knots Health Check and Push to Uptime Kuma - # Checks if bitcoind RPC is responding and pushes status to Uptime Kuma - # - - RPC_HOST="{{ bitcoin_rpc_bind }}" - RPC_PORT={{ bitcoin_rpc_port }} - RPC_USER="{{ bitcoin_rpc_user }}" - RPC_PASSWORD="{{ bitcoin_rpc_password }}" - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - # Check if bitcoind RPC is responding - check_bitcoind() { - local response - response=$(curl -s --max-time 30 \ - --user "${RPC_USER}:${RPC_PASSWORD}" \ - --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ - --header 'Content-Type: application/json' \ - "http://${RPC_HOST}:${RPC_PORT}" 2>&1) - - if [ $? -eq 0 ]; then - # Check if response contains a non-null error - # Successful responses have "error": null, failures have "error": {...} - if echo "$response" | grep -q '"error":null\|"error": null'; then - return 0 - else - return 1 - fi - else - return 1 - fi - } - - # Push status to Uptime Kuma - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - if ! curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then - echo "ERROR: Failed to push to Uptime Kuma" - return 1 - fi - } - - # Main health check - if check_bitcoind; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "bitcoind RPC not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Install curl for health check script - apt: - name: curl - state: present - - - name: Create systemd timer for Bitcoin Knots health check - copy: - dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer - content: | - [Unit] - Description=Bitcoin Knots Health Check Timer - Requires=bitcoind.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for Bitcoin Knots health check - copy: - dest: /etc/systemd/system/bitcoin-knots-healthcheck.service - content: | - [Unit] - Description=Bitcoin Knots Health Check and Push to Uptime Kuma - After=network.target bitcoind.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin Knots health check timer - systemd: - name: bitcoin-knots-healthcheck.timer - enabled: yes - state: started - - - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_bitcoin_knots_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for new monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "Bitcoin Knots" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_bitcoin_knots_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Extract push URL from monitor setup output - set_fact: - uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" - delegate_to: localhost - become: no - when: monitor_setup.stdout is defined - - - name: Display extracted push URL - debug: - msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" - when: uptime_kuma_push_url | default('') != '' - - - name: Set push URL in systemd service environment - lineinfile: - path: /etc/systemd/system/bitcoin-knots-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" - state: present - insertafter: '^\[Service\]' - when: uptime_kuma_push_url | default('') != '' - - - name: Reload systemd daemon after push URL update - systemd: - daemon_reload: yes - when: uptime_kuma_push_url | default('') != '' - - - name: Restart health check timer to pick up new environment - systemd: - name: bitcoin-knots-healthcheck.timer - state: restarted - when: uptime_kuma_push_url | default('') != '' - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_bitcoin_knots_monitor.py - - /tmp/ansible_config.yml - - handlers: - - name: Restart bitcoind - systemd: - name: bitcoind - state: restarted - - -- name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd - hosts: vipy + # Preserves the push URL this check has been reporting to. The role knows + # nothing about Uptime Kuma — this is just "a URL that accepts a ping". + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_bitcoin-knots/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - bitcoin_knots + +- name: Setup public Bitcoin P2P forwarding on the edge host + hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./bitcoin_knots_vars.yml - vars: - bitcoin_tailscale_hostname: "knots-box" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - - name: Create Bitcoin P2P proxy socket unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.socket - content: | - [Unit] - Description=Bitcoin P2P Proxy Socket - - [Socket] - ListenStream={{ bitcoin_p2p_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoin-p2p-proxy socket - - - name: Create Bitcoin P2P proxy service unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.service - content: | - [Unit] - Description=Bitcoin P2P Proxy to {{ bitcoin_tailscale_hostname }} - Requires=bitcoin-p2p-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ bitcoin_tailscale_hostname }}:{{ bitcoin_p2p_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin P2P proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - enabled: yes - state: started - - - name: Allow Bitcoin P2P port through UFW - ufw: - rule: allow - port: "{{ bitcoin_p2p_port | string }}" - proto: tcp - comment: "Bitcoin P2P public access" - - - name: Verify connectivity to knots-box via Tailscale - wait_for: - host: "{{ bitcoin_tailscale_hostname }}" - port: "{{ bitcoin_p2p_port }}" - timeout: 10 - ignore_errors: yes - - - name: Display public endpoint - debug: - msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public P2P - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_bitcoin_p2p_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_bitcoin_p2p_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for TCP monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_bitcoin_p2p_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ bitcoin_p2p_port }} - monitor_name: "Bitcoin Knots P2P Public" - mode: '0644' - - - name: Run Uptime Kuma TCP monitor setup - command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up TCP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_bitcoin_p2p_tcp_monitor.py - - /tmp/ansible_bitcoin_p2p_config.yml - - handlers: - - name: Restart bitcoin-p2p-proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - state: restarted + - name: Expose Bitcoin P2P through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: bitcoin-p2p + socket_proxy_description: "Bitcoin P2P" + socket_proxy_listen_port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}" + socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" + socket_proxy_documentation: "https://github.com/bitcoin/bitcoin" + socket_proxy_free_bind: true + socket_proxy_timeout_stop_sec: 5 + socket_proxy_log_to_journal: true diff --git a/ansible/services/caddy_playbook.yml b/ansible/services/caddy_playbook.yml index de98c8f..29e74b2 100644 --- a/ansible/services/caddy_playbook.yml +++ b/ansible/services/caddy_playbook.yml @@ -1,5 +1,5 @@ - name: Install and configure Caddy on Debian 12 - hosts: vps + hosts: caddy become: yes tasks: diff --git a/ansible/services/datum-gateway/datum_gateway_vars.yml b/ansible/services/datum-gateway/datum_gateway_vars.yml deleted file mode 100644 index 48bb5b7..0000000 --- a/ansible/services/datum-gateway/datum_gateway_vars.yml +++ /dev/null @@ -1,43 +0,0 @@ -# DATUM Gateway Configuration Variables -# https://github.com/OCEAN-xyz/datum_gateway - -# Version - pin to a specific tag -datum_gateway_version: "v0.4.1beta" - -# Directories -datum_gateway_dir: /opt/datum-gateway -datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" -datum_gateway_config_dir: /etc/datum-gateway -datum_gateway_log_dir: /var/log/datum-gateway - -# Binary -datum_gateway_bin_path: /usr/local/bin/datum_gateway - -# Ports -datum_gateway_stratum_port: 23334 # Miners connect here via Stratum v1 -datum_gateway_api_port: 7152 # Web dashboard / API - -# Stratum settings -datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) - -# Service user -datum_gateway_user: datum -datum_gateway_group: datum - -# Build options -datum_gateway_build_jobs: 4 - -# Bitcoin node connection -# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. -# datum_bitcoin_rpc_url should include http:// and port. -datum_bitcoin_rpc_url: "http://127.0.0.1:8332" -# Note: bitcoin_rpc_user and bitcoin_rpc_password come from infra_secrets.yml - -# Mining config -datum_coinbase_tag_primary: "DATUM" -datum_coinbase_tag_secondary: "BY ORDER OF BIP110" - -datum_pool_pass_workers: true -datum_pool_pass_full_users: true -datum_pooled_mining_only: true - diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index f33ded7..2e88a27 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -1,859 +1,58 @@ --- -# DATUM Gateway Deployment Playbook +# DATUM Gateway: solo/pooled mining gateway, built from source on knots-box. # -# Deploys DATUM Gateway (https://github.com/OCEAN-xyz/datum_gateway) on the -# Bitcoin Knots host so it has direct localhost RPC access to bitcoind. -# -# What this does: -# 1. Installs build deps and compiles datum_gateway from source -# 2. Creates a dedicated system user and config/log directories -# 3. Writes /etc/datum-gateway/config.json from vars/secrets -# 4. Patches bitcoin.conf with the required blockmaxsize/blocknotify lines -# 5. Creates and enables a systemd service -# 6. Creates a push-monitor health check script + systemd timer -# 7. Registers a push monitor in Uptime Kuma -# -# Separate play: adds a Caddy reverse proxy on vipy for the dashboard. -# -# Stratum port (23334) is bound on knots_box_local. Expose it to miners via -# a firewall rule, Tailscale, or a socket proxy on vipy — not handled here. -# -# Required secrets in infra_secrets.yml: -# datum_mining_address - Bitcoin address for block rewards -# datum_gateway_admin_password - Password for the /api admin endpoint -# bitcoin_rpc_user - Shared with the bitcoin-knots deployment -# bitcoin_rpc_password - Shared with the bitcoin-knots deployment - -- name: Deploy DATUM Gateway on knots_box_local - hosts: knots_box_local +# This is HALF OF A SYSTEM. The Bitcoin Knots node on the same host feeds it via +# `blocknotify=killall -USR1 datum_gateway` in bitcoin.conf (see +# roles/bitcoin_knots/README.md). Changing either config means thinking about +# both. Interrupting Stratum costs mining shares, so check for connected miners +# before restarting: +# ss -tn state established '( sport = :23334 )' +- name: Deploy DATUM Gateway on the bitcoin host + hosts: bitcoin become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./datum_gateway_vars.yml vars: - datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" - datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + # Preserves the push URL this check reports to. The role knows nothing about + # Uptime Kuma — this is just "a URL that accepts a ping". + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_datum-gateway/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - datum_gateway - tasks: - # =========================================== - # Build Dependencies - # =========================================== - - name: Install DATUM Gateway build dependencies - apt: - name: - - cmake - - build-essential - - git - - libjansson-dev - - libmicrohttpd-dev - - libsodium-dev - - libcurl4-openssl-dev - # Runtime-only (netcat for health check) - - netcat-openbsd - state: present - update_cache: yes - - # =========================================== - # System User and Directories - # =========================================== - - name: Create datum system user - user: - name: "{{ datum_gateway_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ datum_gateway_dir }}" - create_home: no - comment: "DATUM Gateway" - - - name: Create DATUM Gateway directories - file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ datum_gateway_group }}" - mode: "{{ item.mode }}" - loop: - - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - - # =========================================== - # Build from Source - # =========================================== - - name: Clone DATUM Gateway repository at {{ datum_gateway_version }} - git: - repo: https://github.com/OCEAN-xyz/datum_gateway.git - dest: "{{ datum_gateway_source_dir }}" - version: "{{ datum_gateway_version }}" - force: yes - register: git_clone - - - name: Configure cmake build - command: cmake . -DCMAKE_BUILD_TYPE=Release - args: - chdir: "{{ datum_gateway_source_dir }}" - - - name: Compile datum_gateway - command: make -j{{ datum_gateway_build_jobs }} - args: - chdir: "{{ datum_gateway_source_dir }}" - - - name: Install datum_gateway binary - copy: - src: "{{ datum_gateway_source_dir }}/datum_gateway" - dest: "{{ datum_gateway_bin_path }}" - remote_src: yes - owner: root - group: root - mode: "0755" - notify: Restart datum-gateway - - # =========================================== - # Configuration - # =========================================== - - name: Write DATUM Gateway config.json - copy: - dest: "{{ datum_gateway_config_dir }}/config.json" - content: | - { - "bitcoind": { - "rpcuser": "{{ bitcoin_rpc_user }}", - "rpcpassword": "{{ bitcoin_rpc_password }}", - "rpcurl": "{{ datum_bitcoin_rpc_url }}", - "notify_fallback": true - }, - "stratum": { - "listen_port": {{ datum_gateway_stratum_port }}, - "vardiff_min": {{ datum_vardiff_min }} - }, - "mining": { - "pool_address": "{{ datum_mining_address }}", - "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", - "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" - }, - "api": { - "admin_password": "{{ datum_gateway_admin_password }}", - "listen_port": {{ datum_gateway_api_port }}, - "modify_conf": false - }, - "logger": { - "log_to_console": true, - "log_to_file": true, - "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", - "log_rotate_daily": true, - "log_level_console": 2, - "log_level_file": 1 - }, - "datum": { - "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, - "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, - "pooled_mining_only": {{ datum_pooled_mining_only | lower }} - } - } - owner: "{{ datum_gateway_user }}" - group: "{{ datum_gateway_group }}" - mode: "0640" - notify: Restart datum-gateway - - # =========================================== - # Systemd Service - # =========================================== - - name: Create datum-gateway systemd service - copy: - dest: /etc/systemd/system/datum-gateway.service - content: | - [Unit] - Description=DATUM Gateway - Bitcoin Mining Gateway - Documentation=https://github.com/OCEAN-xyz/datum_gateway - After=network.target bitcoind.service - Wants=bitcoind.service - - [Service] - User={{ datum_gateway_user }} - Group={{ datum_gateway_group }} - Type=simple - ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json - Restart=on-failure - RestartSec=10 - StandardOutput=journal - StandardError=journal - - # Prevent config from being read by other users - ReadWritePaths={{ datum_gateway_log_dir }} - ReadOnlyPaths={{ datum_gateway_config_dir }} - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - notify: Restart datum-gateway - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start datum-gateway - systemd: - name: datum-gateway - enabled: yes - state: started - - # =========================================== - # Health Check Script + Systemd Timer - # =========================================== - - name: Create DATUM Gateway health check script - copy: - dest: /usr/local/bin/datum-gateway-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - STRATUM_PORT={{ datum_gateway_stratum_port }} - - check_datum() { - # Service must be active and stratum port must be listening - systemctl is-active --quiet datum-gateway && \ - nc -z 127.0.0.1 "${STRATUM_PORT}" - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_datum; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "DATUM Gateway not responding" - exit 1 - fi - owner: root - group: root - mode: "0755" - - - name: Create datum-gateway health check systemd service - copy: - dest: /etc/systemd/system/datum-gateway-healthcheck.service - content: | - [Unit] - Description=DATUM Gateway Health Check - After=network.target datum-gateway.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - - - name: Create datum-gateway health check systemd timer - copy: - dest: /etc/systemd/system/datum-gateway-healthcheck.timer - content: | - [Unit] - Description=DATUM Gateway Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - - - name: Enable and start datum-gateway health check timer - systemd: - name: datum-gateway-healthcheck.timer - enabled: yes - state: started - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for DATUM Gateway - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_datum_gateway_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - - try: - import socketio.exceptions - except ImportError: - pass - - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_datum_gateway_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - push_url = None - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - else: - print(f"Creating push monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - # socketio timeout: add_monitor may have succeeded server-side - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - - api.disconnect() - - if push_url: - print(f"PUSH_URL={push_url}") - with open('/tmp/datum_gateway_push_url.txt', 'w') as f: - f.write(push_url) - - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for push monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_datum_gateway_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "DATUM Gateway" - mode: "0644" - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_datum_gateway_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URL from file - slurp: - src: /tmp/datum_gateway_push_url.txt - delegate_to: localhost - become: no - register: push_url_file - ignore_errors: yes - - - name: Parse push URL - set_fact: - datum_push_url: "{{ push_url_file.content | b64decode | trim }}" - when: push_url_file.content is defined - - - name: Update health check service with push URL - lineinfile: - path: /etc/systemd/system/datum-gateway-healthcheck.service - regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" - line: "Environment=UPTIME_KUMA_PUSH_URL={{ datum_push_url }}" - when: datum_push_url is defined - notify: Restart datum-gateway health check timer - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_datum_gateway_monitor.py - - /tmp/ansible_datum_gateway_config.yml - - /tmp/datum_gateway_push_url.txt - - handlers: - - name: Restart datum-gateway - systemd: - name: datum-gateway - state: restarted - daemon_reload: yes - - - name: Restart datum-gateway health check timer - systemd: - name: datum-gateway-healthcheck.timer - state: restarted - daemon_reload: yes - - -# =========================================== -# Caddy Reverse Proxy for DATUM Dashboard (on vipy) -# =========================================== -- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on vipy - hosts: vipy +- name: Configure Caddy reverse proxy for the DATUM Gateway dashboard on the edge host + hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./datum_gateway_vars.yml - vars: - datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" - datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: "0755" + - name: Publish the DATUM Gateway dashboard through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: datum-gateway + caddy_site_domain: "{{ subdomains.datum_gateway }}.{{ root_domain }}" + caddy_site_upstream: "{{ hostvars['knots_box_local'].ansible_host }}:{{ hostvars['knots_box_local'].datum_gateway_api_port }}" + caddy_site_resolvers: "100.100.100.100" + caddy_site_basic_auth: + - user: "{{ datum_dashboard_username }}" + hash: "{{ datum_dashboard_password_hash }}" - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: "import sites-enabled/*" - insertafter: EOF - state: present - backup: yes - create: yes - mode: "0644" - - - name: Create Caddy reverse proxy config for DATUM Gateway dashboard - copy: - dest: "{{ caddy_sites_dir }}/datum-gateway.conf" - content: | - {{ datum_gateway_domain }} { - basic_auth { - {{ datum_dashboard_username }} {{ datum_dashboard_password_hash }} - } - reverse_proxy knots-box:{{ datum_gateway_api_port }} { - # Resolve via Tailscale MagicDNS - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: "0644" - - - name: Validate Caddy config - command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile - register: caddy_validate + # The role validates its own site fragment; this re-validates the whole + # assembled Caddyfile, which is the only thing that catches a conflict + # between two sites. + - name: Validate the assembled Caddyfile + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile changed_when: false - - name: Display Caddy validation output - debug: - msg: "{{ caddy_validate.stdout_lines + caddy_validate.stderr_lines }}" - - - name: Reload Caddy - command: systemctl reload caddy - register: caddy_reload - - - name: Display Caddy reload output - debug: - msg: "{{ caddy_reload.stdout_lines + caddy_reload.stderr_lines }}" - - - name: Display DATUM Gateway dashboard URL - debug: - msg: "DATUM Gateway dashboard: https://{{ datum_gateway_domain }}" - - # =========================================== - # Uptime Kuma HTTP Monitor for Public Dashboard - # =========================================== - - name: Create Uptime Kuma HTTP monitor setup script for DATUM dashboard - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_datum_http_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_datum_http_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - else: - print(f"Creating HTTP monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for HTTP monitor - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_datum_http_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ datum_gateway_domain }}" - monitor_name: "DATUM Gateway Dashboard" - mode: "0644" - - - name: Run Uptime Kuma HTTP monitor setup - command: python3 /tmp/setup_datum_http_monitor.py - delegate_to: localhost - become: no - register: http_monitor_setup - changed_when: "'SUCCESS' in http_monitor_setup.stdout" - ignore_errors: yes - - - name: Display HTTP monitor setup output - debug: - msg: "{{ http_monitor_setup.stdout_lines }}" - when: http_monitor_setup.stdout is defined - - - name: Clean up HTTP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_datum_http_monitor.py - - /tmp/ansible_datum_http_config.yml - - -# =========================================== -# Stratum Port Forwarding on vipy via systemd-socket-proxyd -# Miners connect to vipy:23334; traffic is forwarded to knots-box:23334 -# over the Tailscale network, matching the Bitcoin P2P proxy pattern. -# =========================================== -- name: Setup public Stratum port forwarding on vipy via systemd-socket-proxyd - hosts: vipy +- name: Setup public Stratum port forwarding on the edge host + hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./datum_gateway_vars.yml - vars: - datum_tailscale_hostname: "knots-box" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - - name: Create Stratum proxy socket unit - copy: - dest: /etc/systemd/system/datum-stratum-proxy.socket - content: | - [Unit] - Description=DATUM Stratum Proxy Socket - - [Socket] - ListenStream={{ datum_gateway_stratum_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: "0644" - notify: Restart datum-stratum-proxy socket - - - name: Create Stratum proxy service unit - copy: - dest: /etc/systemd/system/datum-stratum-proxy.service - content: | - [Unit] - Description=DATUM Stratum Proxy to {{ datum_tailscale_hostname }} - Requires=datum-stratum-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ datum_tailscale_hostname }}:{{ datum_gateway_stratum_port }} - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Stratum proxy socket - systemd: - name: datum-stratum-proxy.socket - enabled: yes - state: started - - - name: Allow Stratum port through UFW - ufw: - rule: allow - port: "{{ datum_gateway_stratum_port | string }}" - proto: tcp - comment: "DATUM Gateway Stratum public access" - - - name: Verify connectivity to knots-box Stratum via Tailscale - wait_for: - host: "{{ datum_tailscale_hostname }}" - port: "{{ datum_gateway_stratum_port }}" - timeout: 10 - ignore_errors: yes - - - name: Display public Stratum endpoint - debug: - msg: "DATUM Stratum public endpoint: {{ ansible_host }}:{{ datum_gateway_stratum_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public Stratum - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Stratum - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_datum_stratum_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_datum_stratum_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for Stratum TCP monitor - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_datum_stratum_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ datum_gateway_stratum_port }} - monitor_name: "DATUM Stratum (public)" - mode: "0644" - - - name: Run Uptime Kuma TCP monitor setup - command: python3 /tmp/setup_datum_stratum_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up Stratum TCP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_datum_stratum_tcp_monitor.py - - /tmp/ansible_datum_stratum_config.yml - - handlers: - - name: Restart datum-stratum-proxy socket - systemd: - name: datum-stratum-proxy.socket - state: restarted + - name: Expose the DATUM Stratum port through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: datum-stratum + socket_proxy_description: "DATUM Stratum" + socket_proxy_listen_port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}" + socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" + # Matches the UFW comment already on the edge host; the derived default + # would say "DATUM Stratum" and rewrite the rule. + socket_proxy_ufw_comment: "DATUM Gateway Stratum public access" diff --git a/ansible/services/forgejo-runner/SETUP.md b/ansible/services/forgejo-runner/SETUP.md index a66d295..5cb2a3f 100644 --- a/ansible/services/forgejo-runner/SETUP.md +++ b/ansible/services/forgejo-runner/SETUP.md @@ -25,4 +25,6 @@ ansible-playbook ansible/services/forgejo-runner/deploy_forgejo_runner_playbook. 1. On the VM: `systemctl status forgejo-runner` should show active 2. In Forgejo: **Site Administration** > **Actions** > **Runners** should show the runner as online -3. In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings +3. ~~In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings~~ + *(Uptime Kuma was decommissioned 2026-09-11 — this check no longer applies. The + healthcheck timer still runs on the host but pushes nowhere. See `archive/uptime_kuma/`.)* diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index a194178..04081ff 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -1,392 +1,13 @@ +--- - name: Install Forgejo Runner on Debian 13 - hosts: forgejo_runner_local + hosts: ci_runner become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./forgejo_runner_vars.yml vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - healthcheck_interval_seconds: 60 - healthcheck_timeout_seconds: 90 - healthcheck_retries: 1 - healthcheck_script_dir: /opt/forgejo-runner-healthcheck - healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" - healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" - healthcheck_service_name: forgejo-runner-healthcheck - - tasks: - # ── 1. Assert Docker is available ────────────────────────────────── - - name: Check if Docker is installed - command: docker --version - register: docker_check - changed_when: false - failed_when: docker_check.rc != 0 - - - name: Fail if Docker is not available - assert: - that: - - docker_check.rc == 0 - fail_msg: > - Docker is not installed or not in PATH. - Please install Docker before running this playbook. - - # ── 2. Download forgejo-runner binary ────────────────────────────── - - name: Download forgejo-runner binary - get_url: - url: "{{ forgejo_runner_url }}" - dest: "{{ forgejo_runner_bin_path }}" - mode: '0755' - - # ── 3. Create runner system user ─────────────────────────────────── - - name: Create runner system user - user: - name: "{{ forgejo_runner_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ forgejo_runner_dir }}" - create_home: no - groups: docker - append: yes - comment: 'Forgejo Runner' - - # ── 4. Create working directory ──────────────────────────────────── - - name: Create forgejo-runner working directory - file: - path: "{{ forgejo_runner_dir }}" - state: directory - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - mode: '0750' - - # ── 5. Generate default config ───────────────────────────────────── - - name: Check if config already exists - stat: - path: "{{ forgejo_runner_config_path }}" - register: config_stat - - - name: Generate default config - shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not config_stat.stat.exists - - - name: Set config file ownership - file: - path: "{{ forgejo_runner_config_path }}" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not config_stat.stat.exists - - # ── 6. Register runner ───────────────────────────────────────────── - - name: Check if runner is already registered - stat: - path: "{{ forgejo_runner_dir }}/.runner" - register: runner_stat - - - name: Register runner with Forgejo instance - command: > - {{ forgejo_runner_bin_path }} register --no-interactive - --instance {{ forgejo_instance_url }} - --token {{ forgejo_runner_registration_token }} - --name forgejo-runner-box - --labels "{{ forgejo_runner_labels }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not runner_stat.stat.exists - - - name: Set runner registration file ownership - file: - path: "{{ forgejo_runner_dir }}/.runner" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not runner_stat.stat.exists - - # ── 7. Create systemd service ────────────────────────────────────── - - name: Create forgejo-runner systemd service - copy: - dest: /etc/systemd/system/forgejo-runner.service - content: | - [Unit] - Description=Forgejo Runner - Documentation=https://forgejo.org/docs/latest/admin/actions/ - After=docker.service - Requires=docker.service - - [Service] - Type=simple - User={{ forgejo_runner_user }} - Group={{ forgejo_runner_user }} - WorkingDirectory={{ forgejo_runner_dir }} - ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} - Restart=on-failure - RestartSec=10 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - # ── 8. Reload systemd, enable and start ──────────────────────────── - - name: Reload systemd - systemd: - daemon_reload: yes - - - name: Enable and start forgejo-runner service - systemd: - name: forgejo-runner - enabled: yes - state: started - - # ── 9. Verify runner is active ───────────────────────────────────── - - name: Verify forgejo-runner is active - command: systemctl is-active forgejo-runner - register: runner_active - changed_when: false - - - name: Assert runner is running - assert: - that: - - runner_active.stdout == "active" - fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" - - # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── - - name: Create Uptime Kuma push monitor setup script - copy: - dest: /tmp/setup_forgejo_runner_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma push monitor setup - command: > - {{ ansible_playbook_python }} - /tmp/setup_forgejo_runner_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "services" - "forgejo-runner-healthcheck" - "Forgejo Runner healthcheck - ping every {{ healthcheck_interval_seconds }}s" - "{{ healthcheck_timeout_seconds }}" - "{{ healthcheck_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL - set_fact: - uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - - name: Create healthcheck script directory - file: - path: "{{ healthcheck_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create forgejo-runner healthcheck script - copy: - dest: "{{ healthcheck_script_path }}" - content: | - #!/bin/bash - - # Forgejo Runner Healthcheck Script - # Checks if forgejo-runner is active and pings Uptime Kuma on success - - LOG_FILE="{{ healthcheck_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_push_url }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - main() { - if systemctl is-active --quiet forgejo-runner; then - log_message "forgejo-runner is active, sending ping" - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=forgejo-runner%20is%20active" 2>&1) - http_code=$(echo "$response" | tail -n1) - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Ping sent successfully (HTTP $http_code)" - else - log_message "ERROR: Failed to send ping (HTTP $http_code)" - exit 1 - fi - else - log_message "ERROR: forgejo-runner is not active" - exit 1 - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create healthcheck systemd service - copy: - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" - content: | - [Unit] - Description=Forgejo Runner Healthcheck - After=network.target - - [Service] - Type=oneshot - ExecStart={{ healthcheck_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create healthcheck systemd timer - copy: - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" - content: | - [Unit] - Description=Run Forgejo Runner Healthcheck every minute - Requires={{ healthcheck_service_name }}.service - - [Timer] - OnBootSec=30sec - OnUnitActiveSec={{ healthcheck_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd for healthcheck units - systemd: - daemon_reload: yes - - - name: Enable and start healthcheck timer - systemd: - name: "{{ healthcheck_service_name }}.timer" - enabled: yes - state: started - - - name: Test healthcheck script - command: "{{ healthcheck_script_path }}" - register: healthcheck_test - changed_when: false - - - name: Verify healthcheck script works - assert: - that: - - healthcheck_test.rc == 0 - fail_msg: "Healthcheck script failed to execute properly" - - - name: Display deployment summary - debug: - msg: | - Forgejo Runner deployed successfully! - - Runner Name: forgejo-runner-box - Instance: {{ forgejo_instance_url }} - Working Directory: {{ forgejo_runner_dir }} - Service: forgejo-runner.service ({{ runner_active.stdout }}) - - Healthcheck Monitor: forgejo-runner-healthcheck - Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s - Timeout: {{ healthcheck_timeout_seconds }}s - - - name: Clean up temporary monitor setup script - file: - path: /tmp/setup_forgejo_runner_monitor.py - state: absent - delegate_to: localhost - become: no + # Preserves the push URL this host has been reporting to all along, so the + # move to a role changes no behaviour. The role itself knows nothing about + # Uptime Kuma — this is just "a URL that accepts a ping", and whatever + # replaces it sets the same variable. + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_forgejo-runner/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - forgejo_runner diff --git a/ansible/services/forgejo-runner/forgejo_runner_vars.yml b/ansible/services/forgejo-runner/forgejo_runner_vars.yml deleted file mode 100644 index e618fca..0000000 --- a/ansible/services/forgejo-runner/forgejo_runner_vars.yml +++ /dev/null @@ -1,9 +0,0 @@ -forgejo_runner_version: "6.3.1" -forgejo_runner_arch: "linux-amd64" -forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" -forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" -forgejo_runner_user: "runner" -forgejo_runner_dir: "/opt/forgejo-runner" -forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" -forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" -forgejo_instance_url: "https://forgejo.contrapeso.xyz" diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index e17d08f..f908586 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -1,16 +1,11 @@ - name: Install Forgejo on Debian 12 with Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./forgejo_vars.yml vars: forgejo_subdomain: "{{ subdomains.forgejo }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" forgejo_domain: "{{ forgejo_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure required packages are installed @@ -88,138 +83,10 @@ enabled: yes state: started - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for forgejo - copy: - dest: "{{ caddy_sites_dir }}/forgejo.conf" - content: | - {{ forgejo_domain }} { - reverse_proxy localhost:{{ forgejo_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for Forgejo - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_forgejo_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ forgejo_domain }}/api/healthz" - monitor_name: "Forgejo" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_forgejo_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_forgejo_monitor.py - - /tmp/ansible_config.yml - + - name: Publish Forgejo through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: forgejo + caddy_site_domain: "{{ forgejo_domain }}" + caddy_site_upstream: "localhost:{{ forgejo_port }}" diff --git a/ansible/services/forgejo/forgejo_vars.yml b/ansible/services/forgejo/forgejo_vars.yml index 0bbb5a5..7bba0ed 100644 --- a/ansible/services/forgejo/forgejo_vars.yml +++ b/ansible/services/forgejo/forgejo_vars.yml @@ -9,15 +9,12 @@ forgejo_url: "https://codeberg.org/forgejo/forgejo/releases/download/v{{ forgejo forgejo_bin_path: "/usr/local/bin/forgejo" forgejo_user: "git" -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/forgejo-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/forgejo_backup.sh" diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml new file mode 100644 index 0000000..5329e2a --- /dev/null +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -0,0 +1,28 @@ +--- +# Forgejo backup: dumps locally on vipy, encrypted with age. +# +# The biggest artefact in the estate (~2.7 G) and the reason retention here is +# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14. +# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the +# old job did the same. +- name: Configure the Forgejo backup on the edge host + hosts: edge + become: yes + vars_files: + - ./forgejo_vars.yml + + tasks: + - name: Ensure Forgejo dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: forgejo + backup_source_description: "Forgejo" + backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo" + backup_source_stop_service: forgejo + backup_source_retention_days: 2 + backup_source_on_calendar: "*-*-* 02:30:00" + # Reported to Gatus as backup-dump_forgejo. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_forgejo/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml deleted file mode 100644 index b90f0fb..0000000 --- a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml +++ /dev/null @@ -1,122 +0,0 @@ ---- -- name: Configure local backup for Forgejo from remote - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./forgejo_vars.yml - vars: - remote_data_path: "{{ forgejo_data_dir }}" - remote_config_path: "{{ forgejo_config_dir }}" - forgejo_service_name: "forgejo" - gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" - gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" - - tasks: - - name: Debug Forgejo backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "remote_config_path='{{ remote_config_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - "gpg_recipient={{ gpg_recipient }}" - - "gpg_key_id={{ gpg_key_id }}" - - - name: Ensure local backup directory exists - ansible.builtin.file: - path: "{{ local_backup_dir }}" - state: directory - mode: "0755" - - - name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: "0755" - - - name: Create Forgejo backup script - ansible.builtin.copy: - dest: "{{ backup_script_path }}" - mode: "0750" - content: | - #!/bin/bash - set -euo pipefail - - if [ -z "{{ gpg_recipient }}" ]; then - echo "GPG recipient is not configured. Aborting." - exit 1 - fi - - TIMESTAMP=$(date +'%Y-%m-%d') - ENCRYPTED_BACKUP="{{ local_backup_dir }}/forgejo-backup-$TIMESTAMP.tar.gz.gpg" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - echo "Stopping Forgejo service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop {{ forgejo_service_name }}" - - echo "Creating encrypted backup archive..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo tar -czf - {{ remote_data_path }} {{ remote_config_path }}" | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" - - echo "Starting Forgejo service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start {{ forgejo_service_name }}" - - # Rotate old backups (keep 3 days) - # Calculate cutoff date (3 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '3 days ago' +'%Y-%m-%d') - for backup_file in "{{ local_backup_dir }}"/forgejo-backup-*.tar.gz.gpg; do - if [ -f "$backup_file" ]; then - # Extract date from filename: forgejo-backup-YYYY-MM-DD.tar.gz.gpg - file_date=$(basename "$backup_file" | sed -n 's/forgejo-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$backup_file" - fi - fi - done - - echo "Backup completed successfully" - - - name: Ensure cronjob for Forgejo backup exists - ansible.builtin.cron: - name: "Forgejo backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run Forgejo backup script to create initial backup - ansible.builtin.command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check if backup file exists - stat: - path: "{{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg" - register: backup_file_stat - - - name: Verify backup file exists - assert: - that: - - backup_file_stat.stat.exists - - backup_file_stat.stat.isreg - fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" - success_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists" - - - name: Verify backup file is not empty - assert: - that: - - backup_file_stat.stat.size > 0 - fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" - success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 1255cd6..1b7b963 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -1,716 +1,27 @@ +--- +# Fulcrum: Electrum server indexing the Bitcoin Knots node. +# The index takes days to rebuild, so nothing here touches its data directory +# beyond asserting that it exists. - name: Deploy Fulcrum Electrum Server - hosts: fulcrum_box_local + hosts: electrum become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./fulcrum_vars.yml vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + # Preserves the push URL this check has been configured with. The role knows + # nothing about Uptime Kuma — this is just "a URL that accepts a ping". + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_fulcrum/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - fulcrum - tasks: - - name: Calculate 75% of system RAM for db_mem - set_fact: - fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" - changed_when: false - - - name: Display calculated db_mem value - debug: - msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" - - - name: Display Fulcrum version to install - debug: - msg: "Installing Fulcrum version {{ fulcrum_version }}" - - - name: Install required packages - apt: - name: - - curl - - wget - - openssl - state: present - update_cache: yes - - - name: Create fulcrum group - group: - name: "{{ fulcrum_group }}" - system: yes - state: present - - - name: Create fulcrum user - user: - name: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - system: yes - shell: /usr/sbin/nologin - home: /home/{{ fulcrum_user }} - create_home: yes - state: present - - - name: Create Fulcrum database directory (heavy data on special mount) - file: - path: "{{ fulcrum_db_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - - - name: Create Fulcrum config directory - file: - path: "{{ fulcrum_config_dir }}" - state: directory - owner: root - group: "{{ fulcrum_group }}" - mode: '0755' - - - name: Create Fulcrum lib directory (for banner and other data files) - file: - path: "{{ fulcrum_lib_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - - # =========================================== - # SSL Certificate Generation - # =========================================== - - name: Check if SSL certificate already exists - stat: - path: "{{ fulcrum_ssl_cert_path }}" - register: fulcrum_ssl_cert_exists - when: fulcrum_ssl_enabled | default(false) - - - name: Generate self-signed SSL certificate for Fulcrum - command: > - openssl req -x509 -newkey rsa:4096 - -keyout {{ fulcrum_ssl_key_path }} - -out {{ fulcrum_ssl_cert_path }} - -sha256 -days {{ fulcrum_ssl_cert_days }} - -nodes - -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" - args: - creates: "{{ fulcrum_ssl_cert_path }}" - when: fulcrum_ssl_enabled | default(false) - notify: Restart fulcrum - - - name: Set SSL certificate permissions - file: - path: "{{ fulcrum_ssl_cert_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) - - - name: Set SSL key permissions - file: - path: "{{ fulcrum_ssl_key_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0600' - when: fulcrum_ssl_enabled | default(false) - - - name: Check if Fulcrum binary already exists - stat: - path: "{{ fulcrum_binary_path }}" - register: fulcrum_binary_exists - changed_when: false - - - name: Download Fulcrum binary tarball - get_url: - url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - mode: '0644' - when: not fulcrum_binary_exists.stat.exists - - - name: Extract Fulcrum binary - unarchive: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp" - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - - - name: Install Fulcrum binary - copy: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" - dest: "{{ fulcrum_binary_path }}" - owner: root - group: root - mode: '0755' - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - - - name: Verify Fulcrum binary installation - command: "{{ fulcrum_binary_path }} --version" - register: fulcrum_version_check - changed_when: false - - - name: Display Fulcrum version - debug: - msg: "{{ fulcrum_version_check.stdout_lines }}" - - - name: Create Fulcrum banner file - copy: - dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" - content: | - counterinfra - - PER ASPERA AD ASTRA - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - - - name: Create Fulcrum configuration file - copy: - dest: "{{ fulcrum_config_dir }}/fulcrum.conf" - content: | - # Fulcrum Configuration - # Generated by Ansible - - # Bitcoin Core/Knots RPC settings - bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} - rpcuser = {{ bitcoin_rpc_user }} - rpcpassword = {{ bitcoin_rpc_password }} - - # Fulcrum server general settings - datadir = {{ fulcrum_db_dir }} - tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} - peering = {{ 'true' if fulcrum_peering else 'false' }} - zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} - - # SSL/TLS Configuration - {% if fulcrum_ssl_enabled | default(false) %} - ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} - cert = {{ fulcrum_ssl_cert_path }} - key = {{ fulcrum_ssl_key_path }} - {% endif %} - - # Anonymize client IP addresses and TxIDs in logs - anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} - - # Max RocksDB Memory in MiB - db_mem = {{ fulcrum_db_mem_mb }}.0 - - # Banner - banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0640' - notify: Restart fulcrum - - - name: Create systemd service file for Fulcrum - copy: - dest: /etc/systemd/system/fulcrum.service - content: | - # MiniBolt: systemd unit for Fulcrum - # /etc/systemd/system/fulcrum.service - - [Unit] - Description=Fulcrum - After=network.target - - StartLimitBurst=2 - StartLimitIntervalSec=20 - - [Service] - ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf - - User={{ fulcrum_user }} - Group={{ fulcrum_group }} - - # Process management - #################### - Type=simple - KillSignal=SIGINT - TimeoutStopSec=300 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum service - systemd: - name: fulcrum - enabled: yes - state: started - - - name: Wait for Fulcrum to start - wait_for: - port: "{{ fulcrum_tcp_port }}" - host: "{{ fulcrum_tcp_bind }}" - delay: 5 - timeout: 30 - ignore_errors: yes - - - name: Check Fulcrum service status - systemd: - name: fulcrum - register: fulcrum_service_status - changed_when: false - - - name: Display Fulcrum service status - debug: - msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" - - - name: Create Fulcrum health check and push script - copy: - dest: /usr/local/bin/fulcrum-healthcheck-push.sh - content: | - #!/bin/bash - # - # Fulcrum Health Check and Push to Uptime Kuma - # Checks if Fulcrum TCP port is responding and pushes status to Uptime Kuma - # - - FULCRUM_HOST="{{ fulcrum_tcp_bind }}" - FULCRUM_PORT={{ fulcrum_tcp_port }} - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - # Check if Fulcrum TCP port is responding - check_fulcrum() { - # Try to connect to TCP port - timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null - return $? - } - - # Push status to Uptime Kuma - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - if ! curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then - echo "ERROR: Failed to push to Uptime Kuma" - return 1 - fi - } - - # Main health check - if check_fulcrum; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Fulcrum TCP port not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create systemd timer for Fulcrum health check - copy: - dest: /etc/systemd/system/fulcrum-healthcheck.timer - content: | - [Unit] - Description=Fulcrum Health Check Timer - Requires=fulcrum.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for Fulcrum health check - copy: - dest: /etc/systemd/system/fulcrum-healthcheck.service - content: | - [Unit] - Description=Fulcrum Health Check and Push to Uptime Kuma - After=network.target fulcrum.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum health check timer - systemd: - name: fulcrum-healthcheck.timer - enabled: yes - state: started - - - name: Create Uptime Kuma push monitor setup script for Fulcrum - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_fulcrum_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for new monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "Fulcrum" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_fulcrum_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Extract push URL from monitor setup output - set_fact: - uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" - delegate_to: localhost - become: no - when: monitor_setup.stdout is defined - - - name: Display extracted push URL - debug: - msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" - when: uptime_kuma_push_url | default('') != '' - - - name: Set push URL in systemd service environment - lineinfile: - path: /etc/systemd/system/fulcrum-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" - state: present - insertafter: '^\[Service\]' - when: uptime_kuma_push_url | default('') != '' - - - name: Reload systemd daemon after push URL update - systemd: - daemon_reload: yes - when: uptime_kuma_push_url | default('') != '' - - - name: Restart health check timer to pick up new environment - systemd: - name: fulcrum-healthcheck.timer - state: restarted - when: uptime_kuma_push_url | default('') != '' - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_fulcrum_monitor.py - - /tmp/ansible_config.yml - - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz - - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux - - handlers: - - name: Restart fulcrum - systemd: - name: fulcrum - state: restarted - - -- name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd - hosts: vipy +- name: Setup public Fulcrum SSL forwarding on the edge host + hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./fulcrum_vars.yml - vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - tasks: - - name: Create Fulcrum SSL proxy socket unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.socket - content: | - [Unit] - Description=Fulcrum SSL Proxy Socket - - [Socket] - ListenStream={{ fulcrum_ssl_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum-ssl-proxy socket - - - name: Create Fulcrum SSL proxy service unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.service - content: | - [Unit] - Description=Fulcrum SSL Proxy to {{ fulcrum_tailscale_hostname }} - Requires=fulcrum-ssl-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ fulcrum_tailscale_hostname }}:{{ fulcrum_ssl_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum SSL proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - enabled: yes - state: started - - - name: Allow Fulcrum SSL port through UFW - ufw: - rule: allow - port: "{{ fulcrum_ssl_port | string }}" - proto: tcp - comment: "Fulcrum SSL public access" - - - name: Verify connectivity to fulcrum-box via Tailscale - wait_for: - host: "{{ fulcrum_tailscale_hostname }}" - port: "{{ fulcrum_ssl_port }}" - timeout: 10 - ignore_errors: yes - - - name: Display public endpoint - debug: - msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public SSL Port - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_fulcrum_ssl_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_fulcrum_ssl_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for TCP monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_fulcrum_ssl_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ fulcrum_ssl_port }} - monitor_name: "Fulcrum SSL Public" - mode: '0644' - - - name: Run Uptime Kuma TCP monitor setup - command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up TCP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_fulcrum_ssl_tcp_monitor.py - - /tmp/ansible_fulcrum_ssl_config.yml - - handlers: - - name: Restart fulcrum-ssl-proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - state: restarted - + - name: Expose Fulcrum SSL through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: fulcrum-ssl + socket_proxy_description: "Fulcrum SSL" + socket_proxy_listen_port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ hostvars['fulcrum_box_local'].ansible_host }}" diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml deleted file mode 100644 index 8486b14..0000000 --- a/ansible/services/fulcrum/fulcrum_vars.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Fulcrum Configuration Variables - -# Version - Pinned to specific release -fulcrum_version: "2.1.0" # Fulcrum version to install - -# Directories -fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) -fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) -fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk -fulcrum_binary_path: /usr/local/bin/Fulcrum - -# Network - Bitcoin RPC connection -# Bitcoin Knots is on a different host (knots_box_local) -# Using RPC user/password authentication (credentials from infra_secrets.yml) -bitcoin_rpc_host: "192.168.1.140" # Bitcoin Knots RPC host (IP of knots_box_local) -bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml - -# Network - Fulcrum server -fulcrum_tcp_port: 50001 -fulcrum_ssl_port: 50002 -# Binding address for Fulcrum TCP/SSL server: -# - "127.0.0.1" = localhost only (use when Caddy is on the same box) -# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) -# - Specific IP = bind to specific network interface -fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) -fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port -# If Caddy is on a different box, set this to the IP address that Caddy will use to connect - -# SSL/TLS Configuration -fulcrum_ssl_enabled: true -fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" -fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" -fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert - -# Port forwarding configuration (for public access via VPS) -fulcrum_tailscale_hostname: "fulcrum-box" - -# Performance -# db_mem will be calculated as 75% of available RAM automatically in playbook -fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache - -# Configuration options -fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs -fulcrum_peering: false # Disable peering with other Fulcrum servers -fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications - -# Service user -fulcrum_user: fulcrum -fulcrum_group: fulcrum - diff --git a/ansible/services/gatus/deploy_gatus_playbook.yml b/ansible/services/gatus/deploy_gatus_playbook.yml new file mode 100644 index 0000000..d6a5ac1 --- /dev/null +++ b/ansible/services/gatus/deploy_gatus_playbook.yml @@ -0,0 +1,106 @@ +--- +# Gatus: health checks, status page and alerting for the whole estate. +# +# Built from source and run under systemd - upstream publishes no binaries, and +# their Dockerfile shows the runtime needs nothing but the static binary and a +# CA bundle. See roles/gatus/README.md. +- name: Deploy Gatus on the observability host + hosts: observability + become: yes + vars: + gatus_alerting: + signal: + # NOTE: the key is `api-url`, not `url` as upstream's own README table + # says - see alerting/provider/signal/signal.go. Gatus appends /v2/send + # itself if the suffix is missing. + # + # Reached by service name over the shared docker network. Gatus runs in + # a container, so 127.0.0.1 here would be the Gatus container, not the + # host - and the Signal API deliberately publishes no ports because it + # has no authentication. + api-url: "http://signal-api:8080" + number: "{{ signal_number }}" + recipients: "{{ signal_recipients }}" + default-alert: + # Overridden per group by the registration playbooks; these are the + # values that apply if a caller sets nothing. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + # An ongoing outage should not become an ongoing phone buzz. + minimum-reminder-interval: 6h + + roles: + - gatus + +# The dashboard is bound to loopback; Caddy publishes it. +# +# Auth is done HERE, at the edge, and not with Gatus's own `security.basic`. +# Gatus's security middleware protects exactly four routes (api/api.go): +# +# /api/v1/endpoints/statuses +# /api/v1/endpoints/:key/statuses +# /api/v1/suites/statuses +# /api/v1/suites/:key/statuses +# +# Everything else is registered on the UNPROTECTED router, including +# /api/v1/config, every badge, and - the part that matters - +# /api/v1/endpoints/:key/uptimes/:duration and .../response-times/:duration/history, +# which return real per-endpoint data to anyone who can guess a key. Keys are +# just "_". So Gatus's own auth makes the dashboard render empty +# while leaving the data readable, which is worse than it looks. +# +# The one route that must NOT sit behind basic auth is the external-endpoint +# push API. It authenticates with `Authorization: Bearer `, and basic +# auth wants `Authorization: Basic <...>` - same header, two schemes, and the +# push clients lose. It is not actually unauthenticated: the handler 401s on a +# missing prefix, an empty token, or a token that does not match that endpoint's +# own. Upstream's comment on the route says exactly that. +- name: Publish the Gatus status page through Caddy + hosts: observability + become: yes + tasks: + - name: Require the dashboard credentials to be set + ansible.builtin.assert: + that: + - gatus_dashboard_username is defined + - gatus_dashboard_username | length > 0 + - gatus_dashboard_password_hash is defined + - gatus_dashboard_password_hash.startswith('$2') + fail_msg: >- + gatus_dashboard_username and gatus_dashboard_password_hash must be in + the vault. Generate the hash on the observability host, which runs + Caddy natively, so the bcrypt cost and format match what verifies it: + caddy hash-password --plaintext 'your-password' + then: ansible-vault edit group_vars/all/vault.yml + + - name: Configure the Caddy vhost for Gatus + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: gatus + caddy_site_domain: "{{ subdomains.gatus }}.{{ root_domain }}" + # caddy_site_body rather than caddy_site_upstream + caddy_site_basic_auth, + # because that pair applies auth to the whole site with no way to carve + # out the push path. `handle` blocks are mutually exclusive and first + # match wins, so the push API gets a route of its own. + caddy_site_body: | + @push { + path /api/v1/endpoints/*/external + method POST + } + + # Push API: Bearer-authenticated by Gatus itself. No basic auth here, + # or the Authorization header collides. + handle @push { + reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} + } + + # Everything else: the dashboard, the config endpoint, the badges and + # the uptime/response-time history. + handle { + basic_auth { + {{ gatus_dashboard_username }} {{ gatus_dashboard_password_hash }} + } + reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} + } diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 1bcf5bf..e181f24 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -1,18 +1,12 @@ - name: Deploy headscale and configure Caddy reverse proxy - hosts: spacey + hosts: vpn_control become: no vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./headscale_vars.yml vars: headscale_subdomain: "{{ subdomains.headscale }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" headscale_base_domain: "tailnet.{{ root_domain }}" - headscale_namespace: "{{ service_settings.headscale.namespace }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Install required packages @@ -230,144 +224,34 @@ port: '3478' proto: udp - - name: Ensure Caddy sites-enabled directory exists - become: yes - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - become: yes - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for headscale - become: yes - copy: - dest: "{{ caddy_sites_dir }}/headscale.conf" - content: | - {{ headscale_domain }} { - reverse_proxy localhost:{{ headscale_port }} + - name: Publish headscale through Caddy + ansible.builtin.include_role: + name: caddy_site + # This play is become: no and elevates per task. `apply` is how an + # include_role passes become down to the role's tasks - `become:` on + # the include itself is rejected. The role's handler sets its own. + apply: + become: yes + vars: + caddy_site_name: headscale + caddy_site_domain: "{{ headscale_domain }}" + # Raw body, and it must stay raw: the /admin* route in front of + # Headplane is not expressible as a plain reverse_proxy. The previous + # version of this task wrote only `reverse_proxy localhost:8080`, which + # would have deleted the admin route and its auth on the next run. + caddy_site_body: | + @headplane { + path /admin* } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - become: yes - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for Headscale - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_headscale_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ headscale_domain }}/health" - monitor_name: "Headscale" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_headscale_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_headscale_monitor.py - - /tmp/ansible_config.yml + handle @headplane { + basicauth { + {{ headscale_ui_username }} {{ headscale_ui_password_hash }} + } + reverse_proxy http://localhost:{{ headplane_port }} + } + # Headscale API is protected by its own API key authentication + # All API operations require a valid Bearer token in the Authorization header + reverse_proxy * http://localhost:{{ headscale_port }} handlers: - name: Restart headscale diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index 653c175..c3fb948 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -1,5 +1,5 @@ # Headscale service configuration -# (subdomain and caddy_sites_dir now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) headscale_port: 8080 headscale_grpc_port: 50443 @@ -10,15 +10,14 @@ headscale_version: "0.26.1" # Data directory headscale_data_dir: /var/lib/headscale -# Namespace now configured in services_config.yml under service_settings.headscale.namespace +# Namespace is headscale_namespace in group_vars/all/main.yml # Remote access -remote_host_name: "spacey" +remote_host_name: "{{ groups['vpn_control'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/headscale-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/headscale_backup.sh" +# Headplane (headscale admin UI), proxied at /admin* behind Caddy basic auth +headplane_port: 3000 diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml new file mode 100644 index 0000000..15ba5b7 --- /dev/null +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -0,0 +1,24 @@ +--- +- name: Configure the Headscale backup on the vpn_control host + hosts: vpn_control + become: yes + vars_files: + - ./headscale_vars.yml + + tasks: + - name: Ensure Headscale dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + # -C / with relative paths: avoids tar's "removing leading /" and makes + # the restore target explicit. + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_headscale. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_headscale/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml deleted file mode 100644 index 5f9a764..0000000 --- a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml +++ /dev/null @@ -1,75 +0,0 @@ -- name: Configure local backup for Headscale from remote - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./headscale_vars.yml - vars: - remote_data_path: "{{ headscale_data_dir }}" - remote_config_path: "/etc/headscale" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "remote_config_path='{{ remote_config_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - # Stop headscale service for consistent backup - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop headscale" - - # Backup data directory - rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/data/" - - # Backup config directory - rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_config_path }}/ "$BACKUP_DIR/config/" - - # Start headscale service again - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start headscale" - - # Rotate old backups (keep 14 days) - find "{{ local_backup_dir }}" -maxdepth 1 -type d -name '20*' -mtime +13 -exec rm -rf {} \; - - - name: Ensure cronjob for backup exists - cron: - name: "Headscale backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index e5d546b..7b89f33 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -1,16 +1,11 @@ - name: Deploy LNBits with Poetry and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./lnbits_vars.yml vars: lnbits_subdomain: "{{ subdomains.lnbits }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" lnbits_domain: "{{ lnbits_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create lnbits directory @@ -147,142 +142,12 @@ enabled: yes state: started - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for lnbits - copy: - dest: "{{ caddy_sites_dir }}/lnbits.conf" - content: | - {{ lnbits_domain }} { - reverse_proxy localhost:{{ lnbits_port }} { - header_up X-Forwarded-Host {{ lnbits_domain }} - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for LNBits - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_lnbits_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ lnbits_domain }}/api/v1/health" - monitor_name: "LNBits" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_lnbits_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_lnbits_monitor.py - - /tmp/ansible_config.yml - + - name: Publish LNBits through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: lnbits + caddy_site_domain: "{{ lnbits_domain }}" + caddy_site_upstream: "localhost:{{ lnbits_port }}" + caddy_site_headers_up: + X-Forwarded-Host: "{{ lnbits_domain }}" diff --git a/ansible/services/lnbits/lnbits_vars.yml b/ansible/services/lnbits/lnbits_vars.yml index bdb97df..466ce78 100644 --- a/ansible/services/lnbits/lnbits_vars.yml +++ b/ansible/services/lnbits/lnbits_vars.yml @@ -3,15 +3,12 @@ lnbits_dir: /opt/lnbits lnbits_data_dir: "{{ lnbits_dir }}/data" lnbits_port: 8765 -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/lnbits-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/lnbits_backup.sh" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml new file mode 100644 index 0000000..2da0ac0 --- /dev/null +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -0,0 +1,28 @@ +--- +# LNBits backup: dumps locally on vipy, encrypted with age. +# The old job produced TWO gpg artefacts (data, then .env separately). They are +# folded into one tar here so the wallet database and the .env that configures +# it are always the same point in time; two artefacts written by two runs can +# drift. Pulling one file back out needs no unpacking: +# age -d -i | tar -xzO opt/lnbits/lnbits/.env +- name: Configure the LNBits backup on the edge host + hosts: edge + become: yes + vars_files: + - ./lnbits_vars.yml + + tasks: + - name: Ensure LNBits dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: lnbits + backup_source_description: "LNBits" + backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env" + backup_source_stop_service: lnbits + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:20:00" + # Reported to Gatus as backup-dump_lnbits. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_lnbits/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml deleted file mode 100644 index 5d10dec..0000000 --- a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml +++ /dev/null @@ -1,104 +0,0 @@ -- name: Configure local backup for LNBits from remote - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./lnbits_vars.yml - vars: - remote_data_path: "{{ lnbits_data_dir }}" - remote_lnbits_dir: "{{ lnbits_dir }}/lnbits" - gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" - gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - "gpg_recipient={{ gpg_recipient }}" - - "gpg_key_id={{ gpg_key_id }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - ENCRYPTED_BACKUP="{{ local_backup_dir }}/lnbits-backup-$TIMESTAMP.tar.gz.gpg" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - # Stop LNBits service before backup - echo "Stopping LNBits service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop lnbits.service" - - # Create encrypted backup on the fly - # First, create a tar archive of the data directory and pipe it through gpg - echo "Creating backup..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "cd {{ remote_data_path }} && tar -czf - ." | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" - - # Also backup the .env file separately (smaller, might need quick access) - $SSH_CMD {{ remote_user }}@{{ remote_host }} "cat {{ remote_lnbits_dir }}/.env" | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "{{ local_backup_dir }}/lnbits-env-$TIMESTAMP.gpg" - - # Start LNBits service after backup - echo "Starting LNBits service..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start lnbits.service" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for backup_file in "{{ local_backup_dir }}"/lnbits-backup-*.tar.gz.gpg; do - if [ -f "$backup_file" ]; then - # Extract date from filename: lnbits-backup-YYYY-MM-DD.tar.gz.gpg - file_date=$(basename "$backup_file" | sed -n 's/lnbits-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$backup_file" - fi - fi - done - for env_file in "{{ local_backup_dir }}"/lnbits-env-*.gpg; do - if [ -f "$env_file" ]; then - # Extract date from filename: lnbits-env-YYYY-MM-DD.gpg - file_date=$(basename "$env_file" | sed -n 's/lnbits-env-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$env_file" - fi - fi - done - - echo "Backup completed successfully" - - - name: Ensure cronjob for backup exists - cron: - name: "LNBits backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index da56bd6..8b21b85 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -1,10 +1,7 @@ - name: Deploy Memos on memos-box - hosts: memos_box_local + hosts: memos become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" @@ -65,14 +62,6 @@ owner: root group: root - - name: Clean up temporary files - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/memos.tar.gz - - /tmp/memos - - name: Create memos environment file copy: dest: "{{ memos_config_dir }}/memos.env" @@ -145,162 +134,21 @@ state: restarted -- name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) - hosts: vipy +- name: Configure Caddy reverse proxy for Memos on the edge host (via Tailscale) + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for memos (via Tailscale) - copy: - dest: "{{ caddy_sites_dir }}/memos.conf" - content: | - {{ memos_domain }} { - reverse_proxy {{ memos_tailscale_hostname }}:{{ memos_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for Memos - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_memos_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_memos_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_memos_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ memos_domain }}/healthz" - monitor_name: "Memos" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_memos_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_memos_monitor.py - - /tmp/ansible_memos_config.yml + - name: Publish Memos through Caddy (via Tailscale) + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: memos + caddy_site_domain: "{{ memos_domain }}" + caddy_site_upstream: "{{ memos_tailscale_hostname }}:{{ memos_port }}" + caddy_site_resolvers: "100.100.100.100" diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index 99618db..94c6de7 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -12,15 +12,7 @@ memos_url: "https://github.com/usememos/memos/releases/download/v{{ memos_versio memos_tailscale_hostname: "memos-box" memos_tailscale_ip: "100.64.0.4" -# (caddy_sites_dir and subdomain in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access (for backup from lapy via Tailscale) -backup_host: "{{ memos_tailscale_hostname }}" -backup_user: "counterweight" -backup_key_file: "~/.ssh/counterganzua" -backup_port: 22 - -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/memos_backup.sh" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml new file mode 100644 index 0000000..53ab85d --- /dev/null +++ b/ansible/services/memos/setup_backup_memos.yml @@ -0,0 +1,29 @@ +--- +# Memos backup: dumps locally on memos-box, encrypted with age. +# Replaces the lapy pull, which had been writing EMPTY directories since +# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned +# to a different machine that has no rsync. +- name: Configure the Memos backup on its own host + hosts: memos + become: yes + vars_files: + - ./memos_vars.yml + + tasks: + - name: Ensure Memos dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: memos + backup_source_description: "Memos" + backup_source_dump_command: "tar -czf - -C / var/lib/memos" + # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a + # consistent database rather than a torn mid-write copy. The old rsync + # job did not stop it. + backup_source_stop_service: memos + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" + # Reported to Gatus as backup-dump_memos. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_memos/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml deleted file mode 100644 index 6d9c161..0000000 --- a/ansible/services/memos/setup_backup_memos_to_lapy.yml +++ /dev/null @@ -1,106 +0,0 @@ -- name: Configure local backup for Memos from memos-box - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./memos_vars.yml - vars: - backup_data_path: "{{ memos_data_dir }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "backup_host={{ backup_host }}" - - "backup_user={{ backup_user }}" - - "backup_data_path='{{ backup_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if backup_key_file %} - SSH_CMD="ssh -i {{ backup_key_file }} -p {{ backup_port }}" - {% else %} - SSH_CMD="ssh -p {{ backup_port }}" - {% endif %} - - rsync -az -e "$SSH_CMD" --rsync-path="sudo rsync" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for dir in "{{ local_backup_dir }}"/20*; do - if [ -d "$dir" ]; then - dir_date=$(basename "$dir") - if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then - rm -rf "$dir" - fi - fi - done - - - name: Ensure cronjob for backup exists - cron: - name: "Memos backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 15 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check backup directory exists and contains files - stat: - path: "{{ local_backup_dir }}/{{ today_date.stdout }}" - register: backup_dir_stat - - - name: Verify backup directory exists - assert: - that: - - backup_dir_stat.stat.exists - - backup_dir_stat.stat.isdir - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" - success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" - - - name: Check if backup directory contains files - find: - paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" - recurse: yes - register: backup_files - - - name: Verify backup directory is not empty - assert: - that: - - backup_files.files | length > 0 - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" - success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" - diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 658180a..0042fb0 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -1,751 +1,33 @@ +--- - name: Deploy Mempool Block Explorer with Docker - hosts: mempool_box_local + hosts: mempool become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./mempool_vars.yml vars: - mempool_subdomain: "{{ subdomains.mempool }}" - mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + # Preserves the three push URLs these checks have been reporting to all + # along, so the move to a role changes no behaviour. The role knows nothing + # about Uptime Kuma — these are just "URLs that accept a ping", and whatever + # replaces it sets the same values. + mempool_healthchecks: + - {name: mariadb, label: MariaDB, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-mariadb/external"} + - {name: backend, label: Backend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-backend/external"} + - {name: frontend, label: Frontend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-frontend/external"} + # One token for all three components: they run on the same host, so the + # blast radius is already that host. + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - mempool - tasks: - # =========================================== - # Docker Installation (from 910_docker_playbook.yml) - # =========================================== - - name: Remove old Docker-related packages - apt: - name: - - docker.io - - docker-doc - - docker-compose - - podman-docker - - containerd - - runc - state: absent - purge: yes - autoremove: yes - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install prerequisites - apt: - name: - - ca-certificates - - curl - state: present - - - name: Create directory for Docker GPG key - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - - - name: Download Docker GPG key - get_url: - url: https://download.docker.com/linux/debian/gpg - dest: /etc/apt/keyrings/docker.asc - mode: '0644' - - - name: Get Debian architecture - command: dpkg --print-architecture - register: deb_arch - changed_when: false - - - name: Add Docker repository - apt_repository: - repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" - filename: docker - state: present - update_cache: yes - - - name: Install Docker packages - apt: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present - update_cache: yes - - - name: Ensure Docker is started and enabled - systemd: - name: docker - enabled: yes - state: started - - - name: Add user to docker group - user: - name: "{{ ansible_user }}" - groups: docker - append: yes - - # =========================================== - # Mempool Deployment - # =========================================== - - name: Create mempool directories - file: - path: "{{ item }}" - state: directory - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0755' - loop: - - "{{ mempool_dir }}" - - "{{ mempool_data_dir }}" - - "{{ mempool_mysql_dir }}" - - - name: Create docker-compose.yml for Mempool - copy: - dest: "{{ mempool_dir }}/docker-compose.yml" - content: | - # All containers use host network for Tailscale MagicDNS resolution - services: - mariadb: - image: mariadb:10.11 - container_name: mempool-db - restart: unless-stopped - network_mode: host - environment: - MYSQL_DATABASE: "{{ mariadb_database }}" - MYSQL_USER: "{{ mariadb_user }}" - MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" - MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" - volumes: - - {{ mempool_mysql_dir }}:/var/lib/mysql - healthcheck: - test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - mempool-backend: - image: mempool/backend:{{ mempool_version }} - container_name: mempool-backend - restart: unless-stopped - network_mode: host - environment: - # Database (localhost since all containers share host network) - DATABASE_ENABLED: "true" - DATABASE_HOST: "127.0.0.1" - DATABASE_DATABASE: "{{ mariadb_database }}" - DATABASE_USERNAME: "{{ mariadb_user }}" - DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" - # Bitcoin Core/Knots (via Tailnet MagicDNS) - CORE_RPC_HOST: "{{ bitcoin_host }}" - CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" - CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" - CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" - # Electrum (Fulcrum via Tailnet MagicDNS) - ELECTRUM_HOST: "{{ fulcrum_host }}" - ELECTRUM_PORT: "{{ fulcrum_port }}" - ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" - # Mempool settings - MEMPOOL_NETWORK: "{{ mempool_network }}" - MEMPOOL_BACKEND: "electrum" - MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" - MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" - volumes: - - {{ mempool_data_dir }}:/backend/cache - depends_on: - mariadb: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - mempool-frontend: - image: mempool/frontend:{{ mempool_version }} - container_name: mempool-frontend - restart: unless-stopped - network_mode: host - environment: - FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" - BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" - depends_on: - - mempool-backend - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0644' - - - name: Pull Mempool images - command: docker compose pull - args: - chdir: "{{ mempool_dir }}" - - - name: Deploy Mempool containers with docker compose - command: docker compose up -d - args: - chdir: "{{ mempool_dir }}" - - - name: Wait for MariaDB to be healthy - command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db - register: mariadb_health - until: mariadb_health.stdout == 'healthy' - retries: 30 - delay: 10 - changed_when: false - - - name: Wait for Mempool backend to start - uri: - url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - method: GET - status_code: 200 - timeout: 10 - register: backend_check - until: backend_check.status == 200 - retries: 30 - delay: 10 - ignore_errors: yes - - - name: Wait for Mempool frontend to be available - uri: - url: "http://localhost:{{ mempool_frontend_port }}" - method: GET - status_code: 200 - timeout: 10 - register: frontend_check - until: frontend_check.status == 200 - retries: 20 - delay: 5 - ignore_errors: yes - - - name: Display deployment status - debug: - msg: - - "Mempool deployment complete!" - - "Frontend: http://localhost:{{ mempool_frontend_port }}" - - "Backend API: http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - - "Backend check: {{ 'OK' if backend_check.status == 200 else 'Still initializing...' }}" - - "Frontend check: {{ 'OK' if frontend_check.status == 200 else 'Still initializing...' }}" - - # =========================================== - # Health Check Scripts for Uptime Kuma Push Monitors - # =========================================== - - name: Create Mempool MariaDB health check script - copy: - dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - check_container() { - local status=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db 2>/dev/null) - [ "$status" = "healthy" ] - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_container; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "MariaDB container unhealthy" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create Mempool backend health check script - copy: - dest: /usr/local/bin/mempool-backend-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - BACKEND_PORT={{ mempool_backend_port }} - - check_backend() { - curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_backend; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Backend API not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create Mempool frontend health check script - copy: - dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - FRONTEND_PORT={{ mempool_frontend_port }} - - check_frontend() { - curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_frontend; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Frontend not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - # =========================================== - # Systemd Timers for Health Checks - # =========================================== - - name: Create systemd services for health checks - copy: - dest: "/etc/systemd/system/mempool-{{ item.name }}-healthcheck.service" - content: | - [Unit] - Description=Mempool {{ item.label }} Health Check - After=network.target docker.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/mempool-{{ item.name }}-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - loop: - - { name: "mariadb", label: "MariaDB" } - - { name: "backend", label: "Backend" } - - { name: "frontend", label: "Frontend" } - - - name: Create systemd timers for health checks - copy: - dest: "/etc/systemd/system/mempool-{{ item }}-healthcheck.timer" - content: | - [Unit] - Description=Mempool {{ item }} Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - loop: - - mariadb - - backend - - frontend - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start health check timers - systemd: - name: "mempool-{{ item }}-healthcheck.timer" - enabled: yes - state: started - loop: - - mariadb - - backend - - frontend - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for Mempool - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_mempool_monitors.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_mempool_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitors_to_create = config['monitors'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - results = {} - for monitor_name in monitors_to_create: - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - results[monitor_name] = f"{url}/api/push/{push_token}" - print(f"Push URL ({monitor_name}): {results[monitor_name]}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - results[monitor_name] = f"{url}/api/push/{push_token}" - print(f"Push URL ({monitor_name}): {results[monitor_name]}") - - api.disconnect() - print("SUCCESS") - - # Write results to file for Ansible to read - with open('/tmp/mempool_push_urls.yml', 'w') as f: - yaml.dump(results, f) - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_mempool_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitors: - - "Mempool MariaDB" - - "Mempool Backend" - - "Mempool Frontend" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_mempool_monitors.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URLs from file - slurp: - src: /tmp/mempool_push_urls.yml - delegate_to: localhost - become: no - register: push_urls_file - ignore_errors: yes - - - name: Parse push URLs - set_fact: - push_urls: "{{ push_urls_file.content | b64decode | from_yaml }}" - when: push_urls_file.content is defined - ignore_errors: yes - - - name: Update MariaDB health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-mariadb-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool MariaDB'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool MariaDB'] is defined - - - name: Update Backend health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-backend-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Backend'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool Backend'] is defined - - - name: Update Frontend health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-frontend-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Frontend'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool Frontend'] is defined - - - name: Reload systemd after push URL updates - systemd: - daemon_reload: yes - when: push_urls is defined - - - name: Restart health check timers - systemd: - name: "mempool-{{ item }}-healthcheck.timer" - state: restarted - loop: - - mariadb - - backend - - frontend - when: push_urls is defined - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_mempool_monitors.py - - /tmp/ansible_mempool_config.yml - - /tmp/mempool_push_urls.yml - -- name: Configure Caddy reverse proxy for Mempool on vipy - hosts: vipy +- name: Configure Caddy reverse proxy for Mempool on the edge host + hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./mempool_vars.yml vars: - mempool_subdomain: "{{ subdomains.mempool }}" - mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" - + mempool_domain: "{{ subdomains.mempool }}.{{ root_domain }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for Mempool - copy: - dest: "{{ caddy_sites_dir }}/mempool.conf" - content: | - {{ mempool_domain }} { - reverse_proxy mempool-box:{{ mempool_frontend_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - systemd: - name: caddy - state: reloaded - - - name: Display Mempool URL - debug: - msg: "Mempool is now available at https://{{ mempool_domain }}" - - # =========================================== - # Uptime Kuma HTTP Monitor for Public Endpoint - # =========================================== - - name: Create Uptime Kuma HTTP monitor setup script for Mempool - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_mempool_http_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_mempool_http_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating HTTP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for HTTP monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_mempool_http_config.yml - content: | - uptime_kuma_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ mempool_domain }}" - monitor_name: "Mempool Public Access" - mode: '0644' - - - name: Run Uptime Kuma HTTP monitor setup - command: python3 /tmp/setup_mempool_http_monitor.py - delegate_to: localhost - become: no - register: http_monitor_setup - changed_when: "'SUCCESS' in http_monitor_setup.stdout" - ignore_errors: yes - - - name: Display HTTP monitor setup output - debug: - msg: "{{ http_monitor_setup.stdout_lines }}" - when: http_monitor_setup.stdout is defined - - - name: Clean up HTTP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_mempool_http_monitor.py - - /tmp/ansible_mempool_http_config.yml - + - name: Publish Mempool through Caddy (via Tailscale) + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: mempool + caddy_site_domain: "{{ mempool_domain }}" + caddy_site_upstream: "{{ hostvars['mempool_box_local'].ansible_host }}:{{ hostvars['mempool_box_local'].mempool_frontend_port }}" + caddy_site_resolvers: "100.100.100.100" diff --git a/ansible/services/mempool/mempool_vars.yml b/ansible/services/mempool/mempool_vars.yml deleted file mode 100644 index d3051c3..0000000 --- a/ansible/services/mempool/mempool_vars.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Mempool Configuration Variables - -# Version - Pinned to specific release -mempool_version: "v3.2.1" - -# Directories -mempool_dir: /opt/mempool -mempool_data_dir: "{{ mempool_dir }}/data" -mempool_mysql_dir: "{{ mempool_dir }}/mysql" - -# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) -bitcoin_host: "knots-box" -bitcoin_rpc_port: 8332 -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml - -# Network - Fulcrum Electrum server (via Tailnet Magic DNS) -fulcrum_host: "fulcrum-box" -fulcrum_port: 50001 -fulcrum_tls: "false" - -# Mempool network mode -mempool_network: "mainnet" - -# Container ports (internal) -mempool_frontend_port: 8080 -mempool_backend_port: 8999 - -# MariaDB settings -mariadb_database: "mempool" -mariadb_user: "mempool" -# Note: mariadb_mempool_password is loaded from infra_secrets.yml - - diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index b8c0064..58f0348 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -1,20 +1,24 @@ - name: Deploy ntfy-emergency-app with Docker Compose and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../infra_secrets.yml - - ../../services_config.yml - ./ntfy_emergency_app_vars.yml vars: ntfy_emergency_app_subdomain: "{{ subdomains.ntfy_emergency_app }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" ntfy_emergency_app_domain: "{{ ntfy_emergency_app_subdomain }}.{{ root_domain }}" - ntfy_service_domain: "{{ subdomains.ntfy }}.{{ root_domain }}" - ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" + # ⚠ UNRESOLVED: this app sends its notifications to an ntfy server, and the + # server it points at ran on watchtower, which is being destroyed. This was + # derived from subdomains.ntfy, which is now gone with it. + # + # Until an ntfy server exists again this URL is dead, and the app fails + # silently at exactly the moment it matters - it is an EMERGENCY notifier. + # Three ways out, none of them automatic: + # * stand ntfy up somewhere else (the monitoring VPS, or vipy) + # * point this at the public ntfy.sh + # * retire the app + ntfy_emergency_app_ntfy_url: "{{ ntfy_service_url }}" ntfy_emergency_app_ntfy_user: "{{ ntfy_username | default('') }}" ntfy_emergency_app_ntfy_password: "{{ ntfy_password | default('') }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create ntfy-emergency-app directory @@ -49,142 +53,10 @@ args: chdir: "{{ ntfy_emergency_app_dir }}" - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for ntfy-emergency-app - copy: - dest: "{{ caddy_sites_dir }}/ntfy-emergency-app.conf" - content: | - {{ ntfy_emergency_app_domain }} { - reverse_proxy localhost:{{ ntfy_emergency_app_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_ntfy_emergency_app_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ ntfy_emergency_app_domain }}" - monitor_name: "ntfy-emergency-app" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_ntfy_emergency_app_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_ntfy_emergency_app_monitor.py - - /tmp/ansible_config.yml + - name: Publish ntfy-emergency-app through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: ntfy-emergency-app + caddy_site_domain: "{{ ntfy_emergency_app_domain }}" + caddy_site_upstream: "localhost:{{ ntfy_emergency_app_port }}" diff --git a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml index 415bc4d..bfa4bd1 100644 --- a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml +++ b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml @@ -2,15 +2,21 @@ ntfy_emergency_app_dir: /opt/ntfy-emergency-app ntfy_emergency_app_port: 3000 -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # ntfy configuration ntfy_emergency_app_topic: "emergencia" ntfy_emergency_app_ui_message: "Leave Pablo a message, he will respond as soon as possible" # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" + +# Where the emergency notifications are sent. This pointed at ntfy.contrapeso.xyz +# on watchtower; that host is being destroyed, so this MUST be repointed before +# the app can work again. Left at the old value so the break is visible rather +# than silently defaulted to something plausible. +ntfy_service_url: "https://ntfy.contrapeso.xyz" diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml deleted file mode 100644 index 0729baa..0000000 --- a/ansible/services/ntfy/deploy_ntfy_playbook.yml +++ /dev/null @@ -1,120 +0,0 @@ -- name: Deploy ntfy and configure Caddy reverse proxy - hosts: watchtower - become: yes - vars_files: - - ../../infra_vars.yml - - ../../infra_secrets.yml - - ../../services_config.yml - - ./ntfy_vars.yml - vars: - ntfy_subdomain: "{{ subdomains.ntfy }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" - ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" - - tasks: - - name: Ensure /etc/apt/keyrings exists - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - - - name: Download and dearmor ntfy GPG key - shell: curl -fsSL https://archive.heckel.io/apt/pubkey.txt | gpg --dearmor -o /etc/apt/keyrings/archive.heckel.io.gpg - args: - creates: /etc/apt/keyrings/archive.heckel.io.gpg - - - name: Add ntfy APT repository - copy: - dest: /etc/apt/sources.list.d/archive.heckel.io.list - content: | - deb [arch=amd64 signed-by=/etc/apt/keyrings/archive.heckel.io.gpg] https://archive.heckel.io/apt debian main - mode: '0644' - - - name: Update APT cache - apt: - update_cache: yes - - - name: Install ntfy - apt: - name: ntfy - state: present - - - name: Ensure ntfy cache directories exist - file: - path: "{{ item }}" - state: directory - owner: ntfy - group: ntfy - mode: '0755' - loop: - - /var/cache/ntfy - - /var/cache/ntfy/attachments - - - name: Deploy ntfy configuration file - copy: - dest: /etc/ntfy/server.yml - content: | - base-url: "http://{{ ntfy_domain }}" - listen-http: ":{{ ntfy_port }}" - cache-file: "/var/cache/ntfy/cache.db" - attachment-cache-dir: "/var/cache/ntfy/attachments" - behind-proxy: true - auth-file: "/var/lib/ntfy/user.db" - auth-default-access: "deny-all" - owner: root - group: root - mode: '0644' - notify: Restart ntfy - - - name: Enable and start ntfy service - systemd: - name: ntfy - enabled: yes - state: started - - - name: Create ntfy admin user - shell: | - (echo "{{ ntfy_password }}"; echo "{{ ntfy_password }}") | ntfy user add --role=admin "{{ ntfy_username }}" - - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for ntfy - copy: - dest: "{{ caddy_sites_dir }}/ntfy.conf" - content: | - {{ ntfy_domain }}, http://{{ ntfy_domain }} { - reverse_proxy 127.0.0.1:{{ ntfy_port }} - - @httpget { - protocol http - method GET - path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) - } - redir @httpget https://{host}{uri} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - handlers: - - name: Restart ntfy - systemd: - name: ntfy - state: restarted \ No newline at end of file diff --git a/ansible/services/ntfy/ntfy_vars.yml b/ansible/services/ntfy/ntfy_vars.yml deleted file mode 100644 index 5ebec37..0000000 --- a/ansible/services/ntfy/ntfy_vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -ntfy_port: 6674 - -# ntfy_topic now lives in services_config.yml under service_settings.ntfy.topic \ No newline at end of file diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index f4ee8ec..21d8b43 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -1,16 +1,11 @@ - name: Deploy personal blog static site with Caddy file server - hosts: vipy + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./personal_blog_vars.yml vars: personal_blog_subdomain: "{{ subdomains.personal_blog }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" personal_blog_domain: "{{ personal_blog_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure user is in www-data group @@ -51,139 +46,10 @@ group: www-data mode: '0664' - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy file server configuration for personal blog - copy: - dest: "{{ caddy_sites_dir }}/personal-blog.conf" - content: | - {{ personal_blog_domain }} { - root * {{ personal_blog_web_root }} - file_server - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for Personal Blog - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_personal_blog_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ personal_blog_domain }}" - monitor_name: "Personal Blog" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_personal_blog_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_personal_blog_monitor.py - - /tmp/ansible_config.yml - + - name: Publish the blog through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: personal-blog + caddy_site_domain: "{{ personal_blog_domain }}" + caddy_site_root: "{{ personal_blog_web_root }}" diff --git a/ansible/services/personal-blog/personal_blog_vars.yml b/ansible/services/personal-blog/personal_blog_vars.yml index 59e0921..a1f34e3 100644 --- a/ansible/services/personal-blog/personal_blog_vars.yml +++ b/ansible/services/personal-blog/personal_blog_vars.yml @@ -4,7 +4,7 @@ personal_blog_web_root: "/var/www/pablohere.contrapeso.xyz" # Remote access for deployment -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml index 2b90d68..bd9b715 100644 --- a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml +++ b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml @@ -1,8 +1,7 @@ - name: Configure deployment alias for personal blog in lapy .bashrc - hosts: lapy + hosts: control gather_facts: no vars_files: - - ../../infra_vars.yml - ./personal_blog_vars.yml vars: bashrc_path: "{{ lookup('env', 'HOME') }}/.bashrc" diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index 65ad7c9..b9cf243 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -1,535 +1,15 @@ --- -# phoenixd Deployment Playbook -# -# Deploys phoenixd (https://phoenix.acinq.co/server), the server version of the -# Phoenix Lightning wallet, on vipy so LNBits can use it as a wallet backend -# over loopback. -# -# What this does: -# 1. Downloads the pinned phoenixd release and installs phoenixd + phoenix-cli -# 2. Creates a dedicated system user and a 0700 data directory -# 3. Creates and enables a systemd service -# 4. Creates a push-monitor health check script + systemd timer -# 5. Registers a push monitor in Uptime Kuma -# -# The HTTP API stays bound to 127.0.0.1 and is NOT proxied by Caddy: phoenixd -# holds funds and its API is protected by a single password. Anything that needs -# it either runs on this host or reaches it over the Tailscale mesh. -# -# ⚠️ After the first run, back up {{ phoenixd_data_dir }}/seed.dat. Losing it -# means losing the funds. See setup_backup_phoenixd_to_lapy.yml. - -- name: Deploy phoenixd on vipy - hosts: vipy +# phoenixd: Lightning node on the edge host, used by LNBits as a wallet backend. +# Never exposed through Caddy — the HTTP API stays on loopback. +- name: Deploy phoenixd on the edge host + hosts: edge become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./phoenixd_vars.yml vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - # =========================================== - # Prerequisites - # =========================================== - - name: Install phoenixd runtime dependencies - apt: - name: - - unzip - - curl - state: present - update_cache: yes - - # =========================================== - # System User and Directories - # =========================================== - - name: Create phoenixd system group - group: - name: "{{ phoenixd_group }}" - system: yes - - - name: Create phoenixd system user - user: - name: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ phoenixd_home }}" - create_home: yes - comment: "phoenixd Lightning node" - - - name: Create phoenixd home directory - file: - path: "{{ phoenixd_home }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0750" - - - name: Create phoenixd data directory - file: - path: "{{ phoenixd_data_dir }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0700" - - # =========================================== - # Download and Install - # =========================================== - - name: Check if phoenixd is already installed - stat: - path: "{{ phoenixd_bin_dir }}/phoenixd" - register: phoenixd_binary - - - name: Check installed phoenixd version - command: "{{ phoenixd_bin_dir }}/phoenixd --version" - register: phoenixd_installed_version - changed_when: false - failed_when: false - when: phoenixd_binary.stat.exists - - - name: Decide whether phoenixd needs installing - set_fact: - phoenixd_needs_install: >- - {{ not phoenixd_binary.stat.exists - or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} - - - name: Download phoenixd {{ phoenixd_version }} - get_url: - url: "{{ phoenixd_url }}" - dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - mode: "0644" - when: phoenixd_needs_install | bool - - - name: Create temporary extraction directory - file: - path: /tmp/phoenixd-extract - state: directory - mode: "0755" - when: phoenixd_needs_install | bool - - - name: Extract phoenixd archive - unarchive: - src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - dest: /tmp/phoenixd-extract - remote_src: yes - when: phoenixd_needs_install | bool - - - name: Locate extracted binaries - find: - paths: /tmp/phoenixd-extract - patterns: "{{ item }}" - recurse: yes - file_type: file - register: phoenixd_extracted - loop: - - phoenixd - - phoenix-cli - when: phoenixd_needs_install | bool - - - name: Fail if the archive did not contain the expected binaries - assert: - that: - - item.files | length > 0 - fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - - - name: Install phoenixd and phoenix-cli binaries - copy: - src: "{{ item.files[0].path }}" - dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" - remote_src: yes - owner: root - group: root - mode: "0755" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - notify: Restart phoenixd - - - name: Clean up phoenixd download artifacts - file: - path: "{{ item }}" - state: absent - loop: - - "/tmp/phoenixd-{{ phoenixd_version }}.zip" - - /tmp/phoenixd-extract - - # =========================================== - # Systemd Service - # =========================================== - - name: Build phoenixd command line arguments - set_fact: - phoenixd_args: >- - {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) - + ['--chain', phoenixd_chain] - + ['--auto-liquidity', phoenixd_auto_liquidity] - + ['--http-bind-ip', phoenixd_http_bind_ip] - + ['--http-bind-port', phoenixd_http_bind_port | string] - + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) - + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) - + ['--silent'] }} - - - name: Create phoenixd systemd service - copy: - dest: /etc/systemd/system/phoenixd.service - content: | - [Unit] - Description=phoenixd - Lightning Network Node - Documentation=https://phoenix.acinq.co/server - After=network-online.target - Wants=network-online.target - - [Service] - Type=simple - User={{ phoenixd_user }} - Group={{ phoenixd_group }} - WorkingDirectory={{ phoenixd_home }} - Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} - ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} - Restart=always - RestartSec=30 - TimeoutStartSec=120 - TimeoutStopSec=120 - StandardOutput=journal - StandardError=journal - - # Hardening: the node only ever writes to its own data directory - NoNewPrivileges=true - PrivateTmp=true - ProtectSystem=strict - ProtectHome=read-only - ReadWritePaths={{ phoenixd_data_dir }} - - LimitNOFILE=65535 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - notify: Restart phoenixd - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start phoenixd - systemd: - name: phoenixd - enabled: yes - state: started - - - name: Flush handlers so phoenixd is running before we inspect its data dir - meta: flush_handlers - - # =========================================== - # First Boot Checks - # =========================================== - - name: Wait for phoenixd to write its config file - wait_for: - path: "{{ phoenixd_data_dir }}/phoenix.conf" - state: present - timeout: 120 - - - name: Check that the seed file exists - stat: - path: "{{ phoenixd_data_dir }}/seed.dat" - register: phoenixd_seed_file - - - name: Fail if phoenixd did not create a seed - assert: - that: - - phoenixd_seed_file.stat.exists - fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" - - # =========================================== - # Health Check Script + Systemd Timer - # =========================================== - - name: Create phoenixd health check script - copy: - dest: "{{ phoenixd_healthcheck_script_path }}" - content: | - #!/bin/bash - # Checks phoenixd and pushes the result to Uptime Kuma. - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" - - check_phoenixd() { - # Service must be active and the node must answer getinfo. - # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, - # but not the bind address, so pass it explicitly. - systemctl is-active --quiet phoenixd && \ - {{ phoenixd_bin_dir }}/phoenix-cli \ - --http-bind-ip {{ phoenixd_http_bind_ip }} \ - --http-bind-port {{ phoenixd_http_bind_port }} \ - getinfo 2>/dev/null | grep -q '"nodeId"' - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_phoenixd; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "phoenixd not responding" - exit 1 - fi - owner: root - group: root - mode: "0755" - - - name: Create phoenixd health check systemd service - copy: - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" - content: | - [Unit] - Description=phoenixd Health Check - After=network.target phoenixd.service - - [Service] - Type=oneshot - User=root - ExecStart={{ phoenixd_healthcheck_script_path }} - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: "0644" - - - name: Create phoenixd health check systemd timer - copy: - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" - content: | - [Unit] - Description=phoenixd Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - - - name: Enable and start phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - enabled: yes - state: started - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for phoenixd - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_phoenixd_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import time - import traceback - import yaml - - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_phoenixd_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - try: - api.add_monitor(type='group', name='services') - except Exception: - time.sleep(2) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - push_url = None - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - else: - print(f"Creating push monitor '{monitor_name}'...") - try: - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - except Exception as e: - # socketio timeout: add_monitor may have succeeded server-side - print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) - time.sleep(2) - - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - push_url = f"{url}/api/push/{push_token}" - - api.disconnect() - - if push_url: - print(f"PUSH_URL={push_url}") - with open('/tmp/phoenixd_push_url.txt', 'w') as f: - f.write(push_url) - - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: "0755" - - - name: Create temporary config for push monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_phoenixd_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "{{ phoenixd_monitor_name }}" - mode: "0644" - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_phoenixd_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URL from file - slurp: - src: /tmp/phoenixd_push_url.txt - delegate_to: localhost - become: no - register: push_url_file - ignore_errors: yes - - - name: Parse push URL - set_fact: - phoenixd_push_url: "{{ push_url_file.content | b64decode | trim }}" - when: push_url_file.content is defined - - - name: Update health check service with push URL - lineinfile: - path: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" - regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" - line: "Environment=UPTIME_KUMA_PUSH_URL={{ phoenixd_push_url }}" - when: phoenixd_push_url is defined - notify: Restart phoenixd health check timer - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_phoenixd_monitor.py - - /tmp/ansible_phoenixd_config.yml - - /tmp/phoenixd_push_url.txt - - # =========================================== - # Post-install Notes - # =========================================== - - name: Display post-install information - debug: - msg: | - ✓ phoenixd {{ phoenixd_version }} deployed - - Status: systemctl status phoenixd - Logs: journalctl -u phoenixd -f - CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo - HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) - Data dir: {{ phoenixd_data_dir }} - - API password (needed to wire LNBits up to this node): - sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf - - ⚠️ BACK UP THE SEED NOW: {{ phoenixd_data_dir }}/seed.dat - Losing it means losing the funds. Run - services/phoenixd/setup_backup_phoenixd_to_lapy.yml and also keep - the 12 words somewhere offline. - - handlers: - - name: Restart phoenixd - systemd: - name: phoenixd - state: restarted - daemon_reload: yes - - - name: Restart phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - state: restarted - daemon_reload: yes + # phoenixd's health check has never reported anywhere since the Uptime Kuma + # decommissioning — its systemd Environment= was left empty. Leaving it empty + # preserves that; the check still runs and its exit code is still the answer. + # Set this to plug in whatever monitoring replaces it. + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_phoenixd/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - phoenixd diff --git a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml deleted file mode 100644 index cc008e6..0000000 --- a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- -# Backs up the phoenixd seed and config from vipy to Lapy, gpg encrypted. -# -# Only seed.dat and phoenix.conf are backed up, on purpose: -# - seed.dat is what actually recovers the funds. phoenixd keeps its channel -# state with the ACINQ peer, so a wallet is restored from the seed alone. -# - restoring a *stale* channel database to a running Lightning node is -# dangerous (it can trigger a force close and a penalty), so we do not keep -# copies of phoenix.db around to be tempted by. -# - phoenix.conf holds the http api password, which is what LNBits and any -# other local consumer authenticate with. -# -# Because both files are static, phoenixd does not need to be stopped. - -- name: Configure local backup for phoenixd from remote - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./phoenixd_vars.yml - vars: - remote_data_path: "{{ phoenixd_data_dir }}" - gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" - gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" - - tasks: - - name: Debug phoenixd backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - "gpg_recipient={{ gpg_recipient }}" - - "gpg_key_id={{ gpg_key_id }}" - - - name: Ensure local backup directory exists - ansible.builtin.file: - path: "{{ local_backup_dir }}" - state: directory - mode: "0700" - - - name: Ensure ~/.local/bin exists - ansible.builtin.file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: "0755" - - - name: Create phoenixd backup script - ansible.builtin.copy: - dest: "{{ backup_script_path }}" - mode: "0750" - content: | - #!/bin/bash - set -euo pipefail - - if [ -z "{{ gpg_recipient }}" ]; then - echo "GPG recipient is not configured. Aborting." - exit 1 - fi - - TIMESTAMP=$(date +'%Y-%m-%d') - ENCRYPTED_BACKUP="{{ local_backup_dir }}/phoenixd-backup-$TIMESTAMP.tar.gz.gpg" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - # seed.dat + phoenix.conf only, see the header of the playbook. - echo "Creating encrypted backup archive..." - $SSH_CMD {{ remote_user }}@{{ remote_host }} \ - "sudo tar -czf - -C {{ remote_data_path }} seed.dat phoenix.conf" | \ - gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" - - chmod 600 "$ENCRYPTED_BACKUP" - - # Rotate old backups (keep 14 days) - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for backup_file in "{{ local_backup_dir }}"/phoenixd-backup-*.tar.gz.gpg; do - if [ -f "$backup_file" ]; then - # Extract date from filename: phoenixd-backup-YYYY-MM-DD.tar.gz.gpg - file_date=$(basename "$backup_file" | sed -n 's/phoenixd-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') - if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then - rm -f "$backup_file" - fi - fi - done - - echo "Backup completed successfully" - - - name: Ensure cronjob for phoenixd backup exists - ansible.builtin.cron: - name: "phoenixd backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 15 - hour: "9" - - - name: Run phoenixd backup script to create initial backup - ansible.builtin.command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check if backup file exists - stat: - path: "{{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg" - register: backup_file_stat - - - name: Verify backup file exists - assert: - that: - - backup_file_stat.stat.exists - - backup_file_stat.stat.isreg - fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" - success_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists" - - - name: Verify backup file is not empty - assert: - that: - - backup_file_stat.stat.size > 0 - fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" - success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" - - - name: Remind about the offline seed copy - debug: - msg: | - These encrypted backups are only as safe as your GPG key. - Also write the 12 words down offline once: - ssh {{ remote_user }}@{{ remote_host }} "sudo cat {{ remote_data_path }}/seed.dat" diff --git a/ansible/services/signal-api/deploy_signal_api_playbook.yml b/ansible/services/signal-api/deploy_signal_api_playbook.yml new file mode 100644 index 0000000..d4690f9 --- /dev/null +++ b/ansible/services/signal-api/deploy_signal_api_playbook.yml @@ -0,0 +1,44 @@ +--- +# The Signal transport for Gatus alerts. +# +# Deliberately NOT published and NOT fronted by Caddy: the API has no +# authentication of any kind, so it is reachable only from the shared docker +# network that Gatus is on. See roles/signal_api/README.md, including the +# one-time manual step to link the device. +- name: Deploy the Signal API on the observability host + hosts: observability + become: yes + roles: + - signal_api + + # post_tasks, not a second play: the registration below needs the role's + # defaults (service name, port) in scope, and a separate play would not have + # them. + post_tasks: + # Monitored, because a dead alert transport is the worst kind of dead: every + # check could be failing and nothing would tell you. Gatus polls it over the + # shared network - the same path the alerts take - so this proves the actual + # delivery route rather than merely that a container is running. + # + # Deliberately NOT backed up: the data directory holds Signal private keys, + # and the recovery path is to link the device again from the phone. Backing + # it up would copy a credential off the host to buy nothing. + - name: Register the signal-api health endpoint with Gatus + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: signal-api + gatus_endpoint_pulled: + - name: signal-api + group: infrastructure + url: "http://{{ signal_api_service_name }}:{{ signal_api_port }}/v1/health" + interval: 5m + # Deliberately NOT alerted via Signal: if this endpoint is down, + # Signal is exactly what cannot deliver the alert. It is visible on + # the dashboard, and its failure shows up indirectly as every other + # alert going missing. + conditions: + # /v1/health answers 204 No Content, not 200 - checked live. Any + # 2xx is asserted rather than the exact code, so an upstream + # change from 204 to 200 does not read as an outage. + - "[STATUS] < 300" diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 0340538..282fc2e 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -1,16 +1,11 @@ - name: Deploy Vaultwarden with Docker Compose and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./vaultwarden_vars.yml vars: vaultwarden_subdomain: "{{ subdomains.vaultwarden }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" vaultwarden_domain: "{{ vaultwarden_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create vaultwarden directory @@ -81,143 +76,10 @@ name: fail2ban state: restarted - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for vaultwarden - copy: - dest: "{{ caddy_sites_dir }}/vaultwarden.conf" - content: | - {{ vaultwarden_domain }} { - reverse_proxy localhost:{{ vaultwarden_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for Vaultwarden - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_vaultwarden_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ vaultwarden_domain }}/alive" - monitor_name: "Vaultwarden" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_vaultwarden_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_vaultwarden_monitor.py - - /tmp/ansible_config.yml - + - name: Publish Vaultwarden through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: vaultwarden + caddy_site_domain: "{{ vaultwarden_domain }}" + caddy_site_upstream: "localhost:{{ vaultwarden_port }}" diff --git a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml index b041e8e..bccc2cd 100644 --- a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml +++ b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml @@ -1,8 +1,7 @@ - name: Disable Vaultwarden Signups - hosts: vipy + hosts: edge become: yes vars_files: - - ../../infra_vars.yml - ./vaultwarden_vars.yml tasks: diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml new file mode 100644 index 0000000..d0dfdfa --- /dev/null +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -0,0 +1,28 @@ +--- +# Vaultwarden backup: dumps locally on vipy, encrypted with age. +# Previously rsynced to lapy in the CLEAR; the artefact now never exists +# unencrypted, on disk or on the wire. +- name: Configure the Vaultwarden backup on the edge host + hosts: edge + become: yes + vars_files: + - ./vaultwarden_vars.yml + + tasks: + - name: Ensure Vaultwarden dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: vaultwarden + backup_source_description: "Vaultwarden" + backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data" + # Not systemd — a docker compose stack — so stop/start explicitly. + # sqlite in WAL mode, hence stopping at all. + backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" + backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:10:00" + # Reported to Gatus as backup-dump_vaultwarden. The token is this HOST's token, + # shared with its other checks - see infra/400_host_monitoring.yml. + backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_vaultwarden/external" + backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml deleted file mode 100644 index 064d633..0000000 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml +++ /dev/null @@ -1,105 +0,0 @@ -- name: Configure local backup for Vaultwarden from remote - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./vaultwarden_vars.yml - vars: - remote_data_path: "{{ vaultwarden_data_dir }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "remote_host={{ remote_host }}" - - "remote_user={{ remote_user }}" - - "remote_data_path='{{ remote_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if remote_key_file %} - SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" - {% else %} - SSH_CMD="ssh -p {{ remote_port }}" - {% endif %} - - rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for dir in "{{ local_backup_dir }}"/20*; do - if [ -d "$dir" ]; then - dir_date=$(basename "$dir") - if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then - rm -rf "$dir" - fi - fi - done - - - name: Ensure cronjob for backup exists - cron: - name: "Vaultwarden backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 5 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check backup directory exists and contains files - stat: - path: "{{ local_backup_dir }}/{{ today_date.stdout }}" - register: backup_dir_stat - - - name: Verify backup directory exists - assert: - that: - - backup_dir_stat.stat.exists - - backup_dir_stat.stat.isdir - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" - success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" - - - name: Check if backup directory contains files - find: - paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" - recurse: yes - register: backup_files - - - name: Verify backup directory is not empty - assert: - that: - - backup_files.files | length > 0 - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" - success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" diff --git a/ansible/services/vaultwarden/vaultwarden_vars.yml b/ansible/services/vaultwarden/vaultwarden_vars.yml index 75e527d..9edc2cd 100644 --- a/ansible/services/vaultwarden/vaultwarden_vars.yml +++ b/ansible/services/vaultwarden/vaultwarden_vars.yml @@ -3,15 +3,12 @@ vaultwarden_dir: /opt/vaultwarden vaultwarden_data_dir: "{{ vaultwarden_dir }}/data" vaultwarden_port: 8222 -# (caddy_sites_dir and subdomain now in services_config.yml) +# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Local backup -local_backup_dir: "{{ lookup('env', 'HOME') }}/vaultwarden-backups" -backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/vaultwarden_backup.sh" diff --git a/ansible/services_config.yml b/ansible/services_config.yml deleted file mode 100644 index 1bc9db0..0000000 --- a/ansible/services_config.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Centralized Services Configuration -# Subdomains and Caddy settings for all services - -# Edit these subdomains to match your preferences -subdomains: - # Monitoring Services (on watchtower) - ntfy: ntfy - uptime_kuma: uptime - - # VPN Infrastructure (on spacey) - headscale: headscale - - # Core Services (on vipy) - vaultwarden: vault - forgejo: forgejo - lnbits: wallet - - # Secondary Services (on vipy) - ntfy_emergency_app: avisame - personal_blog: pablohere - - # Memos (on memos-box) - memos: memos - - # Mempool Block Explorer (on mempool_box, proxied via vipy) - mempool: mempool - - # DATUM Gateway dashboard (on knots_box, proxied via vipy) - datum_gateway: datum - -# Caddy configuration -caddy_sites_dir: /etc/caddy/sites-enabled - -# Service-specific settings shared across playbooks -service_settings: - ntfy: - topic: alerts - headscale: - namespace: counter-net diff --git a/ansible/site.yml b/ansible/site.yml new file mode 100644 index 0000000..35f5867 --- /dev/null +++ b/ansible/site.yml @@ -0,0 +1,90 @@ +--- +# Everything, in the order it has to happen. +# +# This file is a TABLE OF CONTENTS, not a second source of truth. It says what +# runs and in what order. It does NOT say which hosts get what — that stays on +# the `hosts:` line inside each playbook, exactly where it is today. Nothing +# moves; this file only makes the set readable in one place. +# +# What runs on a host? ansible-playbook site.yml --limit --list-hosts +# Who gets thing Y? the `hosts:` line in Y's own playbook +# What is a host? ansible-inventory --graph +# +# Run a slice with --limit, or run one playbook directly as before. Nothing here +# changes how any individual playbook behaves. + +# ── Baseline: every managed machine ───────────────────────────────────────── +- import_playbook: infra/01_user_and_access_setup_playbook.yml +- import_playbook: infra/02_firewall_and_fail2ban_playbook.yml +- import_playbook: infra/900_install_rsync.yml +- import_playbook: infra/920_join_headscale_mesh.yml +# Idempotent and kept permanently: guarantees a rebuilt or restored host cannot +# quietly bring the Uptime-Kuma-era monitoring back. +- import_playbook: infra/409_remove_legacy_monitoring.yml + +# ── Monitoring ────────────────────────────────────────────────────────────── +# Gatus first: the three plays below register endpoints with it, and registering +# against a host that is not serving yet would simply fail. +- import_playbook: services/gatus/deploy_gatus_playbook.yml +# The Signal transport for Gatus alerts. Shares a docker network with Gatus and +# publishes no ports - the API has no authentication. Needs a one-time manual +# device link; see roles/signal_api/README.md. +- import_playbook: services/signal-api/deploy_signal_api_playbook.yml +- import_playbook: infra/400_host_monitoring.yml +- import_playbook: infra/401_service_monitoring.yml +- import_playbook: infra/402_public_monitoring.yml +# Registers where the per-service probes report. The probes themselves are +# deployed by each service's own playbook further down; the endpoints must exist +# before the first push arrives. +- import_playbook: infra/403_service_probe_registration.yml +# 910_docker says `hosts: managed`, but only 5 of 11 managed hosts have or need +# Docker. Left out until it has a [docker] group — see the note in PLAN_7. + +# ── The hypervisor ────────────────────────────────────────────────────────── +- import_playbook: infra/nodito/31_proxmox_community_repos_playbook.yml +- import_playbook: infra/nodito/32_zfs_pool_setup_playbook.yml +- import_playbook: infra/nodito/34_nut_ups_setup_playbook.yml + +# ── Reverse proxy, before anything that registers a vhost ─────────────────── +- import_playbook: services/caddy_playbook.yml + +# ── Services ──────────────────────────────────────────────────────────────── +- import_playbook: services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +- import_playbook: services/fulcrum/deploy_fulcrum_playbook.yml +- import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml +- import_playbook: services/mempool/deploy_mempool_playbook.yml +- import_playbook: services/memos/deploy_memos_playbook.yml +- import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml +- import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml +- import_playbook: services/headscale/deploy_headscale_playbook.yml +- import_playbook: services/vaultwarden/deploy_vaultwarden_playbook.yml +- import_playbook: services/forgejo/deploy_forgejo_playbook.yml +- import_playbook: services/lnbits/deploy_lnbits_playbook.yml +- import_playbook: services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +- import_playbook: services/personal-blog/deploy_personal_blog_playbook.yml + +# ── Backups: each source dumps itself, the box pulls ──────────────────────── +- import_playbook: services/headscale/setup_backup_headscale.yml +- import_playbook: services/vaultwarden/setup_backup_vaultwarden.yml +- import_playbook: services/forgejo/setup_backup_forgejo.yml +- import_playbook: services/lnbits/setup_backup_lnbits.yml +- import_playbook: services/memos/setup_backup_memos.yml +- import_playbook: playbooks/backups.yml + +# Deliberately not here. Every playbook in the repo is either imported above or +# listed below, so this file accounts for all of them: +# +# infra/910_docker_playbook.yml says `hosts: managed`, but Docker is on 5 +# of 11 managed hosts and those 5 are exactly +# the ones that need it. Running it would +# install Docker on the Bitcoin node and the +# hypervisor. Needs a [docker] group first. +# +# infra/nodito/30_proxmox_bootstrap one-shot: bare-metal bootstrap, run once +# infra/nodito/33_..._cloud_template one-shot: builds the VM template +# +# +# services/vaultwarden/disable_ deliberate manual actions, not convergence +# vaultwarden_sign_ups_playbook.yml +# services/personal-blog/setup_ +# deploy_alias_lapy.yml diff --git a/archive/uptime_kuma/MONITORS.md b/archive/uptime_kuma/MONITORS.md new file mode 100644 index 0000000..53b1d89 --- /dev/null +++ b/archive/uptime_kuma/MONITORS.md @@ -0,0 +1,152 @@ +# Uptime Kuma — monitor inventory (archived) + +Captured from the live instance at `https://uptime.contrapeso.xyz` on 2026-09-11, +immediately before decommissioning. This is the **authoritative** record: most of +these monitors existed only in the Uptime Kuma UI and were never described by any +playbook in this repo. + +**75 monitors total** — 16 group, 8 http, 3 port, 48 push. All were active. + +Push tokens are deliberately **not** recorded here: they are live credentials, and +anything holding one could report a false 'up'. They die with the server. + +--- + +## arbret - production *(7 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| arbret.com - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbret.com - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbret.com - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | +| arbret.com - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | +| arbret.com - health | push | — | 120s | Healthy when GET /api/health returns status ok | +| arbret.com - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | +| arbret.com - postgresql | push | — | 120s | Healthy when postgresql.service is active | + +## arbret - staging *(7 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| arbretstaging.contrapeso.xyz - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbretstaging.contrapeso.xyz - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbretstaging.contrapeso.xyz - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | +| arbretstaging.contrapeso.xyz - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | +| arbretstaging.contrapeso.xyz - health | push | — | 120s | Healthy when GET /api/health returns status ok | +| arbretstaging.contrapeso.xyz - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | +| arbretstaging.contrapeso.xyz - postgresql | push | — | 120s | Healthy when postgresql.service is active | + +## arbret-staging-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-arbret-staging-box-root | push | — | 960s | upside-down, Disk Usage: arbret-staging-box (/) - Alerts when usage excee | +| system-healthcheck-arbret-staging-box | push | — | 90s | System Healthcheck: arbret-staging-box - Regular healthcheck | + +## forgejo-runner-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-forgejo-runner-box-root | push | — | 960s | upside-down, Disk Usage: forgejo-runner-box (/) - Alerts when usage excee | +| system-healthcheck-forgejo-runner-box | push | — | 90s | System Healthcheck: forgejo-runner-box - Regular healthcheck | + +## fulcrum-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-fulcrum-box-root | push | — | 960s | upside-down, Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-fulcrum-box | push | — | 90s | System Healthcheck: fulcrum-box - Regular healthcheck ping e | + +## knots-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-knots-box-root | push | — | 960s | upside-down, Disk Usage: knots-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-knots-box | push | — | 90s | System Healthcheck: knots-box - Regular healthcheck ping eve | + +## memos-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-memos-box-root | push | — | 960s | upside-down, Disk Usage: memos-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-memos-box | push | — | 90s | System Healthcheck: memos-box - Regular healthcheck ping eve | + +## mempool-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-mempool-box-root | push | — | 960s | upside-down, Disk Usage: mempool-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-mempool-box | push | — | 90s | System Healthcheck: mempool-box - Regular healthcheck ping e | + +## nodito - infra *(4 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| UPS ONLINE | push | https:// | 90s | — | +| cpu-temp-nodito | push | — | 120s | upside-down, CPU Temperature: nodito - Alerts when temperature exceeds 80 | +| system-healthcheck-nodito | push | — | 300s | System Healthcheck: nodito - Regular healthcheck ping every | +| zfs-health-nodito | push | — | 90000s | ZFS Pool Health: nodito - Daily health check for pool proxmo | + +## nonkeiwaisi-box - infra *(0 monitors)* + +_(empty)_ + +## prd-arbret - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-arbret-root | push | — | 960s | upside-down, Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-arbret | push | — | 90s | System Healthcheck: prd-arbret - Regular healthcheck ping ev | + +## prd-spacey - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-spacey-root | push | — | 960s | upside-down, Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-spacey | push | — | 90s | System Healthcheck: prd-spacey - Regular healthcheck ping ev | + +## prd-vipy - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-vipy-root | push | — | 960s | upside-down, Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-vipy | push | — | 90s | System Healthcheck: prd-vipy - Regular healthcheck ping ever | + +## prd-watchtower - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-watchtower-root | push | — | 960s | upside-down, Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 8 | +| system-healthcheck-prd-watchtower | push | — | 90s | System Healthcheck: prd-watchtower - Regular healthcheck pin | + +## services *(19 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| Forgejo | http | https://forgejo.contrapeso.xyz/api/healthz | 90s | — | +| Headscale | http | https://headscale.contrapeso.xyz/health | 60s | — | +| LNBits | http | https://wallet.contrapeso.xyz/api/v1/health | 60s | — | +| Memos | http | https://memos.contrapeso.xyz/healthz | 60s | — | +| Mempool Public Access | http | https://mempool.contrapeso.xyz | 60s | — | +| Personal Blog | http | https://pablohere.contrapeso.xyz | 60s | — | +| Vaultwarden | http | https://vault.contrapeso.xyz/alive | 60s | — | +| ntfy-emergency-app | http | https://avisame.contrapeso.xyz | 60s | — | +| Bitcoin Knots P2P Public | port | 167.172.107.33:8333 | 60s | — | +| DATUM Stratum (public) | port | 167.172.107.33:23334 | 60s | — | +| Fulcrum SSL Public | port | 167.172.107.33:50002 | 60s | — | +| Bitcoin Knots | push | — | 90s | — | +| DATUM Gateway | push | — | 90s | — | +| Fulcrum | push | — | 90s | — | +| Mempool Backend | push | — | 180s | — | +| Mempool Frontend | push | — | 90s | — | +| Mempool MariaDB | push | — | 90s | — | +| Phoenixd | push | — | 90s | — | +| forgejo-runner-healthcheck | push | — | 90s | Forgejo Runner healthcheck - ping every 60s | + +## small-backups-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-small-backups-box-root | push | — | 960s | upside-down, Disk Usage: small-backups-box (/) - Alerts when usage exceed | +| system-healthcheck-small-backups-box | push | — | 90s | System Healthcheck: small-backups-box - Regular healthcheck | + diff --git a/archive/uptime_kuma/README.md b/archive/uptime_kuma/README.md new file mode 100644 index 0000000..5405ae0 --- /dev/null +++ b/archive/uptime_kuma/README.md @@ -0,0 +1,51 @@ +# Uptime Kuma — archived + +Uptime Kuma was the monitoring stack for this infrastructure until **2026-09-11**, when +it was decommissioned. Everything that referenced it has been removed from the live +playbooks; this folder is the record of what it was, kept so the setup can be understood +later without digging through git history. + +## Contents + +| File | What it is | +|---|---| +| `MONITORS.md` | Every monitor that existed, grouped as it was in the UI. The authoritative record. | +| `monitors.json` | The same data, machine-readable, as returned by the API. | +| `deploy_uptime_kuma_playbook.yml` | How the server itself was deployed (Docker Compose on `monitoring`, behind Caddy). | +| `uptime_kuma_vars.yml` | Variables the deploy playbook needed — it is unreadable without these. | +| `setup_backup_uptime_kuma_to_lapy.yml` | How its data was backed up, and to where. Useful when disposing of the old volumes. | + +## Why the inventory was captured from the live server, not the repo + +The playbooks only ever created **17** monitors. The live instance had **75**. The +difference was created by hand in the UI and existed nowhere else — so a repo-derived +list would have silently lost two thirds of the picture. `MONITORS.md` is a snapshot of +the real thing, taken immediately before removal. + +Push tokens are deliberately excluded. They are live credentials — anything holding one +can report a false "up" — and they become meaningless once the server is gone. + +## What removal did NOT do + +Removing the playbook code does not touch the machines. **48 push monitors** were driven +by scripts and systemd timers installed *on the hosts*, which keep firing on their +schedule and curling an endpoint that no longer answers. They are harmless but they are +still there, writing logs and failing quietly. + +Left behind, per host: + +- `/opt/disk-monitoring` + `disk-usage-monitor.{service,timer}` — on all 12 `managed` hosts +- `/opt/system-healthcheck` + `system-healthcheck.{service,timer}` — on all 12 `managed` hosts +- `/opt/nodito-monitoring` + `nodito-cpu-temp-monitor.{service,timer}` — `nodito` +- `/opt/zfs-monitoring` — `nodito` +- `/opt/ups-monitoring` — `nodito` +- `bitcoin-knots-healthcheck.{service,timer}` — `bitcoin` +- `datum-gateway-healthcheck.{service,timer}` — `bitcoin` +- `fulcrum-healthcheck.{service,timer}` — `electrum` +- `mempool-{backend,frontend,mariadb}-healthcheck.service` — `mempool` +- phoenixd and forgejo-runner healthcheck units — `edge`, `ci_runner` + +Note `nut-monitor.service` on `nodito` is **NUT's own daemon**, not a monitoring +leftover — do not remove it with the rest. + +Cleaning these up is a separate decommissioning pass and was not part of the removal. diff --git a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml b/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml similarity index 99% rename from ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml rename to archive/uptime_kuma/deploy_uptime_kuma_playbook.yml index 4af3858..8754dce 100644 --- a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml +++ b/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Uptime Kuma with Docker Compose and configure Caddy reverse proxy - hosts: watchtower + hosts: monitoring become: yes vars_files: - ../../infra_vars.yml diff --git a/archive/uptime_kuma/monitors.json b/archive/uptime_kuma/monitors.json new file mode 100644 index 0000000..d1b0b0a --- /dev/null +++ b/archive/uptime_kuma/monitors.json @@ -0,0 +1,1202 @@ +[ + { + "id": 1, + "name": "services", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 3, + "name": "Vaultwarden", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://vault.contrapeso.xyz/alive", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 4, + "name": "prd-spacey - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 5, + "name": "prd-watchtower - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 6, + "name": "prd-vipy - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 7, + "name": "disk-usage-prd-vipy-root", + "type": "push", + "active": true, + "parent": 6, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 8, + "name": "disk-usage-prd-watchtower-root", + "type": "push", + "active": true, + "parent": 5, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 9, + "name": "disk-usage-prd-spacey-root", + "type": "push", + "active": true, + "parent": 4, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 10, + "name": "system-healthcheck-prd-vipy", + "type": "push", + "active": true, + "parent": 6, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-vipy - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 11, + "name": "system-healthcheck-prd-spacey", + "type": "push", + "active": true, + "parent": 4, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-spacey - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 12, + "name": "system-healthcheck-prd-watchtower", + "type": "push", + "active": true, + "parent": 5, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-watchtower - Regular healthcheck ping every 60s", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 13, + "name": "Forgejo", + "type": "http", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": "https://forgejo.contrapeso.xyz/api/healthz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 10, + "resendInterval": 0 + }, + { + "id": 14, + "name": "Personal Blog", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://pablohere.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 15, + "name": "LNBits", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://wallet.contrapeso.xyz/api/v1/health", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 16, + "name": "ntfy-emergency-app", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://avisame.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 17, + "name": "Headscale", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://headscale.contrapeso.xyz/health", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 18, + "name": "knots-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 19, + "name": "disk-usage-knots-box-root", + "type": "push", + "active": true, + "parent": 18, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: knots-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 20, + "name": "system-healthcheck-knots-box", + "type": "push", + "active": true, + "parent": 18, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: knots-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 21, + "name": "nodito - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 22, + "name": "system-healthcheck-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 300, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: nodito - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 23, + "name": "cpu-temp-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "CPU Temperature: nodito - Alerts when temperature exceeds 80\u00b0C", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 24, + "name": "Bitcoin Knots", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 25, + "name": "fulcrum-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 26, + "name": "disk-usage-fulcrum-box-root", + "type": "push", + "active": true, + "parent": 25, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 27, + "name": "system-healthcheck-fulcrum-box", + "type": "push", + "active": true, + "parent": 25, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: fulcrum-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 28, + "name": "Fulcrum", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 35, + "name": "mempool-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 36, + "name": "disk-usage-mempool-box-root", + "type": "push", + "active": true, + "parent": 35, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: mempool-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 37, + "name": "system-healthcheck-mempool-box", + "type": "push", + "active": true, + "parent": 35, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: mempool-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 38, + "name": "Mempool MariaDB", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 39, + "name": "Mempool Backend", + "type": "push", + "active": true, + "parent": 1, + "interval": 180, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 40, + "name": "Mempool Frontend", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 41, + "name": "Mempool Public Access", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://mempool.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 42, + "name": "memos-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 43, + "name": "disk-usage-memos-box-root", + "type": "push", + "active": true, + "parent": 42, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: memos-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 44, + "name": "system-healthcheck-memos-box", + "type": "push", + "active": true, + "parent": 42, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: memos-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 45, + "name": "Memos", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://memos.contrapeso.xyz/healthz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 46, + "name": "Bitcoin Knots P2P Public", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 8333, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 47, + "name": "Fulcrum SSL Public", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 50002, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 49, + "name": "zfs-health-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 90000, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "ZFS Pool Health: nodito - Daily health check for pool proxmox-tank-1", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 50, + "name": "UPS ONLINE", + "type": "push", + "active": true, + "parent": 21, + "interval": 90, + "retries": null, + "url": "https://", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 51, + "name": "forgejo-runner-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 52, + "name": "disk-usage-forgejo-runner-box-root", + "type": "push", + "active": true, + "parent": 51, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: forgejo-runner-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 53, + "name": "system-healthcheck-forgejo-runner-box", + "type": "push", + "active": true, + "parent": 51, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: forgejo-runner-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 55, + "name": "forgejo-runner-healthcheck", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Forgejo Runner healthcheck - ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 59, + "name": "arbret-staging-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 60, + "name": "disk-usage-arbret-staging-box-root", + "type": "push", + "active": true, + "parent": 59, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: arbret-staging-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 61, + "name": "system-healthcheck-arbret-staging-box", + "type": "push", + "active": true, + "parent": 59, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: arbret-staging-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 66, + "name": "arbret - staging", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 67, + "name": "arbretstaging.contrapeso.xyz - health", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when GET /api/health returns status ok", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 68, + "name": "arbretstaging.contrapeso.xyz - https", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when HTTPS front-door returns 200", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 69, + "name": "arbretstaging.contrapeso.xyz - postgresql", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when postgresql.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 70, + "name": "arbretstaging.contrapeso.xyz - arbret-server", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-server.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 71, + "name": "arbretstaging.contrapeso.xyz - arbret-worker", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-worker.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 72, + "name": "arbretstaging.contrapeso.xyz - arbret-analytics", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 73, + "name": "arbretstaging.contrapeso.xyz - arbret-backup", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 74, + "name": "prd-arbret - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 85, + "name": "disk-usage-prd-arbret-root", + "type": "push", + "active": true, + "parent": 74, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 86, + "name": "system-healthcheck-prd-arbret", + "type": "push", + "active": true, + "parent": 74, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-arbret - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 87, + "name": "arbret - production", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 88, + "name": "arbret.com - health", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when GET /api/health returns status ok", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 89, + "name": "arbret.com - https", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when HTTPS front-door returns 200", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 90, + "name": "arbret.com - postgresql", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when postgresql.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 91, + "name": "arbret.com - arbret-server", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-server.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 92, + "name": "arbret.com - arbret-worker", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-worker.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 93, + "name": "arbret.com - arbret-analytics", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 94, + "name": "arbret.com - arbret-backup", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 95, + "name": "small-backups-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 96, + "name": "disk-usage-small-backups-box-root", + "type": "push", + "active": true, + "parent": 95, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: small-backups-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 97, + "name": "system-healthcheck-small-backups-box", + "type": "push", + "active": true, + "parent": 95, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: small-backups-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 109, + "name": "DATUM Gateway", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 111, + "name": "DATUM Stratum (public)", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 23334, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 112, + "name": "Phoenixd", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 113, + "name": "nonkeiwaisi-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + } +] diff --git a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml b/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml similarity index 99% rename from ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml rename to archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml index 9ae9713..54c0b79 100644 --- a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml +++ b/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Uptime Kuma from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml b/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml similarity index 83% rename from ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml rename to archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml index 5ba03f1..97d6c9c 100644 --- a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml +++ b/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml @@ -1,15 +1,24 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup ntfy as Uptime Kuma Notification Channel - hosts: watchtower + hosts: observability become: no vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - ./ntfy_vars.yml vars: ntfy_subdomain: "{{ subdomains.ntfy }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" uptime_kuma_subdomain: "{{ subdomains.uptime_kuma }}" ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" ntfy_server_url: "https://{{ ntfy_domain }}" diff --git a/ansible/services/uptime_kuma/uptime_kuma_vars.yml b/archive/uptime_kuma/uptime_kuma_vars.yml similarity index 92% rename from ansible/services/uptime_kuma/uptime_kuma_vars.yml rename to archive/uptime_kuma/uptime_kuma_vars.yml index 3263f49..33ffb6d 100644 --- a/ansible/services/uptime_kuma/uptime_kuma_vars.yml +++ b/archive/uptime_kuma/uptime_kuma_vars.yml @@ -4,7 +4,7 @@ uptime_kuma_data_dir: "{{ uptime_kuma_dir }}/data" uptime_kuma_port: 3001 # Remote access -remote_host_name: "watchtower" +remote_host_name: "{{ groups['monitoring'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/requirements.txt b/requirements.txt index 972dc4f..653523e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,7 @@ packaging==25.0 pycparser==2.22 PyYAML==6.0.2 resolvelib==1.0.1 +# Only needed by the deprecated Uptime Kuma monitoring blocks, which are kept for +# reference but never run (uptime_kuma_enabled: false). Drop this once they are +# rewired to a replacement. See archive/uptime_kuma/. uptime-kuma-api>=1.2.1 diff --git a/tofu/nodito/.terraform.lock.hcl b/tofu/nodito/.terraform.lock.hcl new file mode 100644 index 0000000..fadff38 --- /dev/null +++ b/tofu/nodito/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/telmate/proxmox" { + version = "3.0.2-rc05" + constraints = "3.0.2-rc05" + hashes = [ + "h1:QfHovHn8h9uJXdJ+urOuiD7R46OXmdZQiRcCBaV6AD4=", + "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f", + "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc", + "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321", + "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3", + "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98", + "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb", + "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e", + "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9", + "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d", + "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3", + "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e", + "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269", + "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1", + "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123", + ] +} diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md index 3a0b18f..bff6585 100644 --- a/tofu/nodito/README.md +++ b/tofu/nodito/README.md @@ -27,16 +27,17 @@ This directory lets you declare VMs on the `nodito` Proxmox node and apply with - The Ansible template exists: `debian-13-cloud-init` (VMID 9001 by default). ### Provider Auth -Create a `terraform.tfvars` (copy from `terraform.tfvars.example`) and set: +Credentials are the only thing not in git. Copy `terraform.tfvars.example` to +`terraform.tfvars` (gitignored) and set: - `proxmox_api_url` (e.g. `https://nodito:8006/api2/json`) - `proxmox_api_token_id` (e.g. `root@pam!tofu`) - `proxmox_api_token_secret` -- `ssh_authorized_keys` (your public key content) -Alternatively, you can export env vars and reference them in a tfvars file. +Alternatively, export them as `TF_VAR_proxmox_api_token_secret` etc. ### Declare VMs -Edit `terraform.tfvars` and fill the `vms` map. Example entry: +VMs are declared in `vms.auto.tfvars`, which is committed. `*.auto.tfvars` is +loaded automatically, so it needs no `-var-file`. Example entry: ``` vms = { web1 = { @@ -54,15 +55,24 @@ All VM disks are created on `zfs_storage_name` (defaults to `proxmox-tank-1`). N ### Usage ``` tofu init -tofu plan -var-file=terraform.tfvars -tofu apply -var-file=terraform.tfvars +tofu plan +tofu apply ``` +`terraform.tfvars` and `vms.auto.tfvars` are both auto-loaded. > VMs are created once and then protected: the module sets `lifecycle.prevent_destroy = true` and ignores subsequent config changes. After the initial apply, manage day‑2 changes directly in Proxmox (or remove the lifecycle block if you need OpenTofu to own ongoing updates). ### Notes - Clones are full clones by default (`full_clone = true`). - Cloud-init injects `cloud_init_user` and `ssh_authorized_keys`. +- `.terraform.lock.hcl` is committed: it pins the provider hashes, which matters + because `versions.tf` tracks a release candidate (`3.0.2-rc05`). +- State is local (`terraform.tfstate`, gitignored) and has no remote backend, so + it exists only on the machine that last ran `tofu apply`. +- The map is not a complete inventory of nodito: state also manages + `bastion-box` (1100) and `nonkeiwaisi-box` (3300), which are not declared in + `vms.auto.tfvars`. `tofu plan` is clean today, but relaxing the `lifecycle` + block without first declaring them would put them up for destruction. - Disks use `scsi0` on ZFS with `discard` enabled. diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example index cc88b3f..37b6d7a 100644 --- a/tofu/nodito/terraform.tfvars.example +++ b/tofu/nodito/terraform.tfvars.example @@ -1,35 +1,7 @@ -proxmox_api_url = "https://nodito:8006/api2/json" -proxmox_api_token_id = "root@pam!tofu" -proxmox_api_token_secret = "REPLACE_ME" - -proxmox_node = "nodito" -zfs_storage_name = "proxmox-tank-1" -template_name = "debian-13-cloud-init" -cloud_init_user = "counterweight" - -# paste your ~/.ssh/id_ed25519.pub or similar -ssh_authorized_keys = <