diff --git a/.gitignore b/.gitignore index 471cd6f..6c0a084 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,21 @@ # OpenTofu / Terraform .terraform/ .tofu/ +.terraform.lock.hcl +.tofu.lock.hcl terraform.tfstate terraform.tfstate.* crash.log +*.tfvars +*.tfvars.json -# Provider credentials only. Non-secret infra config (the vms map) is committed -# as *.auto.tfvars, and *.lock.hcl is committed on purpose so provider hashes -# are pinned. -terraform.tfvars -terraform.tfvars.json -*secrets.auto.tfvars - +test-inventory.ini +inventory.ini venv/* .env -# Secrets are ansible-vault encrypted and ARE committed. -# Anything matching *_secrets.plain.yml is a working decryption — never commit those. -*_secrets.plain.yml - -# Vault password — never commit -ansible/.vault_pass \ No newline at end of file +# Secrets and sensitive files +*_secrets.yml +*_secrets.yaml +secrets/ +.secrets/ diff --git a/01_infra_setup.md b/01_infra_setup.md index a8ba9c0..6ef0978 100644 --- a/01_infra_setup.md +++ b/01_infra_setup.md @@ -162,12 +162,6 @@ Note that, by applying these playbooks, both the root user and the `counterweigh ```bash cp ansible/infra_secrets.yml.example ansible/infra_secrets.yml ``` - > **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server - > deployment was removed from this repo; what it monitored and how it was set up is - > preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in - > the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic - > survives for whatever replaces it. The credentials below no longer exist in the vault. - * Edit `ansible/infra_secrets.yml` and add your Uptime Kuma credentials: ```yaml uptime_kuma_username: "admin" diff --git a/02_vps_core_services_setup.md b/02_vps_core_services_setup.md index 5d75de9..19cd122 100644 --- a/02_vps_core_services_setup.md +++ b/02_vps_core_services_setup.md @@ -49,12 +49,6 @@ Checklist: ## Uptime Kuma -> **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server -> deployment was removed from this repo; what it monitored and how it was set up is -> preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in -> the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic -> survives for whatever replaces it. The credentials below no longer exist in the vault. - Uptime Kuma gets used to monitor the availability of services, keep track of their uptime and notify issues. ### Deploy diff --git a/README.md b/README.md index 89ef092..f343cbc 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,6 @@ My repo documenting my personal infra, along with artifacts, scripts, etc. Go through the different numbered markdowns in the repo root to do the different parts. -## How to edit secrets - -`ansible-vault edit ansible/your_file_with_secrets.yml` - -Assumes that you've set `ansible/.vault_pass` with `chmod 600`. - ## Overview ### Services @@ -22,7 +16,7 @@ Assumes that you've set `ansible/.vault_pass` with `chmod 600`. + Plan install + File based config + Crossbackup to Desky via rsync -* ~~Uptime Kuma~~ — decommissioned 2026-09-11, see `archive/uptime_kuma/` +* Uptime Kuma + Deployed on Vipy + Crossbackup to Desky via rsync * Vaultwarden diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg deleted file mode 100644 index aa10181..0000000 --- a/ansible/ansible.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[defaults] -inventory = inventory.ini -roles_path = roles -collections_path = collections -interpreter_python = auto_silent -stdout_callback = yaml -retry_files_enabled = False -host_key_checking = True -forks = 10 -vault_password_file = .vault_pass - -[ssh_connection] -pipelining = True -ssh_args = -o ControlMaster=auto -o ControlPersist=300s diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml deleted file mode 100644 index f77ae48..0000000 --- a/ansible/group_vars/all/main.yml +++ /dev/null @@ -1,66 +0,0 @@ -new_user: counterweight -ssh_port: 22 -allow_ssh_from: "any" -root_domain: contrapeso.xyz - -# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the -# playbooks are kept deliberately — the check logic is meant to be rewired to -# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. - -# age recipient for all backup artefacts -age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" - -# Public key small-backups-box pulls with -# Authorised on each source host for an unprivileged, dedicated user only -backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" - - -# ───────────────────────────────────────────────────────────────────────────── -# Subdomains. Global because the edge host proxies for services that live on -# other machines, so no single inventory group covers the readers. Combine with -# root_domain above to build an FQDN. -# -# Moved here from services_config.yml, which 30 plays had to remember to name in -# vars_files: - a file everyone must opt into is a file someone will forget. -# ───────────────────────────────────────────────────────────────────────────── -subdomains: - # Monitoring - gatus: status - - # VPN infrastructure (spacey) - headscale: headscale - - # Core services (vipy) - vaultwarden: vault - forgejo: forgejo - lnbits: wallet - - # Secondary services (vipy) - ntfy_emergency_app: avisame - personal_blog: pablohere - - # Memos (memos-box) - memos: memos - - # Mempool block explorer (mempool-box, proxied via vipy) - mempool: mempool - - # DATUM Gateway dashboard (knots-box, proxied via vipy) - datum_gateway: datum - -# Read by plays across several groups, so global rather than group_vars/. -headscale_namespace: counter-net - -# ───────────────────────────────────────────────────────────────────────────── -# Domains whose registration expiry is monitored (infra/402_public_monitoring). -# -# Registration renewal is a manual act at the registrar, and losing a domain is -# not recoverable in the way losing a host is - so these are checked daily and -# alarm with two weeks of runway. -# -# root_domain is the estate's own domain; the rest are domains we own that are -# served from it or from a host in the inventory. -# ───────────────────────────────────────────────────────────────────────────── -monitored_domains: - - "{{ root_domain }}" - - arbret.com diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml deleted file mode 100644 index 76eb7bf..0000000 --- a/ansible/group_vars/all/vault.yml +++ /dev/null @@ -1,164 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -34383033613438623432656438313763613639313139626432303637383363643339613166626665 -6537663336353661373437303636613364643736663031300a623461353135623537343763383333 -37323731646262613436646231656263396532616132316237353762376363383062646132303435 -3435316666316235660a323163613932616661303935336633656332353766393536393736356539 -61396164386266303464373262373862363361363365363235383363613335663933313062663565 -39613466353161356462656539613335613536363532333431393935303430373435373737363537 -62383266323339373062313134316464303263313830646161333530383736336637333865373764 -31313165396366653763343466643561383236643836616662383439666165343139326232326364 -32633133373863616163343365656231353939366435346534383462396134653064663566343566 -66343464373865383864313264646635316233336133346365396634626261613561373965643339 -30353831393037316334313632376537393361336366326561343832373531303139323937613531 -34386164303430316162363536386137316330336638303365393462386465633862663932376563 -34323862636435306563333065383134343362333733623066656439613339376666353331373665 -32353938663364313036373237323062393430306661633732356434333232376338313835663330 -32306536393666323464663238306562613936376236613935616664653865373330626365626330 -64613066653632646638306338393331376430383165316363323437373362613866313962663432 -62356663636630356239613831353066303530363966396135376134633936313530643262353965 -38383330613962313935366633353635363638303937316362303866343433353437363334396431 -32393134626133353564643233616632376633303934653065613262353436333533376137303862 -35303861383833326537376433336634633730373833313036643365373431666438653033653930 -33346230396661636333613031353133353631306331666362386366633062666534303539316238 -39366634656333646634643032393738333330636230663366623265666533356534623465316665 -35336535326237343231653661613736626239316262386363666264613636326332366232333964 -38353635616662663133376534306366643037613732343233373336653166316330643638353539 -35353532643534613864333839393365313439343663333337373639633832383630316164306132 -63636162373034363839656638353534613733663737626432636436373164303239316436313566 -66623466396663353136323565643262383865323830616232373466373431373261306537646463 -61353966623062356438626635663461396165343366393132393965383734633632666335313030 -37333631373239303231396264393031373162373936303462363736313538343562343737356332 -63663762633032356130303964386664366137373432353533386566613037613163383963623732 -32343765376535343630656633333362393765666131653632613361363163383038323138333537 -37373333316466316562613439356365316163336366623435666166653835616531653563343664 -30303464663838646333373837363961356465613234636138633162366636633030383337643131 -62636236663066316630396132333934353139383465363034363033656161383663656462313939 -62353738396132343734343363636164323163616237613861643862303433633934656165343263 -35313737373333386335663731643664393630653839396464376639306231653934386336353630 -61393735336662663435613431386361373561363531643232303831363163636139653538393931 -66313965646562616237383439343637623835303065333730613865666638383131616261346463 -31396464306537613135653830653138323539393731626264326335336432666333333735646534 -30616439393464326631633635363466303336613135346231626232313361303666323661616638 -64363937343231363536353034363966326132333734386638613737636130646232363666643633 -61353563636466356630346232613761636432303430336461663434333636623962643764336639 -35303565613861323431393036303534303061666437366538306236373930313439353330313632 -61363363303562316664303065613863613339626632643931386438326330373938613762303334 -33316430363931393262373661623137633835656136313235613666613932313236343966306331 -36343031303632323432366337633637336335343638393564313738386162386164613339343662 -62383939663265373265613932633265626263623939303638383838646531343433393864363235 -37313761396436623833376435393537376136373162393465343764326533666533393061643965 -61306365346562333737313962633764313232323161623861336566343735353737633539656162 -34323939306661373662663331653333346264323930663636633134666532623438323537383165 -66343033363962633766303331623866306462623235373838616565653066646132633034363535 -31613462356361613535383963643662373162363363653334393937313039666266366363653537 -64356166396139393262313565303731346534646462333638316661383139346362333364613466 -38303561316263636136303431393366343936653161336238613439366266346136636465303337 -61373634376464333037303862616335623031656133636165636264386265643261373735663266 -39396265366138363430343035356337303438346165366361316230656239326633653537626339 -39613334316563376133343336343563363564653237633764393532336437303334363830396635 -31333963666462313830333163646465356337643263636462356363613266323630393866626535 -35336436663063333364353663616564626535396133386461326433323936323161613633353539 -66323935626433353162666335613839346561343264323763303034613366663233613037343538 -66376236373462383430313030373235373337653338333033306431616562353435656433356365 -37653063363435366532306334373438306632306435653334396630343863623938666335656164 -37386634366430383762316435396237313362313965353634613266616532633465646464313662 -35303433363236393465396161346431306231666432666531626136623966663864376663373562 -37653239633732346536323265643434616136326561666330633435383264333937353366653736 -61353430633363323263323163646639663330663632343038623964396433626337353232373334 -32343938356539303938666137323831646562393233323733336464316165623232326132316362 -61313631626234336164656533616231393663363364623130313439623466396231383537656562 -35393163323131383838356130643863633736613635306635306135666261623731613239303962 -61373432336638353339646665653561653738353731373032366361313562306466373365653333 -35306236336331333961613864346165613763383266613236343066623432303766333434333331 -66376263363132616132366165393235626336323665373135653462653533376138346632393363 -35396165383966326635333134663138366136336430393532373935373264616530613762386430 -37643862316232613261343430616438333831623835663239656531356666613032653837373531 -30386232643537306530323936386561646265616339376265353763663833623539653831656634 -63626564633964343663656435636635656562666236386639353332356566316631643431353330 -32643238623932643732353332633535643738663066383830376432346463393464333236386163 -64376165333266323166613564313061643832656239333165393035343836376434626162643061 -31373861393534636236383765313834356638613332626666653762343537363339383939666263 -61366230623930643534353565316165656132396664393138326139613536343738396330653735 -37623463343834626133313231393130646134363331393238663065643930383237326439343831 -32393536613361616637373761393361303936363863333838666633343063386235316237323531 -62633362316436623761616366383336613765373362393161303765613464643632666462343565 -63616430396164346263666436336463623638313130656231643131336663336565616135373433 -39326533643733333564373434333738343634383033353036646534646632383730613161393130 -62653836376435663435353739373536366466646332653030653332353761626466353631663163 -38613734383634663662376464653733306564313761336566336336656366653536643834653631 -35333963306562393234623039613338643266333762333236366331333431343830393864623830 -34323439386133613035363861623332663933313734313637623739613163663730316134303434 -35363239363836666463366334356435666632396132306163303037663430383535393934323537 -30353266373864346662333035393761646632313032383738346439343232663666343833333565 -35363564623038376666363436663830356165323234653230636264303066666230643632356237 -66633734346633366533623966393462313438333965623033343337323135353730613561363236 -30636333613034626162393137323762306432663761393766303163393536656338366664643432 -34636630323965396166356538623165663335333739323034333237306531393734306138343436 -36613937373932633234303336373530663533646635396664343935326334333239646534343336 -35396666353938326337646431353965653836323731313861613031356561376262363965623934 -65336164623134366365303062393161353965363937646363396564313138323938656330333964 -64663862363766613630333531623463616331363533343962653164376665303463336134346662 -66376238663632663932656230323638323135653666363736643065336330636236323064386361 -37646631303138616565383665663634333338613336346438366333636165366635313337373635 -62656236653961393061373262326534393339363236353336613339656462336661663739396261 -35363636653237316436656662633462306333313336313865333037636661383731336466666536 -36646537653431666261353232653233653930346538383438633466346636656432383963333765 -62346363633265306665366630383831303434336139343837616361666539656262633735363064 -30376138386531336430613563366666333964663231363465326230316233323532366632663930 -63653164373931353861653833326130643365373530663039363166366331303134366336613438 -63633336363539636265396331653630633162316366316130326531363463616464376635376538 -32376638353337343639633434613663666437313564333264383766623561393933333561663235 -37666163626538353736383435373530663438363636643665626266316338336532623235353337 -61653762383630616266353438386137653137616532633066623165383663646437343565366237 -33333463613539346136613135386466656135616136663036313661323738363066633038656139 -38353230313861666463383737303038613731316334333031643438343463666164633536313031 -39613663376436643165333362636462303230656336333234363336633262336363643736336231 -66316630613762383832613836333263303038306235663662323162633932653733626130356665 -65316630346534356261373961353130393334396365376237353439326261336339393336303030 -63323439666439313831626639323735653730356565643938376564316337613333616664643865 -30633530383865663731383635366566613565393632336337306334663937396135303539363536 -65656331386532306438383031313164626237393762303636663831666239336637666465633630 -63383961336333663537666561376532376631343132363638633364616531306363623432396332 -31343864626335613131653762363338393738343531626330663663376531336230656365363661 -64363530333965386439376161313534333435393966656231613161633034303964653564363436 -63396131623562303962383137666439373132633330363237353031306561306362666561376435 -66383535333233663736363762323662363264393365643135323834366565653932313863353437 -38363536323739336165356561613335653464363637636234313637383066633865353239393961 -31613532373361383062393238303430353662383930373764373237383035633364323661623661 -62653661623234366666616532663563316234663137616338393036333937633963323635366165 -61613032363531666338353132346365323661626265626461396362646338373536306362303338 -31646564343434633433663039366533656665643235336136333134366231376336343465336539 -37363139346239666232313366613432646265626564313838346364306133306434313337626562 -32346430316532623037633838303363346130623636386134313566383333613565316135373264 -38393261313435323433326130303032393538333963646430646366653363613830613534353832 -32643434613964613239363966376138376661636335373130356430336631333461623734383535 -35356461326136636234663738343738303064386536336462303632303461303733333666383036 -38383563393739343134363038623638393766316533373163336439336238613734656132623163 -32626564666430663339316163393934323133306238353562323866633738363737643937376435 -66306430623334643766623564363239346361393666663766306637313265313833396435626234 -35366239616132623863343365663833363934316362636638303536656631633364646235366566 -39386637613735373761643339396132323031323438316633363464336636316534643435323861 -62663732366430646538626663313035616235643537643234356434323635373962343633336266 -30323031306438656537626430356231393466313334633934623632663661306361313330633339 -33396234386536336331616161363132343765306562303932313963623037366633383765663134 -62636466636433666530363930633765626531613539363832313361666565323635326334326533 -36336439363836663033643262336139613437623633616138656564393032393263393132303031 -31626336663039633362366663306662333432643261613464303939326562653562346239383261 -37373966383635626433363831653936333964626262363839383936356634633233336365623765 -62313031353538636464303234383865613932323164386336316362323731303263346637346534 -32353734343130343364646161643431353230336364366330363261326334613936633234646264 -39623163346663613431323630303034383761663835373565663166366239303130386139396236 -62393565353761636636366462353038366664663430616332326465373364396264323064666536 -61653035633562316634336263636332363733663666626131383161656535383133613939623034 -66363632613733306336323165336534346337386163656631343332323636353539353737316131 -34616336343865656565646338653037333838663736333330376330663834373138363739633064 -61646439636365323838376131663266636235333062616532613936616339633661303634323531 -63356335386534353639323566376134373565353137333134363761666532366431633634333430 -39326662633765626230326130386333393463323433366162363432613234336634663439313339 -33653737333830336264343839396563316462613032376335356634383436353962613636333866 -32333032623635323430356636623138366635386533646133626164316438393937626462313239 -31636635363064363763396566306234363965346438653738333961623435303233396634643763 -65343763386464366636383734336335646464306262623363303934393734333666636635356365 -65353931303862343765343665373139376530656263356365366333323135383838383030613166 -30303963383833643937636539656164336537383337346436303534303035313935323633663765 -333464336235373230316337333632326138 diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml deleted file mode 100644 index 1072b51..0000000 --- a/ansible/group_vars/nodito_vms.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. -# -# ansible_host is a MagicDNS name. If tailscaled is not running on the control -# node that name does not resolve, the probe fails, and the LAN address recorded -# as lan_ip in inventory.ini takes over. -# -# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`: -# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a -# single `nc -w5` silently tears down the SSH session after five quiet seconds. -# That produced intermittent "Data could not be sent to remote host" failures on -# exactly the long, quiet operations (apt) where a dropped connection costs most. -# `nc -z` probes, then `exec nc` carries the session with no timeout at all. -# -# Safe against the LAN addresses drifting again (which is how fulcrum/mempool -# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if -# lan_ip ever points at a different machine the host key will not match and ssh -# aborts. Verified by pointing fulcrum-box at mempool-box's address: -# "Host key verification failed." -# -# lan_ip is a convenience, not an identity. If it goes stale the fallback stops -# working; it will never connect you to the wrong box. -ansible_ssh_common_args: >- - -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'" diff --git a/ansible/host_vars/forgejo_runner_local/main.yml b/ansible/host_vars/forgejo_runner_local/main.yml deleted file mode 100644 index 6e852ce..0000000 --- a/ansible/host_vars/forgejo_runner_local/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - forgejo-runner diff --git a/ansible/host_vars/fulcrum_box_local/main.yml b/ansible/host_vars/fulcrum_box_local/main.yml deleted file mode 100644 index aa189b0..0000000 --- a/ansible/host_vars/fulcrum_box_local/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -# fulcrum-box: the Electrum server. -# -# Read by the fulcrum role here and by the socket-proxy play on the edge host, -# which publishes the port. See host_vars/knots_box_local/main.yml for why this -# lives in host_vars rather than in the role's defaults. -fulcrum_ssl_port: 50002 - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - fulcrum diff --git a/ansible/host_vars/knots_box_local/main.yml b/ansible/host_vars/knots_box_local/main.yml deleted file mode 100644 index 3866541..0000000 --- a/ansible/host_vars/knots_box_local/main.yml +++ /dev/null @@ -1,26 +0,0 @@ -# knots-box: Bitcoin Knots and the DATUM Gateway. -# -# These ports are read twice: by the role that deploys the service here, and by -# the socket-proxy / Caddy plays that run on the EDGE host and publish them. -# A role default is invisible to that second play, which is why these live in -# host_vars rather than roles//defaults/ - the edge play reads them as -# hostvars['knots_box_local']., and the role picks them up automatically -# because host_vars outranks role defaults. -# -# They used to live in services_config.yml, a file 30 plays had to remember to -# name in vars_files: and that four role defaults silently depended on. -bitcoin_p2p_port: 8333 -datum_gateway_api_port: 7152 -datum_gateway_stratum_port: 23334 - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - bitcoind - - datum-gateway diff --git a/ansible/host_vars/memos_box_local/main.yml b/ansible/host_vars/memos_box_local/main.yml deleted file mode 100644 index f1fc19b..0000000 --- a/ansible/host_vars/memos_box_local/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - memos diff --git a/ansible/host_vars/mempool_box_local/main.yml b/ansible/host_vars/mempool_box_local/main.yml deleted file mode 100644 index f46a1a5..0000000 --- a/ansible/host_vars/mempool_box_local/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -# mempool-box: the Mempool block explorer. -# -# Read by the mempool role here and by the Caddy play on the edge host, which -# proxies to it. See host_vars/knots_box_local/main.yml for why this lives in -# host_vars rather than in the role's defaults. -mempool_frontend_port: 8080 diff --git a/ansible/host_vars/monitoring/main.yml b/ansible/host_vars/monitoring/main.yml deleted file mode 100644 index 2ef805d..0000000 --- a/ansible/host_vars/monitoring/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - caddy diff --git a/ansible/host_vars/nodito/vault.yml b/ansible/host_vars/nodito/vault.yml deleted file mode 100644 index e64085e..0000000 --- a/ansible/host_vars/nodito/vault.yml +++ /dev/null @@ -1,11 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -30333035323663393939343061323234336164396465623665346165393534646366333332376463 -3364373463333664363334373964323838336531353364310a636636373539623464336630666164 -61376532616339376562373238383436306664313564663266303534346461666466383965323538 -3163313239626663310a613033336332653165333537313366636361663036383031376561613761 -31313563373062333033323037653939663762343161656264633436343361663737626366663732 -39366666626338323436383134646263643538333564313566346336323563663534653161396136 -39333565393538366238643563323630346166643461643063393631643665363566623631373762 -36643866646637306231653837363838656163613766636265383139333838396535626335343163 -32613935353330636263616333666230323436663935326133636362343836323535623237646235 -6266316363366335323162663039366137633865396237373632 diff --git a/ansible/host_vars/spacey/main.yml b/ansible/host_vars/spacey/main.yml deleted file mode 100644 index a50d116..0000000 --- a/ansible/host_vars/spacey/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - headscale - - caddy diff --git a/ansible/host_vars/vipy/main.yml b/ansible/host_vars/vipy/main.yml deleted file mode 100644 index 3b077df..0000000 --- a/ansible/host_vars/vipy/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - forgejo - - lnbits - - caddy - - phoenixd diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index 0e2c914..13e5149 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,5 +1,7 @@ - name: Secure Debian - hosts: managed + hosts: all + vars_files: + - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index 309db56..e83cbcb 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,5 +1,7 @@ - name: Secure Debian - hosts: managed + hosts: all + vars_files: + - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/400_host_monitoring.yml b/ansible/infra/400_host_monitoring.yml deleted file mode 100644 index 3f6b119..0000000 --- a/ansible/infra/400_host_monitoring.yml +++ /dev/null @@ -1,216 +0,0 @@ ---- -# Host-level monitoring for the whole estate, reported to Gatus. -# -# Every check here PUSHES. Gatus never reaches out, which matters because nodito -# and its VMs sit behind NAT, and because four of the five checks are internal -# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool -# health and UPS mains status cannot be observed from outside the machine. -# -# Liveness is a push too, and that is a choice rather than a limitation. A -# heartbeat proves the host is running AND can reach the internet; an ICMP probe -# from one vantage point only proves it answers pings from there. And because -# Gatus alerts when a heartbeat window expires, a check that stops running -# raises the alarm by itself - a dead timer looks exactly like a dead host, -# which is the correct reading. -# -# Each host has ONE bearer token, shared across its own checks: a token can only -# write results for that host's endpoints, so a compromised host can lie about -# itself, which it could do anyway. -# -# The push URL must use Gatus's own key format (config/key/key.go): -# key = sanitize(group) + "_" + sanitize(name) -# where sanitize lowercases and replaces / _ . , space # + & with "-". So -# knots_box_local becomes knots-box-local in the URL but stays readable in the -# name. host_key below is the Jinja equivalent; do not hand-write these. - -# ───────────────────────────────────────────────────────────────────────────── -# Register everything with Gatus. -# -# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the -# host plays below take minutes, so every endpoint exists before its first push -# arrives. Registering afterwards would 404 every first report. -# -# Heartbeat windows are several times the check interval, so one missed run - a -# slow apt run, a reboot - does not raise an alarm, but a check that has -# genuinely stopped does. -# ───────────────────────────────────────────────────────────────────────────── -# ───────────────────────────────────────────────────────────────────────────── -# Alerting thresholds, and why they differ by check type. -# -# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a -# different amount of wall-clock time per check: -# -# push/heartbeat endpoints a failure is produced once per heartbeat window -# pulled endpoints a failure is produced once per interval -# -# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day -# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m -# window on a 5-minute push is precisely "one missed push forgiven"; stacking a -# threshold of 3 on top triples a tolerance that was already chosen. -# -# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints -# have no built-in tolerance, so the threshold is where it belongs for them. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the host checks with Gatus - hosts: observability - become: yes - vars: - monitored: "{{ groups['managed'] | sort }}" - - tasks: - - name: Build the liveness endpoint list - ansible.builtin.set_fact: - liveness_endpoints: "{{ liveness_endpoints | default([]) + [{ - 'name': item, - 'group': 'liveness', - 'token': gatus_push_tokens[item], - 'heartbeat': '11m'}] }}" - loop: "{{ monitored }}" - - - name: Build the disk endpoint list - ansible.builtin.set_fact: - disk_endpoints: "{{ disk_endpoints | default([]) + [{ - 'name': item, - 'group': 'disk', - 'token': gatus_push_tokens[item], - 'heartbeat': '7h'}] }}" - loop: "{{ monitored }}" - - - name: Register liveness endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: liveness - gatus_endpoint_external: "{{ liveness_endpoints }}" - - - name: Register disk endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: disk - gatus_endpoint_external: "{{ disk_endpoints }}" - - - name: Register the hypervisor endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: hypervisor - gatus_endpoint_external: - - name: cpu - group: hypervisor - token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "11m" - - name: zfs - group: hypervisor - token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "7h" - - name: ups - group: hypervisor - token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "11m" - -- name: Deploy host liveness and disk checks - hosts: managed - become: yes - vars: - gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" - host_token: "{{ gatus_push_tokens[inventory_hostname] }}" - - tasks: - - name: Is the host up? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: liveness - healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}" - healthcheck_check: liveness - healthcheck_interval: "5min" - healthcheck_boot_delay: "1min" - healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external" - healthcheck_push_token: "{{ host_token }}" - - - name: Is the disk packed? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: disk-usage - healthcheck_description: "Disk usage for {{ inventory_hostname }}" - healthcheck_check: disk-usage - # Every 6h rather than daily. Disk usage itself moves slowly, but the - # heartbeat can only be as tight as the push frequency - a daily push - # forces a >24h window, and a stuck check then hides for a day and a - # half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the - # hosts so twelve boxes do not all report in the same second. - healthcheck_on_calendar: "*-*-* 00/6:00:00" - healthcheck_randomized_delay: "900" - healthcheck_boot_delay: "5min" - healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external" - healthcheck_push_token: "{{ host_token }}" - -- name: Deploy the hypervisor-only checks - hosts: hypervisor - become: yes - vars: - gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - host_token: "{{ gatus_push_tokens[inventory_hostname] }}" - - tasks: - - name: Is the CPU hot? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: cpu-temp - healthcheck_description: "CPU temperature for {{ inventory_hostname }}" - healthcheck_check: cpu-temp - healthcheck_packages: [curl, lm-sensors] - healthcheck_interval: "5min" - healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external" - healthcheck_push_token: "{{ host_token }}" - - - name: Is ZFS broken? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: zfs-health - healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}" - healthcheck_check: zfs-health - healthcheck_packages: [curl, jq] - healthcheck_zfs_pool: "{{ zfs_pool_name }}" - healthcheck_on_calendar: "*-*-* 00/6:20:00" - healthcheck_boot_delay: "10min" - healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external" - healthcheck_push_token: "{{ host_token }}" - - - name: Is the UPS online? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: ups-status - healthcheck_description: "UPS mains status for {{ ups_name }}" - healthcheck_check: ups-status - healthcheck_ups_name: "{{ ups_name }}" - healthcheck_interval: "5min" - healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external" - healthcheck_push_token: "{{ host_token }}" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml deleted file mode 100644 index ab564a1..0000000 --- a/ansible/infra/401_service_monitoring.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -# Is each systemd-deployed service actually running? -# -# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before -# it alarms, so a reboot or a slow check does not page anyone, but a host that -# stops reporting does. -# -# This closes the gap that let a real bug run unnoticed: a backup script left -# forgejo, lnbits, headscale and memos stopped, and NOTHING caught it. The dumps -# exited 0, the artefacts were correct, the deploy said failed=0, and liveness -# only proves the HOST is up - not that anything on it is serving. -# -# One endpoint PER UNIT, not per host. A host running four services needs four -# endpoints, or a single red light says "something on vipy is down" without -# saying which - and that is the question you actually have at 3am. But only ONE -# timer per host: the check iterates that host's units and pushes a result for -# each, the same way check-backups.sh reports per source. Four units on vipy -# would otherwise mean four scripts, four services and four timers. -# -# Which units each host runs is in host_vars//main.yml as -# monitored_services, because "what runs here" is a property of the machine. -# -# Keys are host-qualified because unit names collide - caddy runs on four -# machines. Gatus computes sanitize(group)_sanitize(name), so group "services" -# and name "vipy/caddy" give services_vipy-caddy. - -# ───────────────────────────────────────────────────────────────────────────── -# Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the -# host play above takes minutes, so every endpoint exists before its first push. -# ───────────────────────────────────────────────────────────────────────────── -# ───────────────────────────────────────────────────────────────────────────── -# Alerting thresholds, and why they differ by check type. -# -# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a -# different amount of wall-clock time per check: -# -# push/heartbeat endpoints a failure is produced once per heartbeat window -# pulled endpoints a failure is produced once per interval -# -# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day -# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m -# window on a 5-minute push is precisely "one missed push forgiven"; stacking a -# threshold of 3 on top triples a tolerance that was already chosen. -# -# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints -# have no built-in tolerance, so the threshold is where it belongs for them. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the service checks with Gatus - hosts: observability - become: yes - - tasks: - # Two plain steps rather than one clever expression: first collect which - # units each host declares, then flatten that into endpoints. - - name: Collect the units each host declares - ansible.builtin.set_fact: - host_units: "{{ host_units | default([]) + [{'host': item, 'units': hostvars[item].monitored_services}] }}" - loop: "{{ groups['managed'] | sort }}" - when: hostvars[item].monitored_services | default([]) | length > 0 - - - name: Build one endpoint per unit - ansible.builtin.set_fact: - service_endpoints: "{{ service_endpoints | default([]) + [{ - 'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1, - 'group': 'services', - 'token': gatus_push_tokens[item.0.host], - 'heartbeat': '11m'}] }}" - loop: "{{ host_units | subelements('units') }}" - - - name: Register the service endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: services - gatus_endpoint_external: "{{ service_endpoints }}" - -- name: Monitor systemd services on every host that has them - hosts: managed - become: yes - vars: - gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" - - tasks: - - name: Is every deployed service running? - ansible.builtin.include_role: - name: healthcheck - vars: - healthcheck_name: service-health - healthcheck_description: "systemd services on {{ inventory_hostname }}" - healthcheck_check: systemd-units - healthcheck_units: "{{ monitored_services }}" - healthcheck_units_key_prefix: "services_{{ host_key }}" - healthcheck_interval: "5min" - healthcheck_boot_delay: "2min" - # The per-unit results go to keys under this collection; the role's own - # single-result push is unused here, so only the base is set. - healthcheck_push_base: "{{ gatus_api }}" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - when: monitored_services | default([]) | length > 0 diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml deleted file mode 100644 index d924468..0000000 --- a/ansible/infra/402_public_monitoring.yml +++ /dev/null @@ -1,141 +0,0 @@ ---- -# Domain expiry, DNS correctness, and public endpoint reachability. -# -# These are the first checks in the estate that PULL rather than push, and that -# is the right way round for them: all three are about how the outside world -# sees us, so they must be measured from outside. Gatus polls from the -# observability host and needs nothing installed anywhere else - there is no -# script, no timer and no token, because nothing is reporting in. -# -# That also means these have no heartbeat. A heartbeat answers "did the thing -# that was supposed to report in do so"; when Gatus does the checking itself, -# failure is immediate and self-evident. - -- name: Register the public-facing checks with Gatus - hosts: observability - become: yes - - vars: - # Expected A records, derived from inventory rather than written down again. - # The estate's recurring bug is an address recorded in a second place and - # then left behind when the machine moved, so the check asserts against - # ansible_host - if a box is renumbered, inventory is the one edit. - dns_records: - - {sub: "{{ subdomains.gatus }}", host: monitoring} - - {sub: "{{ subdomains.headscale }}", host: spacey} - - {sub: "{{ subdomains.vaultwarden }}", host: vipy} - - {sub: "{{ subdomains.forgejo }}", host: vipy} - - {sub: "{{ subdomains.lnbits }}", host: vipy} - - {sub: "{{ subdomains.ntfy_emergency_app }}", host: vipy} - - {sub: "{{ subdomains.personal_blog }}", host: vipy} - - {sub: "{{ subdomains.memos }}", host: vipy} - - {sub: "{{ subdomains.mempool }}", host: vipy} - - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - - # A public resolver on purpose: this must test what the internet sees, not - # what a local cache or the tailnet's MagicDNS happens to answer. - dns_resolver: "1.1.1.1" - - # Expected status per site, checked live before being written down. - # 401 is the CORRECT answer for the two behind basic auth - asserting 200 - # there would go green precisely when the auth broke. - public_sites: - - {name: gatus, sub: "{{ subdomains.gatus }}", path: "/", status: 401} - - {name: headscale, sub: "{{ subdomains.headscale }}", path: "/health", status: 200} - - {name: vaultwarden, sub: "{{ subdomains.vaultwarden }}", path: "/", status: 200} - - {name: forgejo, sub: "{{ subdomains.forgejo }}", path: "/", status: 200} - - {name: lnbits, sub: "{{ subdomains.lnbits }}", path: "/", status: 200} - - {name: avisame, sub: "{{ subdomains.ntfy_emergency_app }}", path: "/", status: 200} - - {name: blog, sub: "{{ subdomains.personal_blog }}", path: "/", status: 200} - - {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200} - - {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200} - - {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401} - - # Ports published from the edge host by socket_proxy. - public_tcp: - - {name: bitcoin-p2p, host: vipy, port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}"} - - {name: fulcrum-ssl, host: vipy, port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}"} - - {name: datum-stratum, host: vipy, port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}"} - - tasks: - # ── Domain expiry ──────────────────────────────────────────────────────── - # Each domain needs a URL SCHEME: Gatus derives the endpoint type from the - # prefix (endpoint.Type()), so a bare "example.com" is UNKNOWN and the whole - # config is rejected. No status is asserted, only the WHOIS/RDAP expiry, so - # whatever the apex serves - a real site, or the registrar's parking page - - # is irrelevant. - # - # 24h, and upstream enforces a 5m minimum for DOMAIN_EXPIRATION anyway - # because it uses a free whois service that must not be hammered. - # 336h = 14 days of runway, because renewal is a manual act at the registrar. - - name: Build the domain endpoints - ansible.builtin.set_fact: - domain_endpoints: "{{ domain_endpoints | default([]) + [{ - 'name': item, - 'group': 'domain', - 'url': 'https://' ~ item, - 'interval': '24h', - 'conditions': ['[DOMAIN_EXPIRATION] > 336h'], - 'alerts': [{'type': 'signal', 'failure-threshold': 1, - 'success-threshold': 1, 'send-on-resolved': true, - 'minimum-reminder-interval': '168h'}]}] }}" - loop: "{{ monitored_domains }}" - - # ── DNS ────────────────────────────────────────────────────────────────── - # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The - # domain check stays at 24h because it does a WHOIS/RDAP lookup against a - # free service. Alert on the first failure - at a 6h interval, waiting for - # three would be nearly a day. - - name: Build the DNS endpoints - ansible.builtin.set_fact: - dns_endpoints: "{{ dns_endpoints | default([]) + [{ - 'name': item.sub ~ '.' ~ root_domain, - 'group': 'dns', - 'url': dns_resolver, - 'interval': '6h', - 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, - 'conditions': ['[DNS_RCODE] == NOERROR', - '[BODY] == ' ~ hostvars[item.host].ansible_host], - 'alerts': [{'type': 'signal', 'failure-threshold': 1, - 'success-threshold': 1, 'send-on-resolved': true, - 'minimum-reminder-interval': '24h'}]}] }}" - loop: "{{ dns_records }}" - - # ── Public HTTP ────────────────────────────────────────────────────────── - # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has - # no heartbeat window, so unlike the push checks the tolerance has to live - # in the threshold - and one failed poll of a public site is usually a blip. - - name: Build the public HTTP endpoints - ansible.builtin.set_fact: - http_endpoints: "{{ http_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'public', - 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, - 'interval': '5m', - 'conditions': ['[STATUS] == ' ~ item.status, - '[CERTIFICATE_EXPIRATION] > 168h'], - 'alerts': [{'type': 'signal', 'failure-threshold': 3, - 'success-threshold': 2, 'send-on-resolved': true, - 'minimum-reminder-interval': '6h'}]}] }}" - loop: "{{ public_sites }}" - - # ── Public TCP ─────────────────────────────────────────────────────────── - - name: Build the public TCP endpoints - ansible.builtin.set_fact: - tcp_endpoints: "{{ tcp_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'public', - 'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port, - 'interval': '5m', - 'conditions': ['[CONNECTED] == true'], - 'alerts': [{'type': 'signal', 'failure-threshold': 3, - 'success-threshold': 2, 'send-on-resolved': true, - 'minimum-reminder-interval': '6h'}]}] }}" - loop: "{{ public_tcp }}" - - - name: Register the public-facing endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_name: public - gatus_endpoint_pulled: "{{ domain_endpoints + dns_endpoints + http_endpoints + tcp_endpoints }}" diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml deleted file mode 100644 index c92b767..0000000 --- a/ansible/infra/403_service_probe_registration.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -# The per-service health probes. -# -# These are NOT the same thing as the systemd checks in infra/401. Those answer -# "is the unit running"; these answer "does the service actually respond" - an -# RPC call to bitcoind, a TCP connect to Fulcrum's Electrum port, an HTTP fetch -# from the Mempool backend. A process can be perfectly `active` and useless, -# which is precisely the gap these close. -# -# The checks themselves live in each service's own role, deployed by that -# service's playbook. This play only registers where they report, because the -# endpoints must exist in Gatus before the first push arrives. -# -# They used to push to Uptime Kuma. The scripts now POST with a bearer token -# instead of GETting ?status=up, and each host uses its own token. - -# ───────────────────────────────────────────────────────────────────────────── -# Alerting thresholds, and why they differ by check type. -# -# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a -# different amount of wall-clock time per check: -# -# push/heartbeat endpoints a failure is produced once per heartbeat window -# pulled endpoints a failure is produced once per interval -# -# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day -# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m -# window on a 5-minute push is precisely "one missed push forgiven"; stacking a -# threshold of 3 on top triples a tolerance that was already chosen. -# -# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints -# have no built-in tolerance, so the threshold is where it belongs for them. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the per-service probes with Gatus - hosts: observability - become: yes - - vars: - probes: - - {name: bitcoin-knots, host: knots_box_local} - - {name: datum-gateway, host: knots_box_local} - - {name: fulcrum, host: fulcrum_box_local} - - {name: phoenixd, host: vipy} - - {name: forgejo-runner, host: forgejo_runner_local} - - {name: mempool-mariadb, host: mempool_box_local} - - {name: mempool-backend, host: mempool_box_local} - - {name: mempool-frontend, host: mempool_box_local} - - tasks: - - name: Build the probe endpoint list - ansible.builtin.set_fact: - probe_endpoints: "{{ probe_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'probe', - 'token': gatus_push_tokens[item.host], - 'heartbeat': '11m'}] }}" - loop: "{{ probes }}" - - - name: Register the probe endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h - gatus_endpoint_name: probes - gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/infra/409_remove_legacy_monitoring.yml b/ansible/infra/409_remove_legacy_monitoring.yml deleted file mode 100644 index 8f57e46..0000000 --- a/ansible/infra/409_remove_legacy_monitoring.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -# Remove the Uptime-Kuma-era monitoring that 400/401/402 replaced. -# -# Deleting the playbooks that installed these is NOT enough: the units are on -# the hosts, enabled, and keep firing regardless of what the repo says. Two of -# them still push to https://uptime.contrapeso.xyz every 15 minutes. A playbook -# that is deleted without a cleanup leaves its output running forever, with -# nothing in the repo left to explain it. -# -# What replaced what, all verified against the deployed scripts before removal: -# -# disk-usage-monitor -> disk-usage-healthcheck (infra/400) -# The old one checked ONLY "/" at 80%. The replacement walks every real -# filesystem, excluding tmpfs/devtmpfs/squashfs/overlay, at 85%. Strictly -# more coverage, so nothing is lost. -# -# system-healthcheck -> liveness-healthcheck (infra/400) -# The old script computed uptime and pushed. That is exactly a liveness -# heartbeat and nothing more. -# -# nodito-cpu-temp-monitor -> cpu-temp-healthcheck (infra/400) -# zfs-health-monitor -> zfs-health-healthcheck (infra/400) -# The ZFS check logic was ported verbatim - same five conditions - so only -# the reporting transport changed. -# -# NOT removed, because they are not monitoring: -# zfs-monthly-scrub.{timer,service} the actual scrub (infra/nodito/32) -# pull-backups, check-backups the backup machinery (playbooks/backups) -# -# This play is idempotent and kept permanently rather than run once and deleted: -# on a host that never had these it does nothing, and it guarantees a rebuilt or -# restored machine cannot quietly bring them back. - -- name: Remove the legacy Uptime Kuma monitoring units - hosts: managed - become: yes - - vars: - legacy_units: - - disk-usage-monitor - - system-healthcheck - - nodito-cpu-temp-monitor - - zfs-health-monitor - legacy_dirs: - - /opt/disk-monitoring - - /opt/system-healthcheck - - /opt/nodito-monitoring - - /opt/zfs-monitoring - - tasks: - - name: Find which legacy units exist here - ansible.builtin.stat: - path: "/etc/systemd/system/{{ item.0 }}.{{ item.1 }}" - register: legacy_unit_files - loop: "{{ legacy_units | product(['timer', 'service']) | list }}" - - # Stop and disable BEFORE deleting the unit file: systemd cannot disable a - # unit whose file has already gone, which would leave a dangling symlink in - # multi-user.target.wants and a warning on every daemon-reload. - - name: Stop and disable the legacy units - ansible.builtin.systemd: - name: "{{ item.item.0 }}.{{ item.item.1 }}" - state: stopped - enabled: no - loop: "{{ legacy_unit_files.results }}" - loop_control: - label: "{{ item.item.0 }}.{{ item.item.1 }}" - when: item.stat.exists - failed_when: false - - - name: Remove the legacy unit files - ansible.builtin.file: - path: "/etc/systemd/system/{{ item.item.0 }}.{{ item.item.1 }}" - state: absent - loop: "{{ legacy_unit_files.results }}" - loop_control: - label: "{{ item.item.0 }}.{{ item.item.1 }}" - when: item.stat.exists - - - name: Reload systemd - ansible.builtin.systemd: - daemon_reload: yes - - - name: Remove the legacy monitoring scripts and their logs - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: "{{ legacy_dirs }}" - - # An orphan predating all of this: mode 0644, not executable, referenced by - # no unit and no cron entry, pushing to a Kuma monitor. Superseded by - # ups-status-healthcheck. - - name: Remove the orphaned hand-written UPS heartbeat - ansible.builtin.file: - path: /usr/local/bin/ups-heartbeat.sh - state: absent - - - name: Confirm nothing still pushes to Uptime Kuma - ansible.builtin.shell: >- - grep -rl "uptime.contrapeso.xyz" /etc/systemd/system /usr/local/bin /opt 2>/dev/null || true - register: kuma_refs - changed_when: false - - - name: Report any remaining references - ansible.builtin.debug: - msg: >- - {{ 'clean - nothing references Uptime Kuma' - if kuma_refs.stdout | trim | length == 0 - else 'STILL REFERENCING KUMA: ' ~ kuma_refs.stdout_lines | join(', ') }} diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml new file mode 100644 index 0000000..de02f53 --- /dev/null +++ b/ansible/infra/410_disk_usage_alerts.yml @@ -0,0 +1,330 @@ +- name: Deploy Disk Usage Monitoring + hosts: all + become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml + - ../infra_secrets.yml + + vars: + disk_usage_threshold_percent: 80 + disk_check_interval_minutes: 15 + monitored_mount_point: "/" + monitoring_script_dir: /opt/disk-monitoring + monitoring_script_path: "{{ monitoring_script_dir }}/disk_usage_monitor.sh" + log_file: "{{ monitoring_script_dir }}/disk_usage_monitor.log" + systemd_service_name: disk-usage-monitor + # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname and mount point + set_fact: + monitor_name: "disk-usage-{{ host_name.stdout }}-{{ monitored_mount_point | replace('/', 'root') }}" + monitor_friendly_name: "Disk Usage: {{ host_name.stdout }} ({{ monitored_mount_point }})" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': True, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when usage exceeds {{ disk_usage_threshold_percent }}%" + "{{ (disk_check_interval_minutes * 60) + 60 }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_disk_usage_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for disk monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create disk usage monitoring script + copy: + dest: "{{ monitoring_script_path }}" + content: | + #!/bin/bash + + # Disk Usage Monitoring Script + # Monitors disk usage and sends alerts to Uptime Kuma + # Mode: "No news is good news" - only sends alerts when disk usage is HIGH + + LOG_FILE="{{ log_file }}" + USAGE_THRESHOLD="{{ disk_usage_threshold_percent }}" + UPTIME_KUMA_URL="{{ uptime_kuma_disk_usage_push_url }}" + MOUNT_POINT="{{ monitored_mount_point }}" + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to get disk usage percentage + get_disk_usage() { + local mount_point="$1" + local usage="" + + # Get disk usage percentage (without % sign) + usage=$(df -h "$mount_point" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}') + + if [ -z "$usage" ]; then + log_message "ERROR: Could not read disk usage for $mount_point" + return 1 + fi + + echo "$usage" + } + + # Function to get disk usage details + get_disk_details() { + local mount_point="$1" + df -h "$mount_point" 2>/dev/null | awk 'NR==2 {print "Used: "$3" / Total: "$2" ("$5" full)"}' + } + + # Function to send alert to Uptime Kuma when disk usage exceeds threshold + # With upside-down mode enabled, sending status=up will trigger an alert + send_uptime_kuma_alert() { + local usage="$1" + local details="$2" + local message="DISK FULL WARNING: ${MOUNT_POINT} is ${usage}% full (Threshold: ${USAGE_THRESHOLD}%) - ${details}" + + log_message "ALERT: $message" + + # Send push notification to Uptime Kuma with status=up + # In upside-down mode, status=up is treated as down/alert + response=$(curl -s -w "\n%{http_code}" -G \ + --data-urlencode "status=up" \ + --data-urlencode "msg=$message" \ + "$UPTIME_KUMA_URL" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" + else + log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" + fi + } + + # Main monitoring logic + main() { + log_message "Starting disk usage check for $MOUNT_POINT" + + # Get current disk usage + current_usage=$(get_disk_usage "$MOUNT_POINT") + + if [ $? -ne 0 ] || [ -z "$current_usage" ]; then + log_message "ERROR: Could not read disk usage" + exit 1 + fi + + # Get disk details + disk_details=$(get_disk_details "$MOUNT_POINT") + + log_message "Current disk usage: ${current_usage}% - $disk_details" + + # Check if usage exceeds threshold + if [ "$current_usage" -gt "$USAGE_THRESHOLD" ]; then + log_message "WARNING: Disk usage ${current_usage}% exceeds threshold ${USAGE_THRESHOLD}%" + send_uptime_kuma_alert "$current_usage" "$disk_details" + else + log_message "Disk usage is within normal range - no alert needed (no news is good news)" + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for disk usage monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.service" + content: | + [Unit] + Description=Disk Usage Monitor + After=network.target + + [Service] + Type=oneshot + ExecStart={{ monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for disk usage monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" + content: | + [Unit] + Description=Run Disk Usage Monitor every {{ disk_check_interval_minutes }} minute(s) + Requires={{ systemd_service_name }}.service + + [Timer] + OnBootSec={{ disk_check_interval_minutes }}min + OnUnitActiveSec={{ disk_check_interval_minutes }}min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start disk usage monitoring timer + systemd: + name: "{{ systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test disk usage monitoring script + command: "{{ monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "Disk usage monitoring script failed to execute properly" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml new file mode 100644 index 0000000..2580ff0 --- /dev/null +++ b/ansible/infra/420_system_healthcheck.yml @@ -0,0 +1,312 @@ +- name: Deploy System Healthcheck Monitoring + hosts: all + become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml + - ../infra_secrets.yml + + vars: + healthcheck_interval_seconds: 60 # Send healthcheck every 60 seconds (1 minute) + healthcheck_timeout_seconds: 90 # Uptime Kuma should alert if no ping received within 90s + healthcheck_retries: 1 # Number of retries before alerting + monitoring_script_dir: /opt/system-healthcheck + monitoring_script_path: "{{ monitoring_script_dir }}/system_healthcheck.sh" + log_file: "{{ monitoring_script_dir }}/system_healthcheck.log" + systemd_service_name: system-healthcheck + # Uptime Kuma configuration (auto-configured from services_config.yml and infra_secrets.yml) + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "system-healthcheck-{{ host_name.stdout }}" + monitor_friendly_name: "System Healthcheck: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_healthcheck_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_healthcheck_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Regular healthcheck ping every {{ healthcheck_interval_seconds }}s" + "{{ healthcheck_timeout_seconds }}" + "{{ healthcheck_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_healthcheck_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for healthcheck monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create system healthcheck script + copy: + dest: "{{ monitoring_script_path }}" + content: | + #!/bin/bash + + # System Healthcheck Script + # Sends regular heartbeat pings to Uptime Kuma + # This ensures the system is running and able to communicate + + LOG_FILE="{{ log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_healthcheck_push_url }}" + HOSTNAME=$(hostname) + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to send healthcheck ping to Uptime Kuma + send_healthcheck() { + local uptime_seconds=$(awk '{print int($1)}' /proc/uptime) + local uptime_days=$((uptime_seconds / 86400)) + local uptime_hours=$(((uptime_seconds % 86400) / 3600)) + local uptime_minutes=$(((uptime_seconds % 3600) / 60)) + + local message="System healthy - Uptime: ${uptime_days}d ${uptime_hours}h ${uptime_minutes}m" + + log_message "Sending healthcheck ping: $message" + + # Send push notification to Uptime Kuma with status=up + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Healthcheck ping sent successfully (HTTP $http_code)" + else + log_message "ERROR: Failed to send healthcheck ping (HTTP $http_code)" + return 1 + fi + } + + # Main healthcheck logic + main() { + log_message "Starting system healthcheck for $HOSTNAME" + + # Send healthcheck ping + if send_healthcheck; then + log_message "Healthcheck completed successfully" + else + log_message "ERROR: Healthcheck failed" + exit 1 + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for system healthcheck + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.service" + content: | + [Unit] + Description=System Healthcheck Monitor + After=network.target + + [Service] + Type=oneshot + ExecStart={{ monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for system healthcheck + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" + content: | + [Unit] + Description=Run System Healthcheck every minute + Requires={{ systemd_service_name }}.service + + [Timer] + OnBootSec=30sec + OnUnitActiveSec={{ healthcheck_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start system healthcheck timer + systemd: + name: "{{ systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test system healthcheck script + command: "{{ monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "System healthcheck script failed to execute properly" + + - name: Display monitor information + debug: + msg: | + ✓ System healthcheck monitoring deployed successfully! + + Monitor Name: {{ monitor_friendly_name }} + Monitor Group: {{ uptime_kuma_monitor_group }} + Healthcheck Interval: Every {{ healthcheck_interval_seconds }} seconds (1 minute) + Timeout: {{ healthcheck_timeout_seconds }} seconds (90s) + Retries: {{ healthcheck_retries }} + + The system will send a heartbeat ping every minute. + Uptime Kuma will alert if no ping is received within 90 seconds (with 1 retry). + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_healthcheck_monitor.py + state: absent + delegate_to: localhost + become: no + diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml new file mode 100644 index 0000000..3b87102 --- /dev/null +++ b/ansible/infra/430_cpu_temp_alerts.yml @@ -0,0 +1,316 @@ +- name: Deploy CPU Temperature Monitoring + hosts: nodito_host + become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml + - ../infra_secrets.yml + + vars: + temp_threshold_celsius: 80 + temp_check_interval_minutes: 1 + monitoring_script_dir: /opt/nodito-monitoring + monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" + log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" + systemd_service_name: nodito-cpu-temp-monitor + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "cpu-temp-{{ host_name.stdout }}" + monitor_friendly_name: "CPU Temperature: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma CPU temperature monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_cpu_temp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + ntfy_topic = sys.argv[8] if len(sys.argv) > 8 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + monitors = api.get_monitors() + notifications = api.get_notifications() + + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name=group_name) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': True, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + api.edit_monitor(existing_monitor['id'], **monitor_data) + else: + api.add_monitor(**monitor_data) + + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_cpu_temp_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when temperature exceeds {{ temp_threshold_celsius }}°C" + "{{ (temp_check_interval_minutes * 60) + 60 }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_cpu_temp_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for temperature monitoring + package: + name: + - lm-sensors + - curl + - jq + - bc + state: present + + - name: Create monitoring script directory + file: + path: "{{ monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create CPU temperature monitoring script + copy: + dest: "{{ monitoring_script_path }}" + content: | + #!/bin/bash + + # CPU Temperature Monitoring Script + # Monitors CPU temperature and sends alerts to Uptime Kuma + + LOG_FILE="{{ log_file }}" + TEMP_THRESHOLD="{{ temp_threshold_celsius }}" + UPTIME_KUMA_URL="{{ uptime_kuma_cpu_temp_push_url }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + get_cpu_temp() { + local temp="" + + if command -v sensors >/dev/null 2>&1; then + temp=$(sensors 2>/dev/null | grep -E "Core 0|Package id 0|Tdie|Tctl" | head -1 | grep -oE '[0-9]+\.[0-9]+°C' | grep -oE '[0-9]+\.[0-9]+') + fi + + if [ -z "$temp" ] && [ -f /sys/class/thermal/thermal_zone0/temp ]; then + temp=$(cat /sys/class/thermal/thermal_zone0/temp) + temp=$(echo "scale=1; $temp/1000" | bc -l 2>/dev/null || echo "$temp") + fi + + if [ -z "$temp" ] && command -v acpi >/dev/null 2>&1; then + temp=$(acpi -t 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1) + fi + + echo "$temp" + } + + send_uptime_kuma_alert() { + local temp="$1" + local message="CPU Temperature Alert: ${temp}°C (Threshold: ${TEMP_THRESHOLD}°C)" + + log_message "ALERT: $message" + + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/°/%C2%B0/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g') + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Alert sent successfully to Uptime Kuma (HTTP $http_code)" + else + log_message "ERROR: Failed to send alert to Uptime Kuma (HTTP $http_code)" + fi + } + + main() { + log_message "Starting CPU temperature check" + + current_temp=$(get_cpu_temp) + + if [ -z "$current_temp" ]; then + log_message "ERROR: Could not read CPU temperature" + exit 1 + fi + + log_message "Current CPU temperature: ${current_temp}°C" + + if (( $(echo "$current_temp > $TEMP_THRESHOLD" | bc -l) )); then + log_message "WARNING: CPU temperature ${current_temp}°C exceeds threshold ${TEMP_THRESHOLD}°C" + send_uptime_kuma_alert "$current_temp" + else + log_message "CPU temperature is within normal range" + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for CPU temperature monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.service" + content: | + [Unit] + Description=CPU Temperature Monitor + After=network.target + + [Service] + Type=oneshot + ExecStart={{ monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for CPU temperature monitoring + copy: + dest: "/etc/systemd/system/{{ systemd_service_name }}.timer" + content: | + [Unit] + Description=Run CPU Temperature Monitor every {{ temp_check_interval_minutes }} minute(s) + Requires={{ systemd_service_name }}.service + + [Timer] + OnBootSec={{ temp_check_interval_minutes }}min + OnUnitActiveSec={{ temp_check_interval_minutes }}min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start CPU temperature monitoring timer + systemd: + name: "{{ systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test CPU temperature monitoring script + command: "{{ monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "CPU temperature monitoring script failed to execute properly" + + - name: Display monitoring configuration + debug: + msg: + - "CPU Temperature Monitoring configured successfully" + - "Temperature threshold: {{ temp_threshold_celsius }}°C" + - "Check interval: {{ temp_check_interval_minutes }} minute(s)" + - "Monitor Name: {{ monitor_friendly_name }}" + - "Monitor Group: {{ uptime_kuma_monitor_group }}" + - "Uptime Kuma Push URL: {{ uptime_kuma_cpu_temp_push_url }}" + - "Monitoring script: {{ monitoring_script_path }}" + - "Systemd Service: {{ systemd_service_name }}.service" + - "Systemd Timer: {{ systemd_service_name }}.timer" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_cpu_temp_monitor.py + state: absent + delegate_to: localhost + become: no + diff --git a/ansible/infra/900_install_rsync.yml b/ansible/infra/900_install_rsync.yml index b690009..c0b7318 100644 --- a/ansible/infra/900_install_rsync.yml +++ b/ansible/infra/900_install_rsync.yml @@ -1,5 +1,7 @@ - name: Install rsync - hosts: managed + hosts: all + vars_files: + - ../infra_vars.yml become: true tasks: diff --git a/ansible/infra/910_docker_playbook.yml b/ansible/infra/910_docker_playbook.yml index 62b7147..f137b6a 100644 --- a/ansible/infra/910_docker_playbook.yml +++ b/ansible/infra/910_docker_playbook.yml @@ -1,5 +1,5 @@ - name: Install Docker and Docker Compose on Debian 12 - hosts: managed + hosts: all become: yes tasks: diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 4decb5d..8d06d44 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -1,10 +1,14 @@ - name: Join machine to headscale mesh network - hosts: managed + hosts: all become: yes + vars_files: + - ../infra_vars.yml + - ../services_config.yml vars: headscale_host_name: "spacey" headscale_subdomain: "{{ subdomains.headscale }}" headscale_domain: "https://{{ headscale_subdomain }}.{{ root_domain }}" + headscale_namespace: "{{ service_settings.headscale.namespace }}" tasks: - name: Set facts for headscale server connection diff --git a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml index 4edab06..02c6679 100644 --- a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml +++ b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml @@ -1,6 +1,8 @@ - name: Bootstrap Nodito SSH Key Access - hosts: hypervisor + hosts: nodito_host become: true + vars_files: + - ../infra_vars.yml tasks: - name: Install sudo package diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml index 0fab184..b0be2ef 100644 --- a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -1,6 +1,8 @@ - name: Switch Proxmox VE from Enterprise to Community Repositories - hosts: hypervisor + hosts: nodito_host become: true + vars_files: + - ../infra_vars.yml tasks: - name: Check for deb822 sources format diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 2efbf8c..cb72328 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -1,6 +1,9 @@ - name: Setup ZFS RAID 1 Pool for Proxmox Storage - hosts: hypervisor + hosts: nodito_host become: true + vars_files: + - ../infra_vars.yml + - nodito_vars.yml tasks: - name: Verify Proxmox VE is running @@ -136,18 +139,17 @@ Config file exists: {{ storage_cfg_file.stat.exists }} Storage check result: {{ storage_exists_check.rc }} Pool exists: {{ zfs_pool_exists.rc == 0 }} + Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }} Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }} - # Registration is add-only on purpose. There used to be a "Remove existing - # storage if it exists" task here that ran `pvesm remove` whenever the - # storage WAS present, paired with an add that only ran when it was ABSENT. - # The two conditions are mutually exclusive, so a real run against a - # correctly-configured hypervisor removed the storage entry backing every VM - # and never put it back. It would also have dropped `mountpoint /var/lib/vz`, - # which the live entry has and which `pvesm add` below does not set. - # - # If the storage entry ever needs its options changed, edit - # /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here. + - name: Remove existing storage if it exists + command: pvesm remove {{ zfs_pool_name }} + register: pvesm_remove_result + failed_when: false + when: + - zfs_pool_exists.rc == 0 + - storage_exists_check.rc == 0 + - name: Add ZFS pool storage to Proxmox using pvesm command: > pvesm add zfspool {{ zfs_pool_name }} @@ -169,56 +171,498 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" -# ───────────────────────────────────────────────────────────────────────────── -# The monthly scrub. -# -# The ZFS HEALTH CHECK that used to share this play is gone: it is now the -# zfs-health check in infra/400_host_monitoring.yml, which carries the same five -# conditions - pool state, device states, resilver in progress, read/write/ -# checksum errors, and errors from the last scan - but reports to Gatus like -# every other host check instead of owning its own push plumbing. -# -# The scrub itself stays here, because it is not monitoring: it is the -# maintenance that gives the health check something true to report. A pool that -# is never scrubbed has no idea whether it is healthy. -# ───────────────────────────────────────────────────────────────────────────── -- name: Schedule the monthly ZFS scrub - hosts: hypervisor +- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs + hosts: nodito become: true vars_files: - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml vars: + zfs_check_interval_seconds: 86400 # 24 hours + zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) + zfs_check_retries: 1 + zfs_monitoring_script_dir: /opt/zfs-monitoring + zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" + zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" + zfs_systemd_health_service_name: zfs-health-monitor zfs_systemd_scrub_service_name: zfs-monthly-scrub + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "zfs-health-{{ host_name.stdout }}" + monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma ZFS health monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_zfs_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma ZFS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_zfs_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" + "{{ zfs_check_timeout_seconds }}" + "{{ zfs_check_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for ZFS monitoring + package: + name: + - curl + - jq + state: present + + - name: Create monitoring script directory + file: + path: "{{ zfs_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create ZFS health monitoring script + copy: + dest: "{{ zfs_monitoring_script_path }}" + content: | + #!/bin/bash + + # ZFS Pool Health Monitoring Script + # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy + # If any issues detected, does NOT send heartbeat (triggers timeout alert) + + LOG_FILE="{{ zfs_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" + POOL_NAME="{{ zfs_pool_name }}" + HOSTNAME=$(hostname) + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to check pool health using JSON output + check_pool_health() { + local pool="$1" + local issues_found=0 + + # Get pool status as JSON + local pool_json + pool_json=$(zpool status -j "$pool" 2>&1) + + if [ $? -ne 0 ]; then + log_message "ERROR: Failed to get pool status for $pool" + log_message " -> $pool_json" + return 1 + fi + + # Check 1: Pool state must be ONLINE + local pool_state + pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') + + if [ "$pool_state" != "ONLINE" ]; then + log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" + issues_found=1 + else + log_message "OK: Pool state is ONLINE" + fi + + # Check 2: Check all vdevs and devices for non-ONLINE states + local bad_states + bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.state? and .state != "ONLINE") | + "\(.name // "unknown"): \(.state)" + ' 2>/dev/null) + + if [ -n "$bad_states" ]; then + log_message "ISSUE: Found devices not in ONLINE state:" + echo "$bad_states" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: All devices are ONLINE" + fi + + # Check 3: Check for resilvering in progress + local scan_function scan_state + scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + + if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then + local resilver_progress + resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') + log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" + issues_found=1 + fi + + # Check 4: Check for read/write/checksum errors on all devices + # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison + local devices_with_errors + devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | + "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" + ' 2>/dev/null) + + if [ -n "$devices_with_errors" ]; then + log_message "ISSUE: Found devices with I/O errors:" + echo "$devices_with_errors" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: No read/write/checksum errors detected" + fi + + # Check 5: Check for scan errors (from last scrub/resilver) + local scan_errors + scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') + + if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then + log_message "ISSUE: Last scan reported $scan_errors errors" + issues_found=1 + else + log_message "OK: No scan errors" + fi + + return $issues_found + } + + # Function to get last scrub info for status message + get_scrub_info() { + local pool="$1" + local pool_json + pool_json=$(zpool status -j "$pool" 2>/dev/null) + + local scan_func scan_state scan_start + scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') + + if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then + echo "scrub in progress (started $scan_start)" + elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then + echo "last scrub: $scan_start" + else + echo "no scrub history" + fi + } + + # Function to send heartbeat to Uptime Kuma + send_heartbeat() { + local message="$1" + + log_message "Sending heartbeat to Uptime Kuma: $message" + + # URL encode the message + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent successfully (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + # Main health check logic + main() { + log_message "==========================================" + log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" + + # Run all health checks + if check_pool_health "$POOL_NAME"; then + # All checks passed - send heartbeat + local scrub_info + scrub_info=$(get_scrub_info "$POOL_NAME") + + local message="Pool $POOL_NAME healthy ($scrub_info)" + send_heartbeat "$message" + + log_message "Health check completed: ALL OK" + exit 0 + else + # Issues found - do NOT send heartbeat (will trigger timeout alert) + log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" + log_message "Uptime Kuma will alert after timeout due to missing heartbeat" + exit 1 + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" + content: | + [Unit] + Description=ZFS Pool Health Monitor + After=zfs.target network.target + + [Service] + Type=oneshot + ExecStart={{ zfs_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for daily ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Pool Health Monitor daily + Requires={{ zfs_systemd_health_service_name }}.service + + [Timer] + OnBootSec=5min + OnUnitActiveSec={{ zfs_check_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + - name: Create systemd service for ZFS monthly scrub - template: - src: templates/zfs-monthly-scrub.service.j2 + copy: dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" + content: | + [Unit] + Description=ZFS Monthly Scrub for {{ zfs_pool_name }} + After=zfs.target + + [Service] + Type=oneshot + ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target owner: root group: root mode: '0644' - name: Create systemd timer for monthly ZFS scrub - template: - src: templates/zfs-monthly-scrub.timer.j2 + copy: dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Scrub on last day of every month at 4:00 AM + Requires={{ zfs_systemd_scrub_service_name }}.service + + [Timer] + OnCalendar=*-*~01 04:00:00 + Persistent=true + + [Install] + WantedBy=timers.target owner: root group: root mode: '0644' - - name: Enable and start the monthly scrub timer + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start ZFS health monitoring timer + systemd: + name: "{{ zfs_systemd_health_service_name }}.timer" + enabled: yes + state: started + + - name: Enable and start ZFS monthly scrub timer systemd: name: "{{ zfs_systemd_scrub_service_name }}.timer" enabled: yes state: started - daemon_reload: yes - - name: Report the scrub schedule + - name: Test ZFS health monitoring script + command: "{{ zfs_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "ZFS health monitoring script failed - check pool health" + + - name: Display monitoring configuration debug: - msg: >- - Monthly scrub of {{ zfs_pool_name }}: - last day of each month at 04:00. - Health is reported separately by the zfs-health check - (infra/400_host_monitoring.yml). + msg: | + ✓ ZFS Pool Health Monitoring deployed successfully! + + Monitor Name: {{ monitor_friendly_name }} + Monitor Group: {{ uptime_kuma_monitor_group }} + Pool Name: {{ zfs_pool_name }} + + Health Check: + - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) + - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) + - Script: {{ zfs_monitoring_script_path }} + - Log: {{ zfs_log_file }} + - Service: {{ zfs_systemd_health_service_name }}.service + - Timer: {{ zfs_systemd_health_service_name }}.timer + + Monthly Scrub: + - Schedule: Last day of month at 4:00 AM + - Service: {{ zfs_systemd_scrub_service_name }}.service + - Timer: {{ zfs_systemd_scrub_service_name }}.timer + + Conditions monitored: + - Pool state (must be ONLINE) + - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) + - Resilver status (alerts if resilvering) + - Read/Write/Checksum errors + - Scrub errors + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_zfs_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index faf93a5..e8f8332 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -1,6 +1,9 @@ - name: Create Proxmox template from Debian cloud image (no VM clone) - hosts: hypervisor + hosts: nodito_host become: true + vars_files: + - ../../infra_vars.yml + - nodito_vars.yml vars: # Defaults (override via vars_files or --extra-vars as needed) diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index 12fe720..02468d5 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -1,33 +1,12 @@ - name: Setup NUT (Network UPS Tools) for CyberPower UPS - hosts: hypervisor + hosts: nodito_host become: true + vars_files: + - ../../infra_vars.yml + - nodito_vars.yml + - nodito_secrets.yml tasks: - # ------------------------------------------------------------------ - # Safety catch - # - # /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by - # hand in January 2026 and carry a working password. host_vars/nodito/vault.yml - # (formerly infra/nodito/nodito_secrets.yml) still holds the literal string - # CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that - # working pair with a placeholder and restart NUT - leaving the hypervisor's - # UPS unmonitored and unable to trigger a clean shutdown on mains loss. - # - # Until the real password is put in the vault, stop here. - # ansible-vault edit host_vars/nodito/vault.yml - # ------------------------------------------------------------------ - - name: Refuse to run with a placeholder UPS password - assert: - that: - - ups_password is defined - - ups_password | length > 0 - - ups_password != "CHANGE_ME_TO_SECURE_PASSWORD" - fail_msg: >- - ups_password is unset or still the placeholder. Applying this play would - overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on - nodito and restart NUT. Put the real password in the vault first: - ansible-vault edit host_vars/nodito/vault.yml - # ------------------------------------------------------------------ # Installation # ------------------------------------------------------------------ @@ -96,45 +75,90 @@ # Configuration files # ------------------------------------------------------------------ - name: Configure NUT mode (standalone) - template: + copy: dest: /etc/nut/nut.conf - src: templates/nut.conf.j2 + content: | + # Managed by Ansible + MODE=standalone owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure UPS device - template: + copy: dest: /etc/nut/ups.conf - src: templates/ups.conf.j2 + content: | + # Managed by Ansible + [{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd to listen on localhost - template: + copy: dest: /etc/nut/upsd.conf - src: templates/upsd.conf.j2 + content: | + # Managed by Ansible + LISTEN 127.0.0.1 3493 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd users - template: + copy: dest: /etc/nut/upsd.users - src: templates/upsd.users.j2 + content: | + # Managed by Ansible + [{{ ups_user }}] + password = {{ ups_password }} + upsmon master owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsmon - template: + copy: dest: /etc/nut/upsmon.conf - src: templates/upsmon.conf.j2 + content: | + # Managed by Ansible + MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + + MINSUPPLIES 1 + SHUTDOWNCMD "/sbin/shutdown -h +0" + POLLFREQ 5 + POLLFREQALERT 5 + HOSTSYNC 15 + DEADTIME 15 + POWERDOWNFLAG /etc/killpower + + # Notifications + NOTIFYMSG ONLINE "UPS %s on line power" + NOTIFYMSG ONBATT "UPS %s on battery" + NOTIFYMSG LOWBATT "UPS %s battery is low" + NOTIFYMSG FSD "UPS %s: forced shutdown in progress" + NOTIFYMSG COMMOK "Communications with UPS %s established" + NOTIFYMSG COMMBAD "Communications with UPS %s lost" + NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" + NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + + # Log all events to syslog + NOTIFYFLAG ONLINE SYSLOG + NOTIFYFLAG ONBATT SYSLOG + NOTIFYFLAG LOWBATT SYSLOG + NOTIFYFLAG FSD SYSLOG + NOTIFYFLAG COMMOK SYSLOG + NOTIFYFLAG COMMBAD SYSLOG + NOTIFYFLAG SHUTDOWN SYSLOG + NOTIFYFLAG REPLBATT SYSLOG owner: root group: nut mode: "0640" @@ -225,11 +249,321 @@ - nut-server - nut-monitor -# The UPS heartbeat play that used to live here is gone. What it deployed - -# /opt/ups-monitoring plus a ups-heartbeat timer - is now the ups-status check -# in infra/400_host_monitoring.yml, which reports to Gatus like every other -# host check instead of carrying its own push plumbing. -# -# This playbook is now purely NUT setup: the driver, upsd, upsmon and the -# shutdown behaviour. Monitoring whether the UPS is on mains is a separate -# concern and belongs with the other host checks. + +- name: Setup UPS Heartbeat Monitoring with Uptime Kuma + hosts: nodito + become: true + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml + - nodito_secrets.yml + + vars: + ups_heartbeat_interval_seconds: 60 + ups_heartbeat_timeout_seconds: 120 + ups_heartbeat_retries: 1 + ups_monitoring_script_dir: /opt/ups-monitoring + ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" + ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" + ups_systemd_service_name: ups-heartbeat + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "ups-{{ host_name.stdout }}" + monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma UPS monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_ups_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + monitors = api.get_monitors() + notifications = api.get_notifications() + + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name=group_name) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + api.edit_monitor(existing_monitor['id'], **monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + api.add_monitor(**monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma UPS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_ups_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" + "{{ ups_heartbeat_timeout_seconds }}" + "{{ ups_heartbeat_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL as fact + set_fact: + uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Install required packages for UPS monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ ups_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create UPS heartbeat monitoring script + copy: + dest: "{{ ups_monitoring_script_path }}" + content: | + #!/bin/bash + + # UPS Heartbeat Monitoring Script + # Sends heartbeat to Uptime Kuma only when UPS is on mains power + # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) + + LOG_FILE="{{ ups_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" + UPS_NAME="{{ ups_name }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + send_heartbeat() { + local message="$1" + + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent: $message (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + main() { + local status charge runtime load + + status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) + + if [ -z "$status" ]; then + log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" + exit 1 + fi + + charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) + runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) + load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) + + if [[ "$status" == *"OL"* ]]; then + local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" + send_heartbeat "$message" + exit 0 + else + log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" + content: | + [Unit] + Description=UPS Heartbeat Monitor + After=network.target nut-monitor.service + + [Service] + Type=oneshot + ExecStart={{ ups_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" + content: | + [Unit] + Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds + Requires={{ ups_systemd_service_name }}.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start UPS heartbeat timer + systemd: + name: "{{ ups_systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test UPS heartbeat script + command: "{{ ups_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "UPS heartbeat script failed - check UPS status and communication" + + - name: Display monitoring configuration + debug: + msg: + - "UPS Monitoring configured successfully" + - "" + - "NUT Configuration:" + - " UPS Name: {{ ups_name }}" + - " UPS Description: {{ ups_desc }}" + - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" + - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" + - "" + - "Uptime Kuma Monitoring:" + - " Monitor Name: {{ monitor_friendly_name }}" + - " Monitor Group: {{ uptime_kuma_monitor_group }}" + - " Push URL: {{ uptime_kuma_ups_push_url }}" + - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" + - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" + - "" + - "Scripts and Services:" + - " Script: {{ ups_monitoring_script_path }}" + - " Log: {{ ups_log_file }}" + - " Service: {{ ups_systemd_service_name }}.service" + - " Timer: {{ ups_systemd_service_name }}.timer" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_ups_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/host_vars/nodito/main.yml b/ansible/infra/nodito/nodito_vars.yml similarity index 52% rename from ansible/host_vars/nodito/main.yml rename to ansible/infra/nodito/nodito_vars.yml index c8176af..c0002f3 100644 --- a/ansible/host_vars/nodito/main.yml +++ b/ansible/infra/nodito/nodito_vars.yml @@ -14,12 +14,7 @@ systemd_service_name: nodito-cpu-temp-monitor # ZFS Pool Configuration zfs_pool_name: "proxmox-tank-1" -# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the -# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently -# replaced and the repo never caught up. Pool creation is guarded by -# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been -# wrong on any disaster-recovery run. -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ" # First disk for RAID 1 mirror +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" @@ -31,15 +26,3 @@ ups_port: auto ups_user: counterweight ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power - -# Systemd services deployed on this host, monitored every 5 minutes. -# -# The fact lives with the machine rather than in a central map, for the same -# reason the cross-host ports do: "what runs here" is a property of the host, -# and a central list is one more thing to forget to update when a service moves. -# -# Only units WE deploy belong here. Distro units (ssh, cron) have their own -# supervision and would be noise. -monitored_services: - - nut-server - - nut-monitor diff --git a/ansible/infra/nodito/templates/nut.conf.j2 b/ansible/infra/nodito/templates/nut.conf.j2 deleted file mode 100644 index 1f8a72f..0000000 --- a/ansible/infra/nodito/templates/nut.conf.j2 +++ /dev/null @@ -1,2 +0,0 @@ -# Managed by Ansible -MODE=standalone diff --git a/ansible/infra/nodito/templates/ups.conf.j2 b/ansible/infra/nodito/templates/ups.conf.j2 deleted file mode 100644 index 0fa0a60..0000000 --- a/ansible/infra/nodito/templates/ups.conf.j2 +++ /dev/null @@ -1,9 +0,0 @@ -# Managed by Ansible -maxretry = 3 - -[{{ ups_name }}] - driver = {{ ups_driver }} - port = {{ ups_port }} - desc = "{{ ups_desc }}" - offdelay = {{ ups_offdelay }} - ondelay = {{ ups_ondelay }} diff --git a/ansible/infra/nodito/templates/upsd.conf.j2 b/ansible/infra/nodito/templates/upsd.conf.j2 deleted file mode 100644 index baddf21..0000000 --- a/ansible/infra/nodito/templates/upsd.conf.j2 +++ /dev/null @@ -1,2 +0,0 @@ -# Managed by Ansible -LISTEN 127.0.0.1 3493 diff --git a/ansible/infra/nodito/templates/upsd.users.j2 b/ansible/infra/nodito/templates/upsd.users.j2 deleted file mode 100644 index 8d43784..0000000 --- a/ansible/infra/nodito/templates/upsd.users.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# Managed by Ansible -[{{ ups_user }}] - password = {{ ups_password }} - upsmon master diff --git a/ansible/infra/nodito/templates/upsmon.conf.j2 b/ansible/infra/nodito/templates/upsmon.conf.j2 deleted file mode 100644 index dc301e1..0000000 --- a/ansible/infra/nodito/templates/upsmon.conf.j2 +++ /dev/null @@ -1,34 +0,0 @@ -# Managed by Ansible -MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master - -MINSUPPLIES 1 -SHUTDOWNCMD "/sbin/shutdown -h +0" -POLLFREQ 5 -POLLFREQALERT 5 -HOSTSYNC 15 -DEADTIME 15 -POWERDOWNFLAG "/etc/killpower" -OFFDURATION 30 -RBWARNTIME 43200 -NOCOMMWARNTIME 300 -FINALDELAY 5 - -# Notifications -NOTIFYMSG ONLINE "UPS %s on line power" -NOTIFYMSG ONBATT "UPS %s on battery" -NOTIFYMSG LOWBATT "UPS %s battery is low" -NOTIFYMSG FSD "UPS %s: forced shutdown in progress" -NOTIFYMSG COMMOK "Communications with UPS %s established" -NOTIFYMSG COMMBAD "Communications with UPS %s lost" -NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" -NOTIFYMSG REPLBATT "UPS %s battery needs replacing" - -# Log all events to syslog -NOTIFYFLAG ONLINE SYSLOG -NOTIFYFLAG ONBATT SYSLOG -NOTIFYFLAG LOWBATT SYSLOG -NOTIFYFLAG FSD SYSLOG -NOTIFYFLAG COMMOK SYSLOG -NOTIFYFLAG COMMBAD SYSLOG -NOTIFYFLAG SHUTDOWN SYSLOG -NOTIFYFLAG REPLBATT SYSLOG diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 deleted file mode 100644 index 04b3514..0000000 --- a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=ZFS Monthly Scrub for {{ zfs_pool_name }} -After=zfs.target - -[Service] -Type=oneshot -ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} -User=root -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 deleted file mode 100644 index 36ad3d7..0000000 --- a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Run ZFS Scrub on last day of every month at 4:00 AM -Requires={{ zfs_systemd_scrub_service_name }}.service - -[Timer] -OnCalendar=*-*~01 04:00:00 -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example new file mode 100644 index 0000000..d539282 --- /dev/null +++ b/ansible/infra_secrets.yml.example @@ -0,0 +1,40 @@ +# Uptime Kuma login credentials +# Used by the disk monitoring playbook to create monitors automatically + +uptime_kuma_username: "admin" +uptime_kuma_password: "your_password_here" + +# ntfy credentials +# Used for notification channel setup in Uptime Kuma + +ntfy_username: "your_ntfy_username" +ntfy_password: "your_ntfy_password" + +# headscale-ui credentials +# Used for HTTP basic authentication via Caddy +# Provide either: +# - headscale_ui_password: plain text password (will be hashed automatically) +# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate) + +headscale_ui_username: "admin" +headscale_ui_password: "your_secure_password_here" +# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password + +bitcoin_rpc_user: "bitcoinrpc" +bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" + +# Mempool MariaDB credentials +# Used by: services/mempool/deploy_mempool_playbook.yml +mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" + +# Forgejo Runner registration token +# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml +# See: services/forgejo-runner/SETUP.md for how to obtain this token +forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE" + +# DATUM Gateway secrets +# Used by: services/datum-gateway/deploy_datum_gateway_playbook.yml +datum_mining_address: "YOUR_BITCOIN_ADDRESS_FOR_BLOCK_REWARDS" +datum_gateway_admin_password: "CHANGE_ME_TO_SECURE_PASSWORD" +datum_dashboard_username: "admin" +datum_dashboard_password_hash: "$2a$14$..." # Generate with: caddy hash-password diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml new file mode 100644 index 0000000..952df93 --- /dev/null +++ b/ansible/infra_vars.yml @@ -0,0 +1,4 @@ +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: contrapeso.xyz diff --git a/ansible/inventory.ini b/ansible/inventory.ini deleted file mode 100644 index c39439d..0000000 --- a/ansible/inventory.ini +++ /dev/null @@ -1,74 +0,0 @@ -[vps] -vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -monitoring ansible_host=64.226.70.190 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - -[nodito_host] -nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - -# Requires the tailnet to be up on the control node. -[nodito_vms] -knots_box_local ansible_host=knots-box lan_ip=192.168.1.135 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -fulcrum_box_local ansible_host=fulcrum-box lan_ip=192.168.1.140 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -mempool_box_local ansible_host=mempool-box lan_ip=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - -# Local connection to laptop: this assumes you're running ansible commands from your personal laptop -[lapy] -localhost ansible_connection=local ansible_user=counterweight - -[arbret] -prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - -[edge] -vipy - -# The group is `observability`, NOT `monitoring` — there is a HOST named -# `monitoring` on line 5, and a group with the same name makes `hosts: monitoring` -# ambiguous. Ansible resolved it to the host and warned: -# [WARNING]: Found both group and host with same name: monitoring -[observability] -monitoring - -[vpn_control] -spacey - -[hypervisor] -nodito - -[bitcoin] -knots_box_local - -[electrum] -fulcrum_box_local - -[mempool] -mempool_box_local - -[memos] -memos_box_local - -[ci_runner] -forgejo_runner_local - -[control] -localhost - -# Every machine Ansible may configure as a server. -# Deliberately EXCLUDES [control] (your laptop) and [arbret]. -[managed:children] -vps -nodito_host -nodito_vms - -# Hosts that run Caddy and therefore have /etc/caddy/sites-enabled. -[caddy:children] -edge -observability -vpn_control - -[backup_store] -small_backups_local diff --git a/ansible/inventory.ini.example b/ansible/inventory.ini.example new file mode 100644 index 0000000..bde96dd --- /dev/null +++ b/ansible/inventory.ini.example @@ -0,0 +1,16 @@ +[vps] +vipy ansible_host=your.services.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key +watchtower ansible_host=your.monitoring.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key +spacey ansible_host=your.headscale.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key + +[nodito_host] +nodito ansible_host=your.proxmox.ip.here ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key ansible_ssh_pass=your_root_password + +[nodito_vms] +# Example node, replace with your VM names and addresses +# memos_box ansible_host=192.168.1.150 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key + +# Local connection to laptop: this assumes you're running ansible commands from your personal laptop +# Make sure to adjust the username +[lapy] +localhost ansible_connection=local ansible_user=your laptop user gpg_recipient=your_email@example.com gpg_key_id=your_gpg_key_id_here \ No newline at end of file diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml deleted file mode 100644 index 0be3807..0000000 --- a/ansible/playbooks/backups.yml +++ /dev/null @@ -1,107 +0,0 @@ -- name: Configure the offsite backup pull - hosts: backup_store - gather_facts: yes - - tasks: - - name: Ensure the box pulls every source on a timer - ansible.builtin.include_role: - name: backup_store - vars: - # check-backups.sh reports one result per source plus one for the store - # itself, so it needs the collection URL and appends each key. - backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" - backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - backup_store_sources: - - name: arbret - source: "arbret@prd-arbret:/opt/arbret/backups/" - retention_days: 90 - - name: headscale - source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" - retention_days: 90 - - name: memos - source: "backup-pull@memos-box:/opt/backups/memos/" - retention_days: 90 - - name: vaultwarden - source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/" - retention_days: 90 - - name: lnbits - source: "backup-pull@prd-vipy:/opt/backups/lnbits/" - retention_days: 90 - - name: forgejo - source: "backup-pull@prd-vipy:/opt/backups/forgejo/" - retention_days: 14 - -# ───────────────────────────────────────────────────────────────────────────── -# Register the backup checks with Gatus. -# -# Two groups on purpose, because they answer different questions and fail for -# different reasons: -# -# backup-dump did the SOURCE produce an artefact? Pushed by each dump right -# after it runs, so a broken dump is visible within minutes. -# backup-store did it ARRIVE, is it fresh, non-zero, plausibly sized, and is -# retention pruning? Pushed by check-backups.sh at 05:30. -# -# The store alone could catch almost everything, because the artefact filename -# carries the source's dump timestamp - a source whose timer died still pulls -# "ok" forever, but the timestamp gives it away. What the source side adds is -# LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it -# cannot tell you whether the dump broke or the pull did. -# -# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` -# deliberately excludes, so nothing of ours runs there. It is store-checked only. -# ───────────────────────────────────────────────────────────────────────────── -- name: Register the backup checks with Gatus - hosts: observability - become: yes - vars: - # Sources we deploy the dump for, and the host each one runs on. - dump_sources: - - {name: headscale, host: spacey} - - {name: memos, host: memos_box_local} - - {name: vaultwarden, host: vipy} - - {name: lnbits, host: vipy} - - {name: forgejo, host: vipy} - store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] - - tasks: - # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter - # than a day plus slack. The store-side check catches the same fault within - # 6h by reading the artefact's dump timestamp out of the filename, so this is - # the slow backstop rather than the primary signal. - - name: Build the dump endpoint list - ansible.builtin.set_fact: - dump_endpoints: "{{ dump_endpoints | default([]) + [{ - 'name': item.name, - 'group': 'backup-dump', - 'token': gatus_push_tokens[item.host], - 'heartbeat': '26h'}] }}" - loop: "{{ dump_sources }}" - - - name: Build the store endpoint list - ansible.builtin.set_fact: - store_endpoints: "{{ store_endpoints | default([]) + [{ - 'name': item, - 'group': 'backup-store', - 'token': gatus_push_tokens['small_backups_local'], - 'heartbeat': '7h'}] }}" - loop: "{{ store_sources }}" - - - name: Register the backup endpoints - ansible.builtin.include_role: - name: gatus_endpoint - vars: - # Push endpoints: the heartbeat window is the tolerance, so alert on - # the first failure rather than waiting for three 7h windows to pass. - gatus_endpoint_default_alerts: - - type: signal - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 12h - gatus_endpoint_name: backups - gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ - 'name': 'pull job', - 'group': 'backup-store', - 'token': gatus_push_tokens['small_backups_local'], - 'heartbeat': '7h'}] }}" diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 16ef1fd..dd7eef4 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,10 +1,11 @@ +--- # Ansible Galaxy Collections Requirements # Install with: ansible-galaxy collection install -r requirements.yml -# No collections are currently required. -# -# lucasheld.uptime_kuma was pinned here but never used — every monitor was created -# by hand-rolled Python instead. Removed 2026-09-11 along with Uptime Kuma itself. -# See archive/uptime_kuma/. +collections: + # Uptime Kuma Ansible Collection + # Used by: infra/41_disk_usage_alerts.yml + # Provides modules to manage Uptime Kuma monitors programmatically + - name: lucasheld.uptime_kuma + version: ">=1.0.0" -collections: [] diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md deleted file mode 100644 index a7794ab..0000000 --- a/ansible/roles/backup_source/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# `backup_source` - -Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a -local directory, prune, on a systemd timer. `small-backups-box` pulls the -directory later (see `backup_store`). - -Modelled on `prd-arbret`, which has been doing exactly this correctly since -before the rest of the estate was migrated. - -## Usage - -```yaml -- ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: headscale - backup_source_description: "Headscale" - backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" - backup_source_stop_service: headscale - backup_source_retention_days: 7 -``` - -Produces `/opt/backups/headscale/headscale_.tar.gz.age`, -`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`. - -## Why the source encrypts, not the destination - -`age -r ` is asymmetric and the host holds only the **public** key, so -a compromised host cannot read its own backups — or anyone else's. The scripts -this replaces encrypted with GPG *on the laptop, after the data had already -crossed the network*, which protects the artefact at rest but not in transit. - -The matching identity lives only on lapy and is escrowed. **Lose it and every -artefact everywhere becomes noise**, including arbret's. - -## `backup_source_dump_command` writes to STDOUT - -The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` -with relative paths in `tar` rather than absolute ones: it avoids tar's "removing -leading /" and makes the restore target explicit. - -## Services that are not systemd - -`backup_source_stop_service` runs `systemctl stop/start`. For anything else, -give the pair explicitly — vaultwarden is a docker compose stack, so -`systemctl stop vaultwarden` silently does nothing: - -```yaml -backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" -backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" -``` - -The same EXIT trap wraps both forms. The assert refuses a stop command without a -matching start command, because that combination fails in the one way you would -not notice: the service stops and never comes back. - -## More than one thing to back up - -`tar` takes several paths, so multiple files or directories are normally **one** -artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together, -lnbits captures its data directory and its `.env`. - -Prefer one artefact. A backup should be a consistent snapshot, and two artefacts -written by two runs can drift — you can end up restoring an `.env` that does not -match the database it configures. Pulling a single file back out needs no -unpacking: - -```bash -age -d -i | tar -xzO opt/lnbits/lnbits/.env -``` - -If you genuinely need separate artefacts, call the role twice with different -`backup_source_name`s rather than extending it — but only one call may set -`backup_source_stop_service`, or the service is stopped twice per night. - -The case this shape cannot express is a **database dump plus a file tree** -(`pg_dump` and a media directory, say): you cannot merge those into one stream -without staging plaintext on disk, which is exactly what this design avoids. -None of the current services need it — all are file trees, all stopped for the -dump. A future one that does should use two role calls. - -## Everything here is sqlite, so everything stops - -All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files -present). A live copy of a WAL-mode database can be torn or stale, so each is -stopped for the duration. Measured downtime: under a second for headscale and -memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos -and database. That last one is the real cost of a consistent snapshot; if it -becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy, -not skipping the stop. - -## The trap is the reason this role exists - -When `backup_source_stop_service` is set, the script stops the unit and installs -an EXIT trap that starts it again. Without it, a failed dump leaves the service -down until the next timer fires — **every hand-written script this replaced had -that bug**, and it was only ever masked because their `systemctl stop` failed -first, before anything was stopped. - -Verified on spacey: with the dump forced to fail, the log shows -`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit -failed rather than hiding it), and headscale is `active` afterwards. - -If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed — -which is correct, because nothing was stopped. - -## `.partial` - -The dump writes `.partial` and only `mv`s it into place on success, so -a truncated file is never mistaken for a backup. A failure inside the pipeline -does leave one behind, and the prune glob cannot match it (it ends `.partial`, -not `.tar.gz.age`), so the script clears stale partials at the **start** of each -run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run. - -## `backup_source_stop_service` may be a bare name - -`headscale` and `headscale.service` both work. The unit template normalises it, -because systemd rejects a bare name in `After=` with -`Failed to add dependency ... Invalid argument` — which it logs and then ignores, -so the unit appears to work while carrying no ordering at all. - -## Retention is two-tier - -`backup_source_retention_days` is **local** and short — these hosts are -disk-constrained. The long tail lives on `small-backups-box`, which decides its -own retention per source. Losing the local copy is expected and fine. diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml deleted file mode 100644 index b195190..0000000 --- a/ansible/roles/backup_source/defaults/main.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -# Required -backup_source_name: "" # "headscale" -> headscale_.tar.gz.age -backup_source_description: "" # "Headscale" -backup_source_dump_command: "" # must write the payload to STDOUT - -# Placement -backup_source_dir: "/opt/backups/{{ backup_source_name }}" -backup_source_artifact_suffix: "tar.gz.age" - -# Encryption. Asymmetric: the host holds only the public key and cannot decrypt -# what it produces. -backup_source_recipient: "{{ age_backup_recipient }}" - -# The unprivileged account small-backups-box pulls as. It owns the dump -# directory and nothing else; it deliberately has no sudo. -backup_source_pull_user: backup-pull -backup_source_pull_key: "{{ backup_pull_public_key }}" - -# Safety. Give either a systemd unit, or an explicit pair of commands for -# services that are not systemd-managed (vaultwarden is a docker compose stack). -# Whichever is used, a trap guarantees the restart. -backup_source_stop_service: "" # systemd unit stopped for the dump -backup_source_stop_command: "" # overrides stop_service when set -backup_source_start_command: "" # required alongside stop_command - -# Retention here is LOCAL and short; small-backups-box keeps the long tail. -backup_source_retention_days: 7 - -# Schedule. The box pulls at 04:00, so dumps must land before that. -backup_source_on_calendar: "*-*-* 02:00:00" - -# ── Reporting ──────────────────────────────────────────────────────────────── -# Where to report that this dump ran and produced a plausible artefact. -# Gatus external endpoint: -# POST {url}?success=true|false&error=... -# Authorization: Bearer {token} -# Empty is valid and is not an error: the unit's exit code is still the answer, -# and the STORE will independently notice a stale dump within ~26h because the -# artefact filename carries this dump's timestamp. Reporting here only buys -# earlier detection and tells you it was the DUMP that broke rather than the -# pull. -backup_source_push_url: "" -backup_source_push_token: "" diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml deleted file mode 100644 index 37a7f2b..0000000 --- a/ansible/roles/backup_source/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: Reload systemd for backup units - ansible.builtin.systemd: - daemon_reload: yes diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml deleted file mode 100644 index a964c5d..0000000 --- a/ansible/roles/backup_source/tasks/main.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -- name: Assert backup_source parameters are sane - ansible.builtin.assert: - that: - - backup_source_name | length > 0 - - backup_source_description | length > 0 - - backup_source_dump_command | length > 0 - - backup_source_recipient | length > 0 - - backup_source_recipient is match('^age1[0-9a-z]{58}$') - - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0) - fail_msg: >- - backup_source: '{{ backup_source_name | default("") }}' needs a name, - description, dump command and a valid age recipient (age1... 62 chars). - backup_source_stop_command must be paired with backup_source_start_command. - quiet: true - -# Declared here rather than assumed. Stage 1 installed it by hand; this is what -# makes a rebuilt host get it too. -# Cache refresh is best-effort on purpose. An unrelated third-party repo with a -# bad signing key (spacey had two: an expired Caddy subkey and a SHA1 nodesource -# key) makes `apt-get update` return warnings, which the apt module treats as a -# hard failure — and that must not stop backups being configured. Installing the -# package is NOT best-effort: if age is genuinely unavailable, the next task fails. -- name: Refresh the apt cache (best effort) - ansible.builtin.apt: - update_cache: yes - cache_valid_time: 3600 - failed_when: false - changed_when: false - -- name: Ensure age is installed - ansible.builtin.apt: - name: - - age - # curl is needed only when backup_source_push_url is set, but installing it - # unconditionally keeps the task idempotent and it is present on every - # Debian host here anyway. - - curl - state: present - -# The pull account: unprivileged, no sudo, exists only so small-backups-box can -# read the dump directory. Trust points one way — the box can read backups, and -# can do nothing else on this host. -- name: "Ensure the {{ backup_source_pull_user }} account exists" - ansible.builtin.user: - name: "{{ backup_source_pull_user }}" - system: yes - shell: /bin/sh # rsync-over-ssh needs a shell; nologin breaks it - home: "/var/lib/{{ backup_source_pull_user }}" - create_home: yes - password: '!' # no password login, ever - when: backup_source_pull_user | length > 0 - -- name: "Authorise the backup box's key for {{ backup_source_pull_user }}" - ansible.posix.authorized_key: - user: "{{ backup_source_pull_user }}" - key: "{{ backup_source_pull_key }}" - key_options: "restrict" # no pty, no forwarding, no user rc - exclusive: yes - state: present - when: backup_source_pull_user | length > 0 - -# The shared container above the per-service directories. It must be traversable -# or the pull account cannot reach its own directory. The script's `mkdir -p` -# runs under `umask 077` and would otherwise create this 0700. -- name: "Ensure {{ backup_source_dir | dirname }} is traversable" - ansible.builtin.file: - path: "{{ backup_source_dir | dirname }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: "Ensure {{ backup_source_dir }} exists" - ansible.builtin.file: - path: "{{ backup_source_dir }}" - state: directory - owner: root - group: "{{ backup_source_pull_user | default('root', true) }}" - mode: '0750' - -- name: "Install the {{ backup_source_name }} backup script" - ansible.builtin.template: - src: backup.sh.j2 - dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh" - owner: root - group: root - mode: '0750' - validate: "bash -n %s" - -# The .service carries the push token in an Environment= line, so it is 0600. -# The .timer holds nothing secret and stays world-readable. -- name: "Install the {{ backup_source_name }}-backup systemd units" - ansible.builtin.template: - src: "backup.{{ item.unit }}.j2" - dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item.unit }}" - owner: root - group: root - mode: "{{ item.mode }}" - loop: - - {unit: service, mode: "0600"} - - {unit: timer, mode: "0644"} - notify: Reload systemd for backup units - -- name: "Enable the {{ backup_source_name }}-backup timer" - ansible.builtin.systemd: - name: "{{ backup_source_name }}-backup.timer" - enabled: yes - state: started - daemon_reload: yes diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 deleted file mode 100644 index 7a1df21..0000000 --- a/ansible/roles/backup_source/templates/backup.service.j2 +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description={{ backup_source_description }} backup -{% if backup_source_stop_service %} -{# systemd rejects a bare name here ("Failed to add dependency ... Invalid - argument"), so normalise to a full unit name. #} -After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }} -{% endif %} - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh -Environment=BACKUP_PUSH_URL={{ backup_source_push_url }} -Environment=BACKUP_PUSH_TOKEN={{ backup_source_push_token }} -StandardOutput=journal -StandardError=journal -SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 deleted file mode 100644 index 018c313..0000000 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash -# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source) -# -# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}. -# The host holds only the age PUBLIC key, so it cannot read its own backups. -set -euo pipefail -umask 077 - -BACKUP_DIR="{{ backup_source_dir }}" -RETENTION_DAYS={{ backup_source_retention_days }} -RECIPIENT="{{ backup_source_recipient }}" -SUFFIX="{{ backup_source_artifact_suffix }}" -NAME="{{ backup_source_name }}" -{% if backup_source_stop_service or backup_source_stop_command %} -STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }} -START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }} -SERVICE="{{ backup_source_stop_service or backup_source_description }}" # label for the log only -{% endif %} - -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}" - -die() { echo "FATAL: $*" >&2; exit 1; } -log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } - -# --- Pre-flight --- -[[ -n "$RECIPIENT" ]] || die "no age recipient configured" -command -v age >/dev/null || die "age is not installed" - -# Mode must agree with what the role sets, or each undoes the other every run. -mkdir -p "$BACKUP_DIR" -{% if backup_source_pull_user %} -chown root:{{ backup_source_pull_user }} "$BACKUP_DIR" -chmod 750 "$BACKUP_DIR" -{% else %} -chmod 700 "$BACKUP_DIR" -{% endif %} - -# A run that died mid-dump leaves a .partial. It is not a backup, and the prune -# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them -# here or they accumulate forever. -rm -f "${BACKUP_DIR}/${NAME}_"*.partial - -# --- Reporting ------------------------------------------------------------- -# A dump that exits non-zero, or that produces a zero-byte artefact, is a failed -# backup even though the script "finished". Both are reported as failures. -PUSH_URL="${BACKUP_PUSH_URL:-}" -PUSH_TOKEN="${BACKUP_PUSH_TOKEN:-}" - -report() { - local success="$1" message="$2" - [ -n "$PUSH_URL" ] || return 0 - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null || true -} - -# Reports on ANY exit path, so a dump that dies halfway still reports rather -# than going quiet. The size of the FINISHED artefact decides success, not -# merely reaching the end of the script. -# -# This is called FROM the single EXIT trap below - it must never register an -# EXIT trap of its own. `trap ... EXIT` REPLACES the existing handler rather -# than adding to it, so a second trap here silently discards the one that -# restarts the service, and a backup run leaves the service stopped. That is -# precisely the failure the restart trap exists to prevent. -report_outcome() { - local rc="$1" - if [ "$rc" -ne 0 ]; then - report "false" "${NAME} dump exited ${rc}" - elif [ ! -s "$ARTIFACT" ]; then - report "false" "${NAME} produced no artefact at ${ARTIFACT}" - else - report "true" "${NAME} $(du -h "$ARTIFACT" | cut -f1)" - fi -} - -# --- One EXIT handler, doing both jobs ------------------------------------- -# bash keeps exactly ONE EXIT trap: `trap ... EXIT` REPLACES the previous -# handler rather than adding to it. Registering a second one here would -# silently discard the service restart and leave the service stopped after -# every backup - which is the exact bug the restart exists to prevent, and it -# is invisible until someone notices the service is down. -on_exit() { - local rc=$? -{% if backup_source_stop_service or backup_source_stop_command %} - log "Restarting ${SERVICE}..." - eval "$START_CMD" || true -{% endif %} - report_outcome "$rc" -} -trap on_exit EXIT - -{% if backup_source_stop_service or backup_source_stop_command %} -# --- Stop the service; the trap above guarantees it comes back ------------- -# The trap is the point: without it a failed dump leaves the service down until -# the next timer fires. Every hand-written script this replaced had that bug. -# It is armed BEFORE the stop, so even a failure during the stop restarts. -log "Stopping ${SERVICE}..." -eval "$STOP_CMD" -{% endif %} - -# --- Dump straight into age; plaintext never touches the disk --- -log "Writing ${ARTIFACT}..." -{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" -{% if backup_source_pull_user %} -# Match the final ownership immediately, so even a partial left by a later -# failure is not an unreadable obstacle to the pull. -chown root:{{ backup_source_pull_user }} "${ARTIFACT}.partial" -chmod 640 "${ARTIFACT}.partial" -{% endif %} -mv "${ARTIFACT}.partial" "$ARTIFACT" -{% if backup_source_pull_user %} -# Readable by the pull account and nobody else. The contents are age-encrypted -# regardless, so this is depth rather than the actual protection. -chown root:{{ backup_source_pull_user }} "$ARTIFACT" -chmod 640 "$ARTIFACT" -{% else %} -chmod 600 "$ARTIFACT" -{% endif %} -log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))" - -# --- Prune --- -log "Pruning local artefacts older than ${RETENTION_DAYS} days..." -find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete - -log "Done." diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2 deleted file mode 100644 index d9a7e6a..0000000 --- a/ansible/roles/backup_source/templates/backup.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description={{ backup_source_description }} backup - -[Timer] -OnCalendar={{ backup_source_on_calendar }} -# Persistent: a window missed while the host was down runs on next boot. cron on -# a laptop had no equivalent, which is how two backups went unnoticed for months. -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md deleted file mode 100644 index 13d9f5a..0000000 --- a/ansible/roles/backup_store/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# `backup_store` - -Pulls already-encrypted backup artefacts from every source host onto -`small-backups-box`, on a timer, and expires them per source. - -Generalises the hand-written `pull-backups.sh` that had one hardcoded source -(`arbret`). That job's behaviour is preserved exactly: same source path, same -90 days, same destination directory. - -## This host holds no key - -Everything pulled here is ciphertext produced by `backup_source` on the source -host. The box cannot read any of it — the age identity lives only on lapy. That -is deliberate: the machine holding every backup should not also be able to open -them. - -## One failing source must not stop the others - -The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own -function, failures are counted, and the script exits non-zero at the end so -systemd marks the unit failed. A dead host costs you that one source, not the -whole run. - -This is the specific failure the whole plan exists to prevent: the laptop jobs -aborted on first error and then silently produced empty directories for nine -months. - -## Trust points one way - -The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated -account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised -with `restrict`. That account can read one directory and do nothing else — no -sudo, no pty, no forwarding. A compromised backup box cannot reach into -production. - -## Addressing: names, never IPs - -Sources are addressed by name. The job this replaced hardcoded spacey's IP; the -droplet was later rebuilt, the address was recycled to a stranger, and the -backup failed silently from 2025-12-01 while the directory listing still looked -healthy. - -Two kinds of name are in play: - -- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a - headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without - it the box cannot even resolve the peer, let alone reach it. -- **spacey** is *not* a tailnet member — it is the headscale control server — so - its backup is pulled over the public internet via `headscale.contrapeso.xyz`, - which follows the host if the droplet is rebuilt. - -## Retention here is the long tail - -Sources keep a few days locally; this box keeps 90 (or whatever the source entry -says). Losing the source's local copy is expected. diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml deleted file mode 100644 index e6d17a3..0000000 --- a/ansible/roles/backup_store/defaults/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -backup_store_dir: "{{ ansible_env.HOME }}/backups" -backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull" -backup_store_on_calendar: "*-*-* 04:00:00" - -# One entry per source. `retention_days` is the LONG tail; the source keeps its -# own short local retention. -# - name: headscale -# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" -# retention_days: 90 -backup_store_sources: [] - -# ── Reporting ──────────────────────────────────────────────────────────────── -# check-backups.sh reports one result PER SOURCE plus one for the store itself, -# so the base URL is the endpoints collection and the script appends each key. -# Empty is valid: the script still prints its report and exits 0/1. -backup_store_check_push_base: "" -backup_store_check_push_token: "" - -# Every six hours, offset past the 04:00 pull so the first run of the day sees a -# finished pull. The BACKUPS are daily, but this check is not - it reads the -# source's dump timestamp out of the artefact filename, so running it more often -# catches "the source stopped dumping" within hours rather than a day, and lets -# the Gatus heartbeat be 7h instead of 30h. -backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00" - -# An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the -# pull is at 04:00, so 26h tolerates exactly one missed night before alarming. -backup_store_check_max_age_hours: 26 diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml deleted file mode 100644 index 632be35..0000000 --- a/ansible/roles/backup_store/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Reload systemd for pull-backups - ansible.builtin.systemd: - daemon_reload: yes - become: yes diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml deleted file mode 100644 index 55500c6..0000000 --- a/ansible/roles/backup_store/tasks/main.yml +++ /dev/null @@ -1,89 +0,0 @@ ---- -- name: Assert backup_store sources are sane - ansible.builtin.assert: - that: - - backup_store_sources | length > 0 - - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length - - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length - - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length - fail_msg: "backup_store: every source needs name, source and retention_days" - quiet: true - -- name: Ensure rsync is installed - ansible.builtin.apt: - name: rsync - state: present - update_cache: yes - cache_valid_time: 3600 - become: yes - -- name: Ensure the backup store directory exists - ansible.builtin.file: - path: "{{ backup_store_dir }}" - state: directory - mode: '0700' - -- name: Install the pull-backups script - ansible.builtin.template: - src: pull-backups.sh.j2 - dest: /usr/local/bin/pull-backups.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - become: yes - -# An assertion that last night actually worked. Generated from the same source -# list as the puller, so it can never drift out of sync with what is supposed to -# be arriving. Runs on a timer AND is useful by hand. -- name: Install the backup check script - ansible.builtin.template: - src: check-backups.sh.j2 - dest: /usr/local/bin/check-backups.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - become: yes - -# The .service carries the push token, so it is 0600; the .timer is not secret. -- name: Install the check-backups systemd units - ansible.builtin.template: - src: "check-backups.{{ item.unit }}.j2" - dest: "/etc/systemd/system/check-backups.{{ item.unit }}" - owner: root - group: root - mode: "{{ item.mode }}" - loop: - - {unit: service, mode: "0600"} - - {unit: timer, mode: "0644"} - become: yes - -# restarted, not started: `started` is a no-op on an already-active timer, so a -# changed schedule would never be picked up. -- name: Enable the check-backups timer - ansible.builtin.systemd: - name: check-backups.timer - enabled: yes - state: restarted - daemon_reload: yes - become: yes - -- name: Install the pull-backups systemd units - ansible.builtin.template: - src: "pull-backups.{{ item }}.j2" - dest: "/etc/systemd/system/pull-backups.{{ item }}" - owner: root - group: root - mode: '0644' - loop: [service, timer] - become: yes - notify: Reload systemd for pull-backups - -- name: Enable the pull-backups timer - ansible.builtin.systemd: - name: pull-backups.timer - enabled: yes - state: started - daemon_reload: yes - become: yes diff --git a/ansible/roles/backup_store/templates/check-backups.service.j2 b/ansible/roles/backup_store/templates/check-backups.service.j2 deleted file mode 100644 index 3e50b6a..0000000 --- a/ansible/roles/backup_store/templates/check-backups.service.j2 +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Verify the nightly backup pull actually worked -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -User={{ ansible_user_id }} -ExecStart=/usr/local/bin/check-backups.sh {{ backup_store_check_max_age_hours }} -Environment=BACKUP_CHECK_PUSH_BASE={{ backup_store_check_push_base }} -Environment=BACKUP_CHECK_PUSH_TOKEN={{ backup_store_check_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/backup_store/templates/check-backups.sh.j2 b/ansible/roles/backup_store/templates/check-backups.sh.j2 deleted file mode 100644 index 37c9916..0000000 --- a/ansible/roles/backup_store/templates/check-backups.sh.j2 +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash -# Assert the nightly backups actually worked. -# -# Run as {{ ansible_user_id }} on this host. Needs no sudo. -# -# What it CANNOT do: verify contents. The age identity lives only on lapy, so -# this host cannot decrypt anything it holds — by design. These are freshness, -# completeness and integrity checks. To verify content, decrypt on lapy: -# ssh {{ ansible_user_id }}@$(hostname) "cat ~/backups//" \ -# | age -d -i ~/.age/counterweight_age | tar -tzf - | head -# -# Exit 0 = everything passed (warnings allowed), 1 = at least one FAIL. -set -uo pipefail - -STORE="{{ backup_store_dir }}" -MAX_AGE_H="${1:-26}" # an artefact older than this is stale -NOW=$(date +%s) -fails=0; warns=0 - -# Colour only when attached to a terminal: this gets piped into files and, later, -# probably into a notification. -if [ -t 1 ]; then R=$'\033[31m'; Y=$'\033[33m'; G=$'\033[32m'; N=$'\033[0m' -else R=''; Y=''; G=''; N=''; fi - -# Per-source verdicts, so each source can be reported independently. A single -# aggregate red light tells you backups are broken; it does not tell you which -# one, which is the thing you need at 3am. -declare -A SRC_FAIL SRC_MSG -CURRENT="" - -red() { printf ' %sFAIL%s %s\n' "$R" "$N" "$*"; fails=$((fails+1)); - [ -n "$CURRENT" ] && { SRC_FAIL[$CURRENT]=1; SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; }; } -yell() { printf ' %sWARN%s %s\n' "$Y" "$N" "$*"; warns=$((warns+1)); } -ok() { printf ' %sok%s %s\n' "$G" "$N" "$*"; - [ -n "$CURRENT" ] && SRC_MSG[$CURRENT]="${SRC_MSG[$CURRENT]:-}${SRC_MSG[$CURRENT]:+; }$*"; } - -# --- Reporting ------------------------------------------------------------- -# Each source gets its own Gatus external endpoint, plus one for the store -# itself (the pull unit, the timer, and disk capacity). PUSH_BASE empty means -# report nowhere, which is valid: the exit code is still the whole answer. -PUSH_BASE="${BACKUP_CHECK_PUSH_BASE:-}" -PUSH_TOKEN="${BACKUP_CHECK_PUSH_TOKEN:-}" - -report() { - local key="$1" success="$2" message="$3" - [ -n "$PUSH_BASE" ] || return 0 - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true -} - -hours_since() { echo $(( (NOW - $1) / 3600 )); } - -# Pull the dump timestamp out of _YYYYmmdd_HHMMSS.. This is when -# the SOURCE produced it, which is the thing that actually matters: a source -# whose timer died still pulls "ok" forever, because yesterday's artefact is -# still sitting there. Checking only the pull would miss exactly that. -dump_epoch() { - local base ts - base=$(basename "$1") - ts=$(echo "$base" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) || return 1 - [ -n "$ts" ] || return 1 - date -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null -} - -check_source() { - local name="$1" keep="$2" dir="$STORE/$1" - printf '\n%s\n' "== $name" - CURRENT="$name" - SRC_FAIL[$name]=0 - SRC_MSG[$name]="" - - [ -d "$dir" ] || { red "$name: no directory $dir"; return; } - - local n; n=$(find "$dir" -maxdepth 1 -type f -name "${name}_*" | wc -l) - [ "$n" -gt 0 ] || { red "$name: no artefacts at all"; return; } - - local partials; partials=$(find "$dir" -maxdepth 1 -name '*.partial' | wc -l) - [ "$partials" -eq 0 ] || red "$name: $partials .partial file(s) pulled — the pull should exclude these" - - local newest; newest=$(ls -t "$dir"/${name}_* 2>/dev/null | head -1) - local prev; prev=$(ls -t "$dir"/${name}_* 2>/dev/null | sed -n 2p) - - # 1. Did the SOURCE dump recently? - local de; de=$(dump_epoch "$newest") - if [ -z "${de:-}" ]; then - yell "$name: cannot parse a dump timestamp from $(basename "$newest")" - else - local dh; dh=$(hours_since "$de") - if [ "$dh" -lt 0 ]; then - # A future-dated artefact would otherwise stay "fresh" forever and the - # staleness check would never fire again — the exact silent failure this - # script exists to catch. - red "$name: newest dump is dated ${dh#-}h in the FUTURE — clock skew on the source?" - elif [ "$dh" -gt "$MAX_AGE_H" ]; then - red "$name: newest dump is ${dh}h old (>${MAX_AGE_H}h) — the source timer did not run" - else - ok "$name: dumped ${dh}h ago" - fi - fi - - # 2. Did the PULL bring it over recently? - local ph; ph=$(hours_since "$(stat -c %Y "$newest")") - if [ "$ph" -gt "$MAX_AGE_H" ]; then - red "$name: newest artefact was pulled ${ph}h ago (>${MAX_AGE_H}h)" - else - ok "$name: pulled ${ph}h ago" - fi - - # 3. Is it plausibly a real backup? - local sz; sz=$(stat -c %s "$newest") - if [ "$sz" -eq 0 ]; then - red "$name: newest artefact is ZERO bytes" - elif [ -n "$prev" ]; then - local psz; psz=$(stat -c %s "$prev") - if [ "$psz" -gt 0 ] && [ "$sz" -lt $(( psz / 2 )) ]; then - # Not automatically wrong: headscale legitimately shrank 297K -> 20K when - # a clean stop checkpointed its write-ahead log into the database. - yell "$name: $(numfmt --to=iec "$sz") is less than half the previous $(numfmt --to=iec "$psz") — check it decrypts to what you expect" - else - ok "$name: $(numfmt --to=iec "$sz") ($n artefacts)" - fi - else - ok "$name: $(numfmt --to=iec "$sz") (first artefact)" - fi - - # 4. Is retention pruning? Allow generous slack for multiple dumps per day. - if [ "$n" -gt $(( keep * 3 + 10 )) ]; then - yell "$name: $n artefacts for a ${keep}-day retention — pruning may not be working" - fi -} - -echo "Backup check on $(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z')" -echo "Artefacts older than ${MAX_AGE_H}h are treated as stale." - -# --- the pull job itself --- -# Reserved key, reported as backup-store_pull-job. The store's own machinery is -# a different alarm from any one source being stale, and it is a LEADING -# indicator where the per-source checks are lagging ones: those only fire once -# an artefact is >26h stale, i.e. about a day after the fault. A disabled timer, -# a failed pull job or a filling disk are all visible here immediately, and they -# name the cause instead of showing six stale sources with no explanation. -CURRENT="__store" -SRC_FAIL[__store]=0 -SRC_MSG[__store]="" -printf '\n%s\n' "== pull-backups.service" -result=$(systemctl show pull-backups.service -p Result --value 2>/dev/null) -status=$(systemctl show pull-backups.service -p ExecMainStatus --value 2>/dev/null) -when=$(systemctl show pull-backups.service -p ExecMainExitTimestamp --value 2>/dev/null) -[ "$result" = "success" ] && ok "last run result: success" || red "last run result: ${result:-unknown} (exit ${status:-?})" -if [ -n "$when" ]; then - wh=$(hours_since "$(date -d "$when" +%s)") - [ "$wh" -le "$MAX_AGE_H" ] && ok "last ran ${wh}h ago" || red "last ran ${wh}h ago (>${MAX_AGE_H}h) — did the timer fire?" -fi -systemctl is-enabled pull-backups.timer >/dev/null 2>&1 \ - && ok "timer enabled, next $(systemctl show pull-backups.timer -p NextElapseUSecRealtime --value 2>/dev/null)" \ - || red "pull-backups.timer is NOT enabled" - -# --- each source --- -{% for src in backup_store_sources %} -check_source "{{ src.name }}" {{ src.retention_days }} -CURRENT="" -# The store key is what Gatus computes from group+name: sanitize("backup-store") -# + "_" + sanitize("{{ src.name }}"). -report "backup-store_{{ src.name }}" \ - "$([ "${SRC_FAIL[{{ src.name }}]:-1}" -eq 0 ] && echo true || echo false)" \ - "${SRC_MSG[{{ src.name }}]:-no result}" -{% endfor %} - -# --- capacity --- -CURRENT="__store" -printf '\n%s\n' "== disk" -use=$(df --output=pcent "$STORE" | tail -1 | tr -dc '0-9') -avail=$(df -h --output=avail "$STORE" | tail -1 | tr -d ' ') -if [ "$use" -ge 90 ]; then red "store is ${use}% full, ${avail} free" -elif [ "$use" -ge 75 ]; then yell "store is ${use}% full, ${avail} free" -else ok "store is ${use}% full, ${avail} free"; fi - -CURRENT="" -report "backup-store_pull-job" \ - "$([ "${SRC_FAIL[__store]:-1}" -eq 0 ] && echo true || echo false)" \ - "${SRC_MSG[__store]:-no result}" - -printf '\n%s\n' "-----" -if [ "$fails" -gt 0 ]; then - echo "RESULT: $fails failure(s), $warns warning(s)" - echo "Investigate with: journalctl -u pull-backups -n 50 --no-pager" - exit 1 -fi -if [ "$warns" -gt 0 ]; then - echo "RESULT: all checks passed, $warns warning(s)" -else - echo "RESULT: all checks passed" -fi -exit 0 diff --git a/ansible/roles/backup_store/templates/check-backups.timer.j2 b/ansible/roles/backup_store/templates/check-backups.timer.j2 deleted file mode 100644 index ba5c5a8..0000000 --- a/ansible/roles/backup_store/templates/check-backups.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Run the backup verification after the nightly pull -Requires=check-backups.service - -[Timer] -OnCalendar={{ backup_store_check_on_calendar }} -# Run a missed occurrence on the next boot rather than skipping the day. -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2 deleted file mode 100644 index c65516c..0000000 --- a/ansible/roles/backup_store/templates/pull-backups.service.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Pull encrypted backups from production - -[Service] -Type=oneshot -User={{ ansible_user_id }} -ExecStart=/usr/local/bin/pull-backups.sh -StandardOutput=journal -StandardError=journal -SyslogIdentifier=pull-backups diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 deleted file mode 100644 index 1db7d9c..0000000 --- a/ansible/roles/backup_store/templates/pull-backups.sh.j2 +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Pull encrypted backups from production — managed by Ansible (roles/backup_store) -# -# Everything here is already ciphertext: this host only moves and expires files, -# and holds no key that can read them. -set -uo pipefail # deliberately NOT -e; see the loop below - -SSH_KEY="{{ backup_store_ssh_key }}" -STORE="{{ backup_store_dir }}" - -log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } -fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); } - -failures=0 - -# One source failing must not stop the others. The whole point of this box is -# that a single dead host cannot silently take the rest of the backups with it — -# which is exactly how the laptop-based jobs failed unnoticed for nine months. -{% for src in backup_store_sources %} -# --- {{ src.name }} --- -pull_{{ src.name | replace('-', '_') }}() { - local dir="${STORE}/{{ src.name }}" - mkdir -p "$dir" - log "Pulling {{ src.name }} from {{ src.source }}..." - # --exclude '*.partial': a dump that died mid-write leaves one behind, owned - # root:root 0600 because the chown only happens after a successful mv. Without - # this exclude the pull account cannot read it and rsync fails for the WHOLE - # source — so one failed dump would silently block every subsequent pull of - # that service. An incomplete artefact is never worth transferring anyway. - if rsync -az --timeout=120 --exclude '*.partial' \ - -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ - "{{ src.source }}" "$dir/"; then - log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" - else - fail "{{ src.name }}: rsync failed" - return 1 - fi - log " {{ src.name }}: pruning older than {{ src.retention_days }} days" - find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete -} -pull_{{ src.name | replace('-', '_') }} || true - -{% endfor %} -if [ "$failures" -gt 0 ]; then - log "FAILED: $failures source(s) did not pull" - exit 1 -fi -log "All sources pulled." diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2 deleted file mode 100644 index 336e1ca..0000000 --- a/ansible/roles/backup_store/templates/pull-backups.timer.j2 +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Daily offsite backup pull - -[Timer] -OnCalendar={{ backup_store_on_calendar }} -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/bitcoin_knots/README.md b/ansible/roles/bitcoin_knots/README.md deleted file mode 100644 index 8e35900..0000000 --- a/ansible/roles/bitcoin_knots/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# `bitcoin_knots` - -Builds Bitcoin Knots from source with PGP + SHA256 verification of the release -tarball, runs it as a full node on `knots-box`, and keeps a health check on a -systemd timer. The second play in the calling playbook publishes the P2P port -from the edge host via `socket_proxy`. - -Converted from `deploy_bitcoin_knots_playbook.yml` (892 lines) under Plan 6. The -playbook is now 40 lines. - -## The build is guarded; the chain is never touched - -`build.yml` is 32 tasks, every one carrying -`when: not bitcoind_binary_exists.stat.exists`. On a host that already has the -binary the whole download / verify / 30-60 minute compile skips — **including the -two `state: absent` deletions**, which target `/opt/bitcoin-knots/source` and the -extracted build directory. - -The chain lives elsewhere and nothing here touches it: - -| | | -|---|---| -| `bitcoin_knots_dir` | `/opt/bitcoin-knots` — build tree, safe to delete | -| `bitcoin_data_dir` | `/var/lib/bitcoin` — config, logs, wallets | -| `bitcoin_large_data_dir` | `/mnt/knots_data` — **~875 GB of blockchain** | - -The signature-verification tasks are the security control of this role. They are -copied verbatim; do not "simplify" them. - -## ⚠ This node is half of the mining setup - -`bitcoin.conf` carries a DATUM Gateway section that was hand-added on the node -and was **missing from the playbook's template**: - -```ini -blockmaxsize=3985000 -blockmaxweight=3985000 -blocknotify=killall -USR1 datum_gateway -maxmempool=1000 -blockreconstructionextratxn=1000000 -``` - -`blocknotify` is how `datum_gateway` learns a new block landed. Running the old -playbook would have deleted all of it, and solo mining would have carried on -grinding against a stale template — a silent failure that costs money rather -than raising an error. The template now carries it behind -`bitcoin_datum_gateway_enabled`. - -**bitcoin-knots and datum-gateway are one system, not two services.** Changing -either config means thinking about both. - -## The restart handler, and why exactness matters now - -The hand-written `Restart bitcoind` handler carried -`when: uptime_kuma_enabled | default(false)`, so it had been inert since the -decommissioning: `bitcoin.conf` and the systemd unit both notify it and neither -could restart anything. A config change applied to disk, reported success, and -never took effect. - -It is ungated here — which raises the bar for the template. **Any** residual -difference between the template and the live file, down to a trailing newline, -means the task reports `changed` and restarts a Bitcoin node on every run. It -took four rounds of `--check --diff` to reach `changed=0`: the DATUM section, an -explanatory comment that was rendering into the deployed file (now a `{# #}` -Jinja comment), a `# Pruning (optional)` comment the live file had, and one -trailing blank line. - -## `dbcache` - -Computed as 90% of RAM unless `bitcoin_dbcache_mb_override` is set. The live node -was hand-tuned to **200 MB**; the calculation produces 3528. As with fulcrum, -`set_fact` outranks role defaults, so the *calculation* honours the override — a -value pinned only in `defaults/` is silently ignored. - -## Monitoring: one variable, no product knowledge - -The check tests bitcoind's RPC and records the answer in its exit code, which -systemd keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. Set -`healthcheck_push_url` to report anywhere that accepts an HTTP ping. - -The timer had last fired **2026-08-09** while still reporting `active` and -`enabled` — the same `OnBootSec` + `OnUnitActiveSec` dead chain as fulcrum, where -nothing re-arms it if the service does not run in a given boot. The role runs the -check once after enabling, which both smoke-tests it and supplies the reference -the timer schedules from. diff --git a/ansible/roles/bitcoin_knots/defaults/main.yml b/ansible/roles/bitcoin_knots/defaults/main.yml deleted file mode 100644 index 1045e77..0000000 --- a/ansible/roles/bitcoin_knots/defaults/main.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Bitcoin Knots Configuration Variables - -# Version - REQUIRED: Specify exact version/tag to build -# The only version string. There used to be a second, v-prefixed copy -# (bitcoin_knots_version) that nothing read - two hand-maintained copies of one -# fact, with nothing keeping them in step. -bitcoin_knots_version_short: "29.2.knots20251110" - -# Directories -bitcoin_knots_dir: /opt/bitcoin-knots -bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" -bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets -bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) -bitcoin_conf_dir: /etc/bitcoin - -# Network -bitcoin_rpc_port: 8332 -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -bitcoin_p2p_port: 8333 -bitcoin_rpc_bind: "0.0.0.0" - -# Build options -bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores -bitcoin_build_prefix: /usr/local - -# Configuration options -bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) -bitcoin_max_connections: 125 -# dbcache will be calculated as 90% of host RAM automatically in playbook - -# ZMQ Configuration -bitcoin_zmq_enabled: true -bitcoin_zmq_bind: "tcp://0.0.0.0" -bitcoin_zmq_port_rawblock: 28332 -bitcoin_zmq_port_rawtx: 28333 -bitcoin_zmq_port_hashblock: 28334 -bitcoin_zmq_port_hashtx: 28335 - -# Service user -bitcoin_user: bitcoin -bitcoin_group: bitcoin - -# --- Health check ---------------------------------------------------------- -# Checks bitcoind RPC and records the answer in its exit code, which systemd -# keeps: `systemctl is-failed bitcoin-knots-healthcheck.service`. -# -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means -# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping -# works; nothing here is specific to a monitoring product. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" - -# --- Logging ---------------------------------------------------------------- -# The live node logs to a file. Set to "" to use printtoconsole=1 (journald). -bitcoin_logfile: "{{ bitcoin_data_dir }}/debug.log" - -# --- dbcache ---------------------------------------------------------------- -# Computed as 90% of RAM unless this is set. The live node was hand-tuned to -# 200 MB; the calculation would have produced 3528. As with fulcrum, note that -# set_fact outranks role defaults, so the CALCULATION has to honour this - a -# value pinned only in defaults/ is silently ignored. -bitcoin_dbcache_mb_override: 200 - -# --- DATUM Gateway ---------------------------------------------------------- -# This node feeds block templates to datum_gateway on knots-box. These settings -# were hand-added to bitcoin.conf and were missing from the template, so a -# playbook run would have removed them and broken the mining setup. -bitcoin_datum_gateway_enabled: true -bitcoin_blockmaxsize: 3985000 -bitcoin_blockmaxweight: 3985000 -bitcoin_blocknotify: "killall -USR1 datum_gateway" -bitcoin_maxmempool: 1000 -bitcoin_blockreconstructionextratxn: 1000000 diff --git a/ansible/roles/bitcoin_knots/handlers/main.yml b/ansible/roles/bitcoin_knots/handlers/main.yml deleted file mode 100644 index 49ac219..0000000 --- a/ansible/roles/bitcoin_knots/handlers/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Ungated on purpose. The hand-written handler carried -# when: uptime_kuma_enabled | default(false) -# so it has been inert since the decommissioning. Two tasks notify it — -# bitcoin.conf and the systemd unit — and neither could actually restart -# bitcoind. A configuration change to a Bitcoin node therefore applied to disk, -# reported success, and silently never took effect. -# -# Restarting bitcoind re-opens the chainstate; it does not reindex. -- name: Restart bitcoind - systemd: - name: bitcoind - state: restarted - daemon_reload: yes diff --git a/ansible/roles/bitcoin_knots/tasks/build.yml b/ansible/roles/bitcoin_knots/tasks/build.yml deleted file mode 100644 index 5a1b900..0000000 --- a/ansible/roles/bitcoin_knots/tasks/build.yml +++ /dev/null @@ -1,222 +0,0 @@ ---- -# Every task here is guarded by `when: not bitcoind_binary_exists.stat.exists`, -# so on a host that already has the binary the whole download / verify / build -# sequence skips — including the two `state: absent` deletions, which target -# /opt/bitcoin-knots/{source,bitcoin-} and never the chain data in -# /mnt/knots_data. -- name: Check if bitcoind binary already exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_binary_exists - changed_when: false - -- name: Install gnupg for signature verification - apt: - name: gnupg - state: present - when: not bitcoind_binary_exists.stat.exists - -- name: Import Luke Dashjr's Bitcoin Knots signing key - command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 - register: key_import - changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" - when: not bitcoind_binary_exists.stat.exists - failed_when: key_import.rc != 0 - -- name: Display imported key fingerprint - command: gpg --fingerprint 90C8019E36C2E964 - register: key_fingerprint - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Download SHA256SUMS file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - -- name: Download SHA256SUMS.asc signature file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - -- name: Verify PGP signature on SHA256SUMS file - command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS - register: sha256sums_verification - changed_when: false - failed_when: false # Don't fail here - check for 'Good signature' in next task - when: not bitcoind_binary_exists.stat.exists - - -- name: Display SHA256SUMS verification result - debug: - msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" - when: not bitcoind_binary_exists.stat.exists - -- name: Fail if SHA256SUMS signature verification failed - fail: - msg: "SHA256SUMS signature verification failed. Aborting build." - when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) - -- name: Remove any existing tarball to force fresh download - file: - path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - state: absent - when: not bitcoind_binary_exists.stat.exists - -- name: Download Bitcoin Knots source tarball - get_url: - url: "{{ bitcoin_source_tarball_url }}" - dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - mode: '0644' - validate_certs: yes - force: yes - when: not bitcoind_binary_exists.stat.exists - -- name: Calculate SHA256 checksum of downloaded tarball - command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - register: tarball_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Extract expected checksum from SHA256SUMS file - shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' - register: expected_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - failed_when: expected_checksum.stdout == "" - -- name: Display checksum comparison - debug: - msg: - - "Expected: {{ expected_checksum.stdout | trim }}" - - "Actual: {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists - -- name: Verify tarball checksum matches SHA256SUMS - fail: - msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] - -- name: Remove existing source directory if it exists (to force fresh extraction) - file: - path: "{{ bitcoin_knots_source_dir }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - -- name: Remove extracted directory if it exists (from previous runs) - file: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - -- name: Extract verified source tarball - unarchive: - src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - dest: "{{ bitcoin_knots_dir }}" - remote_src: yes - when: not bitcoind_binary_exists.stat.exists - -- name: Check if extracted directory exists - stat: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - register: extracted_dir_stat - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Rename extracted directory to expected name - command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" - when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists - -- name: Check if CMakeLists.txt exists - stat: - path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" - register: cmake_exists - changed_when: false - when: not bitcoind_binary_exists.stat.exists - -- name: Create CMake build directory - file: - path: "{{ bitcoin_knots_source_dir }}/build" - state: directory - mode: '0755' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - -- name: Configure Bitcoin Knots build with CMake - command: > - cmake - -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} - -DBUILD_BITCOIN_WALLET=OFF - -DCMAKE_BUILD_TYPE=Release - -DWITH_ZMQ=ON - .. - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: configure_result - changed_when: true - -- name: Verify CMake enabled ZMQ - shell: | - set -e - cd "{{ bitcoin_knots_source_dir }}/build" - cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: zmq_check - changed_when: false - -- name: Fail if CMakeLists.txt not found - fail: - msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." - when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) - -- name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) - command: cmake --build . -j{{ bitcoin_build_jobs }} - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - async: 3600 - poll: 0 - register: build_result - changed_when: true - -- name: Check build status - async_status: - jid: "{{ build_result.ansible_job_id }}" - register: build_job_result - until: build_job_result.finished - retries: 120 - delay: 60 - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined - -- name: Fail if build failed - fail: - msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) - -- name: Install Bitcoin Knots binaries - command: cmake --install . - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - changed_when: true - -- name: Verify bitcoind binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_installed - changed_when: false - -- name: Verify bitcoin-cli binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" - register: bitcoin_cli_installed - changed_when: false - -- name: Fail if binaries not found - fail: - msg: "Bitcoin Knots binaries not found after installation" - when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists diff --git a/ansible/roles/bitcoin_knots/tasks/configure.yml b/ansible/roles/bitcoin_knots/tasks/configure.yml deleted file mode 100644 index 52fd3fd..0000000 --- a/ansible/roles/bitcoin_knots/tasks/configure.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Ownership copied verbatim from the playbook this replaces; verified -# mechanically against `git show HEAD:` rather than retyped from memory. -- name: Create bitcoin.conf configuration file - ansible.builtin.template: - src: bitcoin.conf.j2 - dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0640' - notify: Restart bitcoind - -- name: Create systemd service file for bitcoind - ansible.builtin.template: - src: bitcoind.service.j2 - dest: /etc/systemd/system/bitcoind.service - owner: root - group: root - mode: '0644' - notify: Restart bitcoind diff --git a/ansible/roles/bitcoin_knots/tasks/healthcheck.yml b/ansible/roles/bitcoin_knots/tasks/healthcheck.yml deleted file mode 100644 index c6c8db3..0000000 --- a/ansible/roles/bitcoin_knots/tasks/healthcheck.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -# Everything here answers "is bitcoind healthy" and records the answer. The -# Uptime Kuma specifics that used to follow — an embedded Python script creating -# monitors over the API, a /tmp credentials file, push-URL extraction and a -# systemd Environment= rewrite — are gone. Where it reports is now one variable, -# healthcheck_push_url. See the role README. -- name: Install curl for health check script - apt: - name: curl - state: present - -- name: Create Bitcoin Knots health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create systemd service for Bitcoin Knots health check - ansible.builtin.template: - src: healthcheck.service.j2 - dest: /etc/systemd/system/bitcoin-knots-healthcheck.service - owner: root - group: root - mode: "0600" - -- name: Create systemd timer for Bitcoin Knots health check - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer - owner: root - group: root - mode: '0644' - -- name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - -- name: Enable and restart the Bitcoin Knots health check timer - systemd: - name: bitcoin-knots-healthcheck.timer - enabled: yes - state: restarted - daemon_reload: yes - -# Runs the check once, which is both a smoke test and the thing that actually -# arms the timer. This timer is OnBootSec + OnUnitActiveSec with no OnCalendar: -# OnBootSec elapses once, and OnUnitActiveSec needs the SERVICE to have run this -# boot to have anything to schedule from. Restarting the timer does not supply -# that reference; running the service does. The live timer had last fired on -# 2026-08-09 while still reporting `active` and `enabled`. -- name: Run the Bitcoin Knots health check once to arm the timer - command: systemctl start bitcoin-knots-healthcheck.service - changed_when: false diff --git a/ansible/roles/bitcoin_knots/tasks/install.yml b/ansible/roles/bitcoin_knots/tasks/install.yml deleted file mode 100644 index 3003b6e..0000000 --- a/ansible/roles/bitcoin_knots/tasks/install.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -- name: Calculate dbcache as a share of system RAM - set_fact: - bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" - when: bitcoin_dbcache_mb_override | string | length == 0 - -- name: Use the explicit dbcache override - set_fact: - bitcoin_dbcache_mb: "{{ bitcoin_dbcache_mb_override }}" - when: bitcoin_dbcache_mb_override | string | length > 0 - changed_when: false - -- name: Display calculated dbcache value - debug: - msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" - - -- name: Install build dependencies - apt: - name: - - build-essential - - libtool - - autotools-dev - - automake - - pkg-config - - bsdmainutils - - python3 - - python3-pip - - libevent-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-test-dev - - libboost-thread-dev - - libboost-chrono-dev - - libboost-program-options-dev - - libboost-dev - - libssl-dev - - libdb-dev - - libminiupnpc-dev - - libzmq3-dev - - libnatpmp-dev - - libsqlite3-dev - - git - - curl - - wget - - cmake - state: present - update_cache: yes - -- name: Create bitcoin group - group: - name: "{{ bitcoin_group }}" - system: yes - state: present - -- name: Create bitcoin user - user: - name: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ bitcoin_data_dir }}" - create_home: yes - state: present - -- name: Create bitcoin-knots directory - file: - path: "{{ bitcoin_knots_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Create bitcoin-knots source directory - file: - path: "{{ bitcoin_knots_source_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Create bitcoin data directory (for config, logs, wallets) - file: - path: "{{ bitcoin_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - -- name: Create bitcoin large data directory (for blockchain) - file: - path: "{{ bitcoin_large_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - -- name: Create bitcoin config directory - file: - path: "{{ bitcoin_conf_dir }}" - state: directory - owner: root - group: root - mode: '0755' diff --git a/ansible/roles/bitcoin_knots/tasks/main.yml b/ansible/roles/bitcoin_knots/tasks/main.yml deleted file mode 100644 index ab869a7..0000000 --- a/ansible/roles/bitcoin_knots/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: build.yml -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/bitcoin_knots/tasks/service.yml b/ansible/roles/bitcoin_knots/tasks/service.yml deleted file mode 100644 index 6f6149e..0000000 --- a/ansible/roles/bitcoin_knots/tasks/service.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start bitcoind service - systemd: - name: bitcoind - enabled: yes - state: started - -- name: Wait for bitcoind RPC to be available - uri: - url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" - method: POST - body_format: json - body: - jsonrpc: "1.0" - id: "healthcheck" - method: "getblockchaininfo" - params: [] - user: "{{ bitcoin_rpc_user }}" - password: "{{ bitcoin_rpc_password }}" - status_code: 200 - timeout: 10 - register: rpc_check - until: rpc_check.status == 200 - retries: 30 - delay: 5 - ignore_errors: yes - -- name: Display RPC connection status - debug: - msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" - -# ═════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 deleted file mode 100644 index 5277a3c..0000000 --- a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 +++ /dev/null @@ -1,67 +0,0 @@ -# Bitcoin Knots Configuration -# Generated by Ansible - -# Data directory (blockchain storage) -datadir={{ bitcoin_large_data_dir }} - -# RPC Configuration -server=1 -rpcuser={{ bitcoin_rpc_user }} -rpcpassword={{ bitcoin_rpc_password }} -rpcbind={{ bitcoin_rpc_bind }} -rpcport={{ bitcoin_rpc_port }} -rpcallowip=0.0.0.0/0 - -# Network Configuration -listen=1 -port={{ bitcoin_p2p_port }} -maxconnections={{ bitcoin_max_connections }} - -# Performance -dbcache={{ bitcoin_dbcache_mb }} - -# Transaction Index (optional) -{% if bitcoin_enable_txindex %} -txindex=1 -{% endif %} - -{# The live node carries this comment and the template never produced it, so a - run would have silently deleted it. Harmless in itself, but matching it keeps - this task at `ok` - which means any future `changed` here is a real signal - rather than known noise. #} -# Pruning (optional) - -# Logging -logtimestamps=1 -{% if bitcoin_logfile %} -logfile={{ bitcoin_logfile }} -{% else %} -printtoconsole=1 -{% endif %} - -# ZMQ Configuration -{% if bitcoin_zmq_enabled | default(false) %} -zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} -zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} -zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} -zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} -{% endif %} - -# Security -disablewallet=1 -{% if bitcoin_datum_gateway_enabled %} - -{# These were hand-added on the node and were NOT in this template, so running - the playbook would have stripped them. blocknotify is how datum_gateway - learns a new block landed; without it solo mining keeps grinding on a stale - template - a silent failure that costs money rather than raising an error. - Kept as a Jinja comment so the explanation stays in the repo and out of the - deployed config. #} -# Specific for DATUM gateway -blockmaxsize={{ bitcoin_blockmaxsize }} -blockmaxweight={{ bitcoin_blockmaxweight }} -blocknotify={{ bitcoin_blocknotify }} -maxmempool={{ bitcoin_maxmempool }} -blockreconstructionextratxn={{ bitcoin_blockreconstructionextratxn }} - -{% endif %} diff --git a/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 b/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 deleted file mode 100644 index ac1140f..0000000 --- a/ansible/roles/bitcoin_knots/templates/bitcoind.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Bitcoin Knots daemon -After=network.target - -[Service] -Type=simple -User={{ bitcoin_user }} -Group={{ bitcoin_group }} -ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf -Restart=always -RestartSec=10 -TimeoutStopSec=600 -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 deleted file mode 100644 index 9df056e..0000000 --- a/ansible/roles/bitcoin_knots/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Bitcoin Knots Health Check -After=network.target bitcoind.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 deleted file mode 100644 index 538b8dd..0000000 --- a/ansible/roles/bitcoin_knots/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Bitcoin Knots health check — managed by Ansible (roles/bitcoin_knots) -# -# The exit code is the answer and systemd keeps it: -# systemctl is-failed bitcoin-knots-healthcheck.service -# Reporting anywhere else is optional and generic. -# -# - -RPC_HOST="{{ bitcoin_rpc_bind }}" -RPC_PORT={{ bitcoin_rpc_port }} -RPC_USER="{{ bitcoin_rpc_user }}" -RPC_PASSWORD="{{ bitcoin_rpc_password }}" -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -# Check if bitcoind RPC is responding -check_bitcoind() { - local response - response=$(curl -s --max-time 30 \ - --user "${RPC_USER}:${RPC_PASSWORD}" \ - --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ - --header 'Content-Type: application/json' \ - "http://${RPC_HOST}:${RPC_PORT}" 2>&1) - - if [ $? -eq 0 ]; then - # Check if response contains a non-null error - # Successful responses have "error": null, failures have "error": {...} - if echo "$response" | grep -q '"error":null\|"error": null'; then - return 0 - else - return 1 - fi - else - return 1 - fi -} - -report() { - local status=$1 - local msg=$2 - - # No push URL is normal, not an error: the exit code below is still a - # complete answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - if ! curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${encoded_msg}"; then - return 1 - fi -} - -# Main health check -if check_bitcoind; then - report "up" "OK" - exit 0 -else - report "down" "bitcoind RPC not responding" - exit 1 -fi diff --git a/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 b/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 deleted file mode 100644 index d5857ab..0000000 --- a/ansible/roles/bitcoin_knots/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Bitcoin Knots Health Check Timer -Requires=bitcoind.service - -[Timer] -OnBootSec=1min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/caddy_site/README.md b/ansible/roles/caddy_site/README.md deleted file mode 100644 index c8e32af..0000000 --- a/ansible/roles/caddy_site/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# `caddy_site` - -Writes one Caddy site file into `{{ caddy_sites_dir }}`, makes sure the main -Caddyfile imports that directory, validates the result, and reloads Caddy once. - -Replaces the four-task block that was copy-pasted into 10 playbooks. - -Runs on any host in the `[caddy]` group — `edge` (vipy), `monitoring` -(watchtower) and `vpn_control` (spacey). - -## Usage - -```yaml -- ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: forgejo # -> forgejo.conf - caddy_site_domain: "{{ forgejo_domain }}" - caddy_site_upstream: "localhost:{{ forgejo_port }}" -``` - -Use `include_role`, not a `roles:` block, so the call stays in task order next -to the tasks it depends on. Variables passed this way are scoped to the include -and do not leak into later calls — so **every call must pass everything it -needs**; nothing carries over. - -## Shapes - -Pick exactly one of `caddy_site_upstream`, `caddy_site_root`, `caddy_site_body`. - -| Want | Set | -|---|---| -| `reverse_proxy host:port` | `caddy_site_upstream` | -| static `root *` + `file_server` | `caddy_site_root` | -| anything else | `caddy_site_body` (raw, indented 4 for you) | - -`caddy_site_upstream` accepts two modifiers, which add a block to the -`reverse_proxy`: - -- `caddy_site_headers_up: {"X-Forwarded-Host": "..."}` -- `caddy_site_resolvers: "100.100.100.100"` — Tailscale MagicDNS - -and `caddy_site_basic_auth` wraps the site in a `basic_auth` block. - -## `caddy_site_basic_auth` is a LIST, not a dict - -```yaml -caddy_site_basic_auth: - - user: "{{ datum_dashboard_username }}" - hash: "{{ datum_dashboard_password_hash }}" -``` - -**Ansible does not template dictionary keys.** With `{ "{{ user }}": "hash" }` -the value is rendered and the key is not, so the literal string -`{{ datum_dashboard_username }}` lands in the config file. Found while building -this role; the `assert` refuses a mapping so it cannot happen again. - -## Secrets and `--diff` - -Rendered site files can carry credentials — `datum-gateway.conf` holds a bcrypt -hash — and `--diff` prints rendered content. The template task therefore sets -`diff: "{{ caddy_site_reveal | bool }}"`, default `false`, so `--diff` runs are -safe everywhere. Pass `-e caddy_site_reveal=true` to see what moved on a site -you know is not secret. - -## Validation - -`validate: "caddy validate --adapter caddyfile --config %s"` runs against the -rendered temp file before it is moved into place. Verified on vipy that a single -site fragment validates cleanly (rc=0, `Valid configuration`) and that a -malformed one is rejected (rc=1, with the syntax error and line number). A -failed validate leaves the live file untouched, so a broken config can no longer -reach a running Caddy. - -What it cannot catch is a conflict with the global `/etc/caddy/Caddyfile`. - -## The reload is a handler - -`Reload caddy` fires **once, at the end of the play**, however many sites -notified it. The code this replaced ran `command: systemctl reload caddy` -immediately, mid-play. If a later task in the same play needs the new config to -be live, flush first: - -```yaml -- ansible.builtin.meta: flush_handlers -``` - -## Known intentional difference - -The `resolvers` block is commented `# Use Tailscale MagicDNS to resolve the -upstream hostname` in every case. `datum-gateway` previously said `# Resolve via -Tailscale MagicDNS`. Migrating it therefore rewrites one comment line, which -Caddy ignores. Every other site renders byte-identical to what its playbook -produced. - -## Sites on the hosts that this role does NOT manage - -Four vhosts exist in `/etc/caddy/sites-enabled/` that no playbook writes. They -were made by hand. The role only ever writes the one file it is told to, so it -leaves them alone — but nothing in the repo records them, and that is why they -are listed here. Checked 2026-09-11: - -| File | Host | Serves | State | -|---|---|---|---| -| `uptime-kuma.conf` | watchtower | `localhost:3001` | **HTTP 302 — still live**, see below | -| `arbretstaging.conf` | vipy | `arbret-staging-box:80` via MagicDNS | HTTP 200 | -| `bitcoininfra.conf` | vipy | static `file_server` from `/var/www/bitcoin-services-home` | HTTP 200 | -| `scriberr.conf` | vipy | `scriberr-box:8080` via MagicDNS | HTTP 502 — upstream down | - -**`uptime-kuma.conf` must not be deleted as dead config.** Uptime Kuma was -"decommissioned" in the repo — its playbooks archived and its credentials pulled -from the vault — but the container is **still running** on watchtower -(`louislam/uptime-kuma:latest`, created 2026-02-07, `restart=unless-stopped`) -and is still reachable at its public subdomain. Only the Ansible code was -retired; the service was not. See `archive/uptime_kuma/`. - -`scriberr` returning 502 is the one that looks like genuine rot: it proxies to a -`scriberr-box` that is not answering, and `scriberr-box` is not in the inventory. diff --git a/ansible/roles/caddy_site/defaults/main.yml b/ansible/roles/caddy_site/defaults/main.yml deleted file mode 100644 index 0a59906..0000000 --- a/ansible/roles/caddy_site/defaults/main.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Required -caddy_site_name: "" # file basename -> .conf -caddy_site_domain: "" # site address line; may hold several, comma separated - -# Pick exactly one shape -caddy_site_upstream: "" # "localhost:3000" -> reverse_proxy -caddy_site_root: "" # filesystem path -> root * + file_server -caddy_site_body: "" # raw escape hatch for one-off sites; wins over both - -# reverse_proxy modifiers -caddy_site_resolvers: "" # "100.100.100.100" for Tailscale MagicDNS -caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"} -# A LIST, not a dict: Ansible does not template dict *keys*, so a Jinja -# expression for the username silently passes through as literal text. -caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}] - -# Placement. This is now the only definition of caddy_sites_dir - services_config.yml -# used to carry an identical copy, which was removed as redundant. -caddy_sites_dir: /etc/caddy/sites-enabled - -# Rendered site files can carry credentials (basic_auth hashes), so --diff is -# suppressed by default. Pass -e caddy_site_reveal=true to see what moved. -caddy_site_reveal: false diff --git a/ansible/roles/caddy_site/handlers/main.yml b/ansible/roles/caddy_site/handlers/main.yml deleted file mode 100644 index 4bfa7ff..0000000 --- a/ansible/roles/caddy_site/handlers/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Fires once at the end of the play, however many sites notified it. -# Anything later in the same play that needs the new config live must be -# preceded by `- ansible.builtin.meta: flush_handlers`. -# become is explicit because handlers do not inherit it from the task that -# notified them. headscale's play runs become: no and elevates per task, so -# without this the reload would run unprivileged and fail. -- name: Reload caddy - become: true - ansible.builtin.systemd: - name: caddy - state: reloaded diff --git a/ansible/roles/caddy_site/tasks/main.yml b/ansible/roles/caddy_site/tasks/main.yml deleted file mode 100644 index 5ad3976..0000000 --- a/ansible/roles/caddy_site/tasks/main.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- name: Assert caddy_site parameters are sane - ansible.builtin.assert: - that: - - caddy_site_name | length > 0 - - caddy_site_domain | length > 0 - - (caddy_site_upstream | length > 0) or (caddy_site_root | length > 0) or (caddy_site_body | length > 0) - - caddy_site_basic_auth is not mapping - fail_msg: >- - caddy_site: '{{ caddy_site_name | default("") }}' needs a name, a domain and - one of caddy_site_upstream / caddy_site_root / caddy_site_body. - caddy_site_basic_auth must be a LIST of {user, hash} — Ansible does not template dict keys. - quiet: true - -- name: Ensure Caddy sites-enabled directory exists - ansible.builtin.file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Ensure Caddyfile imports sites-enabled - ansible.builtin.lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - create: yes - mode: '0644' - backup: yes - -# `validate` runs `caddy validate` against the rendered temp file before it is -# moved into place: verified on vipy that a single site fragment validates -# cleanly (rc=0, "Valid configuration") and that a malformed one is rejected -# (rc=1). A failed validate leaves the live file untouched. -- name: "Write Caddy site '{{ caddy_site_name }}'" - ansible.builtin.template: - src: site.conf.j2 - dest: "{{ caddy_sites_dir }}/{{ caddy_site_name }}.conf" - owner: root - group: root - mode: '0644' - validate: "caddy validate --adapter caddyfile --config %s" - diff: "{{ caddy_site_reveal | bool }}" - notify: Reload caddy diff --git a/ansible/roles/caddy_site/templates/site.conf.j2 b/ansible/roles/caddy_site/templates/site.conf.j2 deleted file mode 100644 index 5d34c01..0000000 --- a/ansible/roles/caddy_site/templates/site.conf.j2 +++ /dev/null @@ -1,34 +0,0 @@ -{{ caddy_site_domain }} { -{% if caddy_site_body %} -{{ caddy_site_body | trim | indent(4, first=True) }} -{% else %} -{% if caddy_site_basic_auth %} - basic_auth { -{% for cred in caddy_site_basic_auth %} - {{ cred.user }} {{ cred.hash }} -{% endfor %} - } -{% endif %} -{% if caddy_site_root %} - root * {{ caddy_site_root }} - file_server -{% endif %} -{% if caddy_site_upstream %} -{% if caddy_site_headers_up or caddy_site_resolvers %} - reverse_proxy {{ caddy_site_upstream }} { -{% for key, value in caddy_site_headers_up.items() %} - header_up {{ key }} {{ value }} -{% endfor %} -{% if caddy_site_resolvers %} - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers {{ caddy_site_resolvers }} - } -{% endif %} - } -{% else %} - reverse_proxy {{ caddy_site_upstream }} -{% endif %} -{% endif %} -{% endif %} -} diff --git a/ansible/roles/datum_gateway/README.md b/ansible/roles/datum_gateway/README.md deleted file mode 100644 index cdf35e5..0000000 --- a/ansible/roles/datum_gateway/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# `datum_gateway` - -Builds and runs [DATUM Gateway](https://github.com/OCEAN-xyz/datum_gateway), the -solo/pooled mining gateway, on `knots-box`. The calling playbook adds two more -plays on the edge host: the dashboard via `caddy_site`, and the public Stratum -port via `socket_proxy`. - -Converted from `deploy_datum_gateway_playbook.yml` (802 lines) under Plan 6. The -playbook is now 68 lines and keeps all three plays. - -## ⚠ This is half of a system - -The Bitcoin Knots node on the same host feeds this gateway through -`blocknotify=killall -USR1 datum_gateway` in `bitcoin.conf` — see -`roles/bitcoin_knots/README.md`, where that line was found to be missing from the -template entirely. **Changing either config means thinking about both.** - -Interrupting Stratum costs mining shares. Check before any run that restarts it: - -```bash -ss -tn state established '( sport = :23334 )' -``` - -## Two pieces of drift where the node was right - -The repo and the node had diverged on values that matter, and the deployment -would have applied the repo's: - -| | node (correct) | repo said | -|---|---|---| -| `datum_mining_address` | `bc1qvrj3g84…` | `bc1qdse9dsg…` | -| `pool_pass_workers` / `_full_users` | `false` | `true` | - -The address is the one that would have hurt: **it is where block rewards are -paid**, and unlike fulcrum and bitcoin-knots the `Restart datum-gateway` handler -here was *never* gated, so the change would have applied immediately rather than -sitting inert. Both corrected in the vault and defaults, with notes. - -Verify semantics rather than text when touching `config.json` — render it and -compare parsed JSON, because the live file is single-line and the template is -pretty-printed, so a textual diff is all noise: - -```python -json.load(open('live.json')) == json.load(open('rendered.json')) -``` - -## `config.json` holds real secrets — diff is suppressed - -The file carries `bitcoind.rpcpassword` and `api.admin_password`. `--diff` -prints rendered content, so the task sets `diff: false` by default; pass -`-e datum_reveal_config=true` to opt in. - -Note `pool_pass_workers` / `pool_pass_full_users` are **booleans**, not -passwords, despite the names — they control DATUM's pool-password passthrough. -`mining.pool_address` is a Bitcoin address and public by nature. - -## Expect `changed` on the compile every run - -`Configure cmake build` and `Compile datum_gateway` are bare `command:` tasks -with no `changed_when`, so they always report changed and always re-run. The -build is reproducible — `Install datum_gateway binary` sees identical content and -does not replace it, so the installed binary keeps its original timestamp — but -the compile itself is wasted work on every run. That is the idempotent floor, not -drift. - -## Monitoring: one variable, no product knowledge - -The check tests the gateway API and records the answer in its exit code, which -systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. Set -`healthcheck_push_url` to report anywhere accepting an HTTP ping. - -Unlike the other services here, only the health-check *timer* handler was gated -by `uptime_kuma_enabled`; the main deployment restart worked throughout. diff --git a/ansible/roles/datum_gateway/defaults/main.yml b/ansible/roles/datum_gateway/defaults/main.yml deleted file mode 100644 index 0c30aa7..0000000 --- a/ansible/roles/datum_gateway/defaults/main.yml +++ /dev/null @@ -1,61 +0,0 @@ -# DATUM Gateway Configuration Variables -# https://github.com/OCEAN-xyz/datum_gateway - -# Version - pin to a specific tag -datum_gateway_version: "v0.4.1beta" - -# Directories -datum_gateway_dir: /opt/datum-gateway -datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" -datum_gateway_config_dir: /etc/datum-gateway -datum_gateway_log_dir: /var/log/datum-gateway - -# Binary -datum_gateway_bin_path: /usr/local/bin/datum_gateway - -# Ports -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/knots_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -datum_gateway_stratum_port: 23334 -datum_gateway_api_port: 7152 - -# Stratum settings -datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) - -# Service user -datum_gateway_user: datum -datum_gateway_group: datum - -# Build options -datum_gateway_build_jobs: 4 - -# Bitcoin node connection -# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. -# datum_bitcoin_rpc_url should include http:// and port. -datum_bitcoin_rpc_url: "http://127.0.0.1:8332" -# Note: bitcoin_rpc_user and bitcoin_rpc_password come from group_vars/all/vault.yml - -# Mining config -datum_coinbase_tag_primary: "DATUM" -datum_coinbase_tag_secondary: "BY ORDER OF BIP110" - -# Both false on the node; the vars file said true. Corrected 2026-09-13 to -# match reality, on the same basis as datum_mining_address: the running node -# is authoritative. These control DATUM's pool-password passthrough. -datum_pool_pass_workers: false -datum_pool_pass_full_users: false -datum_pooled_mining_only: true - - -# --- Health check ----------------------------------------------------------- -# Checks the DATUM Gateway API and records the answer in its exit code, which -# systemd keeps: `systemctl is-failed datum-gateway-healthcheck.service`. -# -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means -# check, exit honestly, report nowhere. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" diff --git a/ansible/roles/datum_gateway/handlers/main.yml b/ansible/roles/datum_gateway/handlers/main.yml deleted file mode 100644 index b7cfe25..0000000 --- a/ansible/roles/datum_gateway/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Restart datum-gateway - systemd: - name: datum-gateway - state: restarted - daemon_reload: yes - -# Ungated. This one carried `when: uptime_kuma_enabled | default(false)` while -# the main Restart datum-gateway handler above did not — so on this service the -# deployment restart worked and only the health-check timer restart was dead. -- name: Restart datum-gateway health check timer - systemd: - name: datum-gateway-healthcheck.timer - state: restarted - daemon_reload: yes diff --git a/ansible/roles/datum_gateway/tasks/configure.yml b/ansible/roles/datum_gateway/tasks/configure.yml deleted file mode 100644 index 463dec0..0000000 --- a/ansible/roles/datum_gateway/tasks/configure.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Ownership copied verbatim from the playbook this replaces and verified -# mechanically against `git show HEAD:`. -- name: Write DATUM Gateway config.json - ansible.builtin.template: - src: config.json.j2 - dest: "{{ datum_gateway_config_dir }}/config.json" - owner: "{{ datum_gateway_user }}" - group: "{{ datum_gateway_group }}" - mode: '0640' - # config.json carries the bitcoind RPC password, the API admin password and - # the pool passwords. `--diff` prints rendered content, so running with --diff - # put all of them on the terminal and into any log capturing it. Suppressed by - # default; pass -e datum_reveal_config=true when you genuinely need the diff. - diff: "{{ datum_reveal_config | default(false) | bool }}" - notify: Restart datum-gateway - -- name: Create datum-gateway systemd service - ansible.builtin.template: - src: datum-gateway.service.j2 - dest: /etc/systemd/system/datum-gateway.service - owner: root - group: root - mode: '0644' - notify: Restart datum-gateway diff --git a/ansible/roles/datum_gateway/tasks/healthcheck.yml b/ansible/roles/datum_gateway/tasks/healthcheck.yml deleted file mode 100644 index b5ccb90..0000000 --- a/ansible/roles/datum_gateway/tasks/healthcheck.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -# Everything here answers "is DATUM Gateway healthy" and records the answer. The -# Uptime Kuma specifics that used to follow — an embedded Python script creating -# monitors over the API, a /tmp credentials file, a push-URL file read back and -# parsed, and a systemd Environment= rewrite — are gone. Where it reports is now -# one variable, healthcheck_push_url. -- name: Create DATUM Gateway health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: /usr/local/bin/datum-gateway-healthcheck-push.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create datum-gateway health check systemd service - ansible.builtin.template: - src: healthcheck.service.j2 - dest: /etc/systemd/system/datum-gateway-healthcheck.service - owner: root - group: root - mode: "0600" - notify: Restart datum-gateway health check timer - -- name: Create datum-gateway health check systemd timer - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: /etc/systemd/system/datum-gateway-healthcheck.timer - owner: root - group: root - mode: '0644' - notify: Restart datum-gateway health check timer - -- name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - -# Ungated: enabling a timer is deployment, not monitoring. -- name: Enable and restart the datum-gateway health check timer - systemd: - name: datum-gateway-healthcheck.timer - enabled: yes - state: restarted - daemon_reload: yes - -# Arms the timer and smoke-tests the check. See roles/bitcoin_knots/README.md for -# why restarting the timer alone is not enough with OnBootSec + OnUnitActiveSec. -- name: Run the DATUM Gateway health check once to arm the timer - command: systemctl start datum-gateway-healthcheck.service - changed_when: false diff --git a/ansible/roles/datum_gateway/tasks/install.yml b/ansible/roles/datum_gateway/tasks/install.yml deleted file mode 100644 index 61dcadb..0000000 --- a/ansible/roles/datum_gateway/tasks/install.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Install DATUM Gateway build dependencies - apt: - name: - - cmake - - build-essential - - git - - libjansson-dev - - libmicrohttpd-dev - - libsodium-dev - - libcurl4-openssl-dev - # Runtime-only (netcat for health check) - - netcat-openbsd - state: present - update_cache: yes - -# =========================================== -# System User and Directories -# =========================================== -- name: Create datum system user - user: - name: "{{ datum_gateway_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ datum_gateway_dir }}" - create_home: no - comment: "DATUM Gateway" - -- name: Create DATUM Gateway directories - file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ datum_gateway_group }}" - mode: "{{ item.mode }}" - loop: - - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } - - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } - -# =========================================== -# Build from Source -# =========================================== -- name: Clone DATUM Gateway repository at {{ datum_gateway_version }} - git: - repo: https://github.com/OCEAN-xyz/datum_gateway.git - dest: "{{ datum_gateway_source_dir }}" - version: "{{ datum_gateway_version }}" - force: yes - register: git_clone - -- name: Configure cmake build - command: cmake . -DCMAKE_BUILD_TYPE=Release - args: - chdir: "{{ datum_gateway_source_dir }}" - -- name: Compile datum_gateway - command: make -j{{ datum_gateway_build_jobs }} - args: - chdir: "{{ datum_gateway_source_dir }}" - -- name: Install datum_gateway binary - copy: - src: "{{ datum_gateway_source_dir }}/datum_gateway" - dest: "{{ datum_gateway_bin_path }}" - remote_src: yes - owner: root - group: root - mode: "0755" - notify: Restart datum-gateway - -# =========================================== -# Configuration diff --git a/ansible/roles/datum_gateway/tasks/main.yml b/ansible/roles/datum_gateway/tasks/main.yml deleted file mode 100644 index 57d9a45..0000000 --- a/ansible/roles/datum_gateway/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/datum_gateway/tasks/service.yml b/ansible/roles/datum_gateway/tasks/service.yml deleted file mode 100644 index c8581fd..0000000 --- a/ansible/roles/datum_gateway/tasks/service.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start datum-gateway - systemd: - name: datum-gateway - enabled: yes - state: started - -# =========================================== -# Health Check Script + Systemd Timer -# =========================================== -# ═════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/roles/datum_gateway/templates/config.json.j2 b/ansible/roles/datum_gateway/templates/config.json.j2 deleted file mode 100644 index e17a386..0000000 --- a/ansible/roles/datum_gateway/templates/config.json.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{ - "bitcoind": { - "rpcuser": "{{ bitcoin_rpc_user }}", - "rpcpassword": "{{ bitcoin_rpc_password }}", - "rpcurl": "{{ datum_bitcoin_rpc_url }}", - "notify_fallback": true - }, - "stratum": { - "listen_port": {{ datum_gateway_stratum_port }}, - "vardiff_min": {{ datum_vardiff_min }} - }, - "mining": { - "pool_address": "{{ datum_mining_address }}", - "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", - "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" - }, - "api": { - "admin_password": "{{ datum_gateway_admin_password }}", - "listen_port": {{ datum_gateway_api_port }}, - "modify_conf": false - }, - "logger": { - "log_to_console": true, - "log_to_file": true, - "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", - "log_rotate_daily": true, - "log_level_console": 2, - "log_level_file": 1 - }, - "datum": { - "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, - "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, - "pooled_mining_only": {{ datum_pooled_mining_only | lower }} - } -} diff --git a/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 b/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 deleted file mode 100644 index 18f7a3c..0000000 --- a/ansible/roles/datum_gateway/templates/datum-gateway.service.j2 +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=DATUM Gateway - Bitcoin Mining Gateway -Documentation=https://github.com/OCEAN-xyz/datum_gateway -After=network.target bitcoind.service -Wants=bitcoind.service - -[Service] -User={{ datum_gateway_user }} -Group={{ datum_gateway_group }} -Type=simple -ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json -Restart=on-failure -RestartSec=10 -StandardOutput=journal -StandardError=journal - -# Prevent config from being read by other users -ReadWritePaths={{ datum_gateway_log_dir }} -ReadOnlyPaths={{ datum_gateway_config_dir }} - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.service.j2 b/ansible/roles/datum_gateway/templates/healthcheck.service.j2 deleted file mode 100644 index e21672e..0000000 --- a/ansible/roles/datum_gateway/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=DATUM Gateway Health Check -After=network.target datum-gateway.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 b/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 deleted file mode 100644 index 43ec6b2..0000000 --- a/ansible/roles/datum_gateway/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# DATUM Gateway health check — managed by Ansible (roles/datum_gateway) -# -# The exit code is the answer and systemd keeps it: -# systemctl is-failed datum-gateway-healthcheck.service -# Reporting anywhere else is optional and generic. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -STRATUM_PORT={{ datum_gateway_stratum_port }} - -check_datum() { - # Service must be active and stratum port must be listening - systemctl is-active --quiet datum-gateway && \ - nc -z 127.0.0.1 "${STRATUM_PORT}" -} - -report() { - local status=$1 - local msg=$2 - # No push URL is normal, not an error: the exit code below is still a - # complete answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true -} - -if check_datum; then - report "up" "OK" - exit 0 -else - report "down" "DATUM Gateway not responding" - exit 1 -fi diff --git a/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 b/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 deleted file mode 100644 index c14d1d2..0000000 --- a/ansible/roles/datum_gateway/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=DATUM Gateway Health Check Timer - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/forgejo_runner/README.md b/ansible/roles/forgejo_runner/README.md deleted file mode 100644 index e7f761b..0000000 --- a/ansible/roles/forgejo_runner/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# `forgejo_runner` - -Installs and runs a Forgejo Actions runner, registers it with the Forgejo -instance, and keeps a health check on a systemd timer. - -Converted from `deploy_forgejo_runner_playbook.yml` (409 lines) under Plan 6. -The playbook is now 16 lines. - -## Phases - -`tasks/main.yml` imports five files in order: - -| | | -|---|---| -| `prerequisites.yml` | Docker must be present | -| `install.yml` | binary, system user, working directory | -| `configure.yml` | config file, registration with the instance | -| `service.yml` | systemd unit, start, assert it came up | -| `healthcheck.yml` | check script, unit, timer | - -`import_tasks`, not `include_tasks` — static imports are visible to -`--list-tasks`, which is how the conversion was verified against the playbook it -replaced. - -## Monitoring: one variable, no product knowledge - -This role contains **nothing specific to any monitoring system**. What used to -be here — an ~80-line embedded Python script creating monitors over the Uptime -Kuma API, a `/tmp` credentials file, token extraction, a systemd `Environment=` -rewrite, and 8 `when: uptime_kuma_enabled` guards — is gone. - -What remains answers the actual question, *is this service healthy*, and records -it two ways: - -- **the exit code**, which systemd keeps: `systemctl is-failed - forgejo-runner-healthcheck.service` is a complete answer with no monitoring - system involved at all; -- **a log file** at `{{ healthcheck_log_file }}`. - -To report health somewhere, set one variable: - -```yaml -healthcheck_push_url: "https://example/api/push/TOKEN" -``` - -Any endpoint accepting an HTTP ping works. Empty (the default) means check, log, -exit honestly, report nowhere — which is also the right setting for a *pull*-based -monitor like Prometheus' textfile collector, since that reads unit state instead. - -The push URL is a credential (anyone holding it can forge an "up"), so callers -pass it from the vault rather than committing it. - -## One behaviour change, deliberate - -`Assert runner is running` used to be guarded by `uptime_kuma_enabled`, so it -never ran. It is not a monitoring task — it is the deployment checking its own -work — and the deprecation banner swept it up by mistake. It is ungated here, -which means a runner that fails to start now fails the play instead of -deploying "successfully" in silence. diff --git a/ansible/roles/forgejo_runner/defaults/main.yml b/ansible/roles/forgejo_runner/defaults/main.yml deleted file mode 100644 index aba6b61..0000000 --- a/ansible/roles/forgejo_runner/defaults/main.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -# Binary -forgejo_runner_version: "6.3.1" -forgejo_runner_arch: "linux-amd64" -forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" -forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" - -# Runtime -forgejo_runner_user: "runner" -forgejo_runner_dir: "/opt/forgejo-runner" -forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" -forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" - -# The Forgejo instance this runner registers with. -forgejo_instance_url: "https://forgejo.contrapeso.xyz" -# forgejo_runner_registration_token comes from the vault. - -# --- Health check ----------------------------------------------------------- -# The check answers "is this service healthy" and records the answer two ways: -# a log file, and its own exit code. The exit code is the durable artefact — -# systemd stores it, so `systemctl is-failed forgejo-runner-healthcheck.service` -# answers the question with no monitoring system involved at all. -healthcheck_interval_seconds: 60 -healthcheck_script_dir: /opt/forgejo-runner-healthcheck -healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" -healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" -healthcheck_service_name: forgejo-runner-healthcheck - -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. -# -# Empty means "check, log, exit honestly, report nowhere". Set it to any URL -# that accepts an HTTP ping and the check will report there. Nothing in this -# role is specific to a particular monitoring product: the Uptime Kuma API -# calls, monitor creation and token handling that used to live here are gone. -# -# A pull-based monitor (Prometheus node_exporter textfile, say) needs this left -# empty — it reads the systemd unit state instead. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" diff --git a/ansible/roles/forgejo_runner/tasks/configure.yml b/ansible/roles/forgejo_runner/tasks/configure.yml deleted file mode 100644 index ab96d52..0000000 --- a/ansible/roles/forgejo_runner/tasks/configure.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Check if config already exists - stat: - path: "{{ forgejo_runner_config_path }}" - register: config_stat - -- name: Generate default config - shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not config_stat.stat.exists - -- name: Set config file ownership - file: - path: "{{ forgejo_runner_config_path }}" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not config_stat.stat.exists - -# ── 6. Register runner ───────────────────────────────────────────── -- name: Check if runner is already registered - stat: - path: "{{ forgejo_runner_dir }}/.runner" - register: runner_stat - -- name: Register runner with Forgejo instance - command: > - {{ forgejo_runner_bin_path }} register --no-interactive - --instance {{ forgejo_instance_url }} - --token {{ forgejo_runner_registration_token }} - --name forgejo-runner-box - --labels "{{ forgejo_runner_labels }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not runner_stat.stat.exists - -- name: Set runner registration file ownership - file: - path: "{{ forgejo_runner_dir }}/.runner" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not runner_stat.stat.exists - diff --git a/ansible/roles/forgejo_runner/tasks/healthcheck.yml b/ansible/roles/forgejo_runner/tasks/healthcheck.yml deleted file mode 100644 index 42fbaf7..0000000 --- a/ansible/roles/forgejo_runner/tasks/healthcheck.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -# Everything here answers "is the service healthy" and records the answer. -# The Uptime Kuma specifics that used to surround it — an embedded Python script -# that created monitors over the API, a /tmp credentials file, token extraction, -# and a systemd Environment= rewrite — are gone. What reports where is now one -# variable, healthcheck_push_url. See the role README. -- name: Create healthcheck script directory - ansible.builtin.file: - path: "{{ healthcheck_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: Create forgejo-runner healthcheck script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: "{{ healthcheck_script_path }}" - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create healthcheck systemd service - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" - owner: root - group: root - mode: "0600" - -- name: Create healthcheck systemd timer - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" - owner: root - group: root - mode: '0644' - -- name: Reload systemd for healthcheck units - systemd: - daemon_reload: yes - -- name: Enable and start healthcheck timer - systemd: - name: "{{ healthcheck_service_name }}.timer" - enabled: yes - state: started - -- name: Test healthcheck script - command: "{{ healthcheck_script_path }}" - register: healthcheck_test - changed_when: false - -- name: Verify healthcheck script works - assert: - that: - - healthcheck_test.rc == 0 - fail_msg: "Healthcheck script failed to execute properly" - -- name: Display deployment summary - debug: - msg: | - Forgejo Runner deployed successfully! - - Runner Name: forgejo-runner-box - Instance: {{ forgejo_instance_url }} - Working Directory: {{ forgejo_runner_dir }} - Service: forgejo-runner.service ({{ runner_active.stdout }}) - - Healthcheck Monitor: {{ healthcheck_service_name }} - Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s - Reporting to: {{ healthcheck_push_url | default('', true) | regex_replace('/api/push/.*', '/api/push/***') | default('(nowhere - set healthcheck_push_url)', true) }} diff --git a/ansible/roles/forgejo_runner/tasks/install.yml b/ansible/roles/forgejo_runner/tasks/install.yml deleted file mode 100644 index b95ab4c..0000000 --- a/ansible/roles/forgejo_runner/tasks/install.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Download forgejo-runner binary - get_url: - url: "{{ forgejo_runner_url }}" - dest: "{{ forgejo_runner_bin_path }}" - mode: '0755' - -# ── 3. Create runner system user ─────────────────────────────────── -- name: Create runner system user - user: - name: "{{ forgejo_runner_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ forgejo_runner_dir }}" - create_home: no - groups: docker - append: yes - comment: 'Forgejo Runner' - -# ── 4. Create working directory ──────────────────────────────────── -- name: Create forgejo-runner working directory - file: - path: "{{ forgejo_runner_dir }}" - state: directory - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - mode: '0750' diff --git a/ansible/roles/forgejo_runner/tasks/main.yml b/ansible/roles/forgejo_runner/tasks/main.yml deleted file mode 100644 index aa71001..0000000 --- a/ansible/roles/forgejo_runner/tasks/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# import_tasks, not include_tasks: these are unconditional phases, and static -# imports are visible to `--list-tasks`. That matters because the task list is -# how this refactor was verified against the playbook it replaced. -- ansible.builtin.import_tasks: prerequisites.yml -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/forgejo_runner/tasks/prerequisites.yml b/ansible/roles/forgejo_runner/tasks/prerequisites.yml deleted file mode 100644 index d441709..0000000 --- a/ansible/roles/forgejo_runner/tasks/prerequisites.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Check if Docker is installed - command: docker --version - register: docker_check - changed_when: false - failed_when: docker_check.rc != 0 - -- name: Fail if Docker is not available - assert: - that: - - docker_check.rc == 0 - fail_msg: "Docker is required for forgejo-runner but is not installed" diff --git a/ansible/roles/forgejo_runner/tasks/service.yml b/ansible/roles/forgejo_runner/tasks/service.yml deleted file mode 100644 index f971e52..0000000 --- a/ansible/roles/forgejo_runner/tasks/service.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Create forgejo-runner systemd service - ansible.builtin.template: - src: forgejo-runner.service.j2 - dest: /etc/systemd/system/forgejo-runner.service - owner: root - group: root - mode: '0644' - -- name: Reload systemd - systemd: - daemon_reload: yes - -- name: Enable and start forgejo-runner service - systemd: - name: forgejo-runner - enabled: yes - state: started - -- name: Verify forgejo-runner is active - command: systemctl is-active forgejo-runner - register: runner_active - changed_when: false - -# Ungated on purpose. This was previously guarded by `uptime_kuma_enabled`, but -# it is not a monitoring task — it is the deployment asserting its own success. -# The deprecation banner swept it up along with the Kuma plumbing, which meant a -# broken runner deployed "successfully" and silently. -- name: Assert runner is running - assert: - that: - - runner_active.stdout == "active" - fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" diff --git a/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 b/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 deleted file mode 100644 index d3db25d..0000000 --- a/ansible/roles/forgejo_runner/templates/forgejo-runner.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Forgejo Runner -Documentation=https://forgejo.org/docs/latest/admin/actions/ -After=docker.service -Requires=docker.service - -[Service] -Type=simple -User={{ forgejo_runner_user }} -Group={{ forgejo_runner_user }} -WorkingDirectory={{ forgejo_runner_dir }} -ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 deleted file mode 100644 index 173f42b..0000000 --- a/ansible/roles/forgejo_runner/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Forgejo Runner Healthcheck -After=network.target - -[Service] -Type=oneshot -ExecStart={{ healthcheck_script_path }} -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -User=root -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 deleted file mode 100644 index 9aa5ac1..0000000 --- a/ansible/roles/forgejo_runner/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Forgejo Runner healthcheck — managed by Ansible (roles/forgejo_runner) -# -# Answers "is forgejo-runner healthy" and records it two ways: this log, and the -# exit code. The exit code is the durable artefact — systemd keeps it, so -# systemctl is-failed {{ healthcheck_service_name }}.service -# answers the question with no monitoring system involved. -# -# Reporting is optional and generic: if a push URL is configured it also pings -# it. Nothing here knows or cares which monitoring product is on the other end. - -LOG_FILE="{{ healthcheck_log_file }}" -# Read from the environment rather than templated in, so the unit file is the -# only place the token lives and the script is not secret. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" -} - -# Gatus external endpoint: a POST with a bearer token and success=true|false. -# -# This used to report ONLY success - it exited before pushing when the runner -# was down - so a failure was invisible until the heartbeat window expired. -# Reporting the failure is the whole point of having a check. -report() { - local ok="$1" msg="$2" - [ -n "$PUSH_URL" ] || return 0 - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${ok}&error=${msg// /%20}" || true -} - -main() { - if ! systemctl is-active --quiet forgejo-runner; then - log_message "ERROR: forgejo-runner is not active" - report false "forgejo-runner is not active" - exit 1 - fi - - log_message "forgejo-runner is active" - report true "active" - exit 0 -} - -main diff --git a/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 b/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 deleted file mode 100644 index 4cba51a..0000000 --- a/ansible/roles/forgejo_runner/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Run Forgejo Runner Healthcheck every minute -Requires={{ healthcheck_service_name }}.service - -[Timer] -OnBootSec=30sec -OnUnitActiveSec={{ healthcheck_interval_seconds }}sec -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/fulcrum/README.md b/ansible/roles/fulcrum/README.md deleted file mode 100644 index ddefabb..0000000 --- a/ansible/roles/fulcrum/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# `fulcrum` - -Deploys [Fulcrum](https://github.com/cculianu/Fulcrum), an Electrum server -indexing the Bitcoin Knots node, on `fulcrum-box`. The second play in the -calling playbook publishes its SSL port from the edge host via `socket_proxy`. - -Converted from `deploy_fulcrum_playbook.yml` (685 lines) under Plan 6. The -playbook is now 33 lines. - -## The index is the expensive thing - -`{{ fulcrum_db_dir }}` is ~192 GB and takes days to rebuild. Nothing in this role -touches it beyond `state: directory` with the ownership it already has -(`fulcrum:fulcrum 0755`). Restarting Fulcrum re-opens the database; it does not -reindex. - -## Three things this conversion fixed, all pre-existing - -**`bitcoind` pointed at the wrong machine.** The vars file carried -`bitcoin_rpc_host: "192.168.1.140"` commented "IP of knots_box_local", but `.140` -is **fulcrum-box itself** — knots-box is `.135`. The DHCP leases had reshuffled. -The live config had already been hand-corrected to `knots-box`; running the -playbook would have reverted it and broken indexing. Now addressed by Tailscale -name, like everything else in this repo. - -**The restart handler was inert.** It carried -`when: uptime_kuma_enabled | default(false)`, so the three tasks that notify it -(SSL certificate, `fulcrum.conf`, systemd unit) could not restart anything. A -configuration change would write to disk, report success, and silently never take -effect. Ungated. - -**`db_mem` was about to quadruple.** The role computes a share of RAM; on this -5931 MB host 75% is 4448 MB, leaving ~1.4 GB for the OS and Fulcrum's non-cache -memory. The live value had been hand-tuned to 2048. `fulcrum_db_mem_mb_override` -pins it. Note `set_fact` outranks role defaults, so the *calculation* has to -honour the override — pinning it in `defaults/` alone is silently ignored. - -## The health check timer, and how to read it - -The timer is `OnBootSec` + `OnUnitActiveSec` with no `OnCalendar`. That -combination has a failure mode worth knowing: `OnBootSec` is monotonic and -elapses once; `OnUnitActiveSec` schedules relative to the **service** last being -active. If the service does not run in a given boot, there is no reference to -schedule from and the timer sits `active` and `enabled` doing nothing. That is -exactly what had happened here — last trigger **2026-02-17**, seven months of no -health check, with every surface-level indicator green. - -Restarting the timer does not supply that reference; running the service does. -So the role runs the check once after enabling the timer, which is both the fix -and a smoke test. - -**Diagnosing this is easy to get wrong**: `NextElapseUSecRealtime` is always -empty for a monotonic timer, so it looks broken even when it is fine. Read -`NextElapseUSecMonotonic`, or just use `systemctl list-timers`. - -The timer also no longer carries `Requires=fulcrum.service`. On a timer that -means "stop watching when the watched thing stops", which is backwards for a -health check. - -## Monitoring: one variable, no product knowledge - -The check tests the Electrum TCP port and records the answer in its exit code, -which systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. To report -elsewhere set `healthcheck_push_url` to any endpoint accepting an HTTP ping. The -Uptime Kuma API calls, monitor creation and token handling are gone. diff --git a/ansible/roles/fulcrum/defaults/main.yml b/ansible/roles/fulcrum/defaults/main.yml deleted file mode 100644 index 3e97fa0..0000000 --- a/ansible/roles/fulcrum/defaults/main.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Fulcrum Configuration Variables - -# Version - Pinned to specific release -fulcrum_version: "2.1.0" # Fulcrum version to install - -# Directories -fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) -fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) -fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk -fulcrum_binary_path: /usr/local/bin/Fulcrum - -# Network - Bitcoin RPC connection -# Bitcoin Knots is on a different host (knots_box_local) -# Using RPC user/password authentication (credentials from group_vars/all/vault.yml) -# Addressed by Tailscale name, never a LAN IP. This was -# bitcoin_rpc_host: "192.168.1.140" # IP of knots_box_local -# but .140 is fulcrum-box ITSELF - knots-box is .135. The DHCP leases had -# reshuffled (the same drift that transposed the inventory), so running this -# playbook would have pointed Fulcrum at itself and broken indexing. The live -# config had already been hand-corrected to knots-box; this makes the repo -# agree with it. -bitcoin_rpc_host: "knots-box" -bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml - -# Network - Fulcrum server -fulcrum_tcp_port: 50001 -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/fulcrum_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -fulcrum_ssl_port: 50002 -# Binding address for Fulcrum TCP/SSL server: -# - "127.0.0.1" = localhost only (use when Caddy is on the same box) -# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) -# - Specific IP = bind to specific network interface -fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) -fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port -# If Caddy is on a different box, set this to the IP address that Caddy will use to connect - -# SSL/TLS Configuration -fulcrum_ssl_enabled: true -fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" -fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" -fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert - - -# Performance -# db_mem will be calculated as 75% of available RAM automatically in playbook -# db_mem is computed as this share of RAM unless fulcrum_db_mem_mb is set -# explicitly. On a 5931 MB host 75% is 4448 MB, which leaves ~1.4 GB for the -# OS and for Fulcrum's non-cache memory; the live config had been hand-tuned -# down to 2048 and that setting is respected below. -fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache - -# Configuration options -fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs -fulcrum_peering: false # Disable peering with other Fulcrum servers -fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications - -# Service user -fulcrum_user: fulcrum -fulcrum_group: fulcrum - - -# --- Health check ----------------------------------------------------------- -# Checks the Electrum TCP port and records the answer in its exit code, which -# systemd keeps: `systemctl is-failed fulcrum-healthcheck.service`. -# -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means -# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping -# works; nothing here is specific to a monitoring product. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" - -# Explicit db_mem in MB. When set it wins over fulcrum_db_mem_percent; empty -# means compute from RAM. Set here because the live host had been hand-tuned to -# 2048 and a silent jump to 4448 is not something a refactor should do. -fulcrum_db_mem_mb_override: 2048 diff --git a/ansible/roles/fulcrum/handlers/main.yml b/ansible/roles/fulcrum/handlers/main.yml deleted file mode 100644 index c0d9cc4..0000000 --- a/ansible/roles/fulcrum/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# Ungated on purpose. The hand-written handler carried -# when: uptime_kuma_enabled | default(false) -# so it has been inert since the decommissioning: three tasks notify it (the SSL -# certificate, fulcrum.conf and the systemd unit), and none of them could -# actually restart Fulcrum. A configuration change therefore applied to disk and -# silently never took effect — the worst kind of quiet failure, because the -# playbook reports success and the running service keeps its old settings. -# -# Restarting Fulcrum re-opens its database; it does not reindex. -- name: Restart fulcrum - systemd: - name: fulcrum - state: restarted - daemon_reload: yes diff --git a/ansible/roles/fulcrum/tasks/healthcheck.yml b/ansible/roles/fulcrum/tasks/healthcheck.yml deleted file mode 100644 index 983bffb..0000000 --- a/ansible/roles/fulcrum/tasks/healthcheck.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -# Everything here answers "is Fulcrum healthy" and records the answer. The -# Uptime Kuma specifics that used to follow — an embedded Python script creating -# monitors over the API, a /tmp credentials file, push-URL extraction and a -# systemd Environment= rewrite — are gone. Where it reports is now one variable, -# healthcheck_push_url. See the role README. -- name: Create Fulcrum health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: /usr/local/bin/fulcrum-healthcheck-push.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - -- name: Create systemd service for Fulcrum health check - ansible.builtin.template: - src: healthcheck.service.j2 - dest: /etc/systemd/system/fulcrum-healthcheck.service - owner: root - group: root - mode: "0600" - -- name: Create systemd timer for Fulcrum health check - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: /etc/systemd/system/fulcrum-healthcheck.timer - owner: root - group: root - mode: '0644' - -- name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - -# state: restarted, not started. The hand-written timer had got itself stuck -# `active` with no next elapse and had not fired since 2026-02-17; `started` on -# an already-active timer is a no-op and would have left it stuck. Restarting -# re-arms it. See the note in healthcheck.timer.j2. -- name: Enable and restart the Fulcrum health check timer - systemd: - name: fulcrum-healthcheck.timer - enabled: yes - state: restarted - daemon_reload: yes - -# Run the check once, which is both a smoke test and the thing that actually -# arms the timer. -# -# This timer is OnBootSec + OnUnitActiveSec with no OnCalendar. OnBootSec is -# monotonic and had long since elapsed; OnUnitActiveSec schedules relative to the -# SERVICE last being active, and the service had not run since 2026-02-17 — so -# there was no reference to schedule from and the timer sat `active` and -# `enabled` with NextElapseUSecMonotonic=infinity. Restarting the timer alone -# does not supply that reference; running the service does. -# -# (Diagnosing this is easy to get wrong: NextElapseUSecRealtime is always empty -# for a monotonic timer, so it looks broken even when it is fine. Read -# NextElapseUSecMonotonic, or just use `systemctl list-timers`.) -- name: Run the Fulcrum health check once to arm the timer - command: systemctl start fulcrum-healthcheck.service - changed_when: false diff --git a/ansible/roles/fulcrum/tasks/install.yml b/ansible/roles/fulcrum/tasks/install.yml deleted file mode 100644 index 1216a96..0000000 --- a/ansible/roles/fulcrum/tasks/install.yml +++ /dev/null @@ -1,146 +0,0 @@ ---- -- name: Calculate db_mem as a share of system RAM - set_fact: - fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" - when: fulcrum_db_mem_mb_override | string | length == 0 - -- name: Use the explicit db_mem override - set_fact: - fulcrum_db_mem_mb: "{{ fulcrum_db_mem_mb_override }}" - when: fulcrum_db_mem_mb_override | string | length > 0 - changed_when: false - -- name: Display calculated db_mem value - debug: - msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" - -- name: Display Fulcrum version to install - debug: - msg: "Installing Fulcrum version {{ fulcrum_version }}" - -- name: Install required packages - apt: - name: - - curl - - wget - - openssl - state: present - update_cache: yes - -- name: Create fulcrum group - group: - name: "{{ fulcrum_group }}" - system: yes - state: present - -- name: Create fulcrum user - user: - name: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - system: yes - shell: /usr/sbin/nologin - home: /home/{{ fulcrum_user }} - create_home: yes - state: present - -- name: Create Fulcrum database directory (heavy data on special mount) - file: - path: "{{ fulcrum_db_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - -- name: Create Fulcrum config directory - file: - path: "{{ fulcrum_config_dir }}" - state: directory - owner: root - group: "{{ fulcrum_group }}" - mode: '0755' - -- name: Create Fulcrum lib directory (for banner and other data files) - file: - path: "{{ fulcrum_lib_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - -# =========================================== -# SSL Certificate Generation -# =========================================== -- name: Check if SSL certificate already exists - stat: - path: "{{ fulcrum_ssl_cert_path }}" - register: fulcrum_ssl_cert_exists - when: fulcrum_ssl_enabled | default(false) - -- name: Generate self-signed SSL certificate for Fulcrum - command: > - openssl req -x509 -newkey rsa:4096 - -keyout {{ fulcrum_ssl_key_path }} - -out {{ fulcrum_ssl_cert_path }} - -sha256 -days {{ fulcrum_ssl_cert_days }} - -nodes - -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" - args: - creates: "{{ fulcrum_ssl_cert_path }}" - when: fulcrum_ssl_enabled | default(false) - notify: Restart fulcrum - -- name: Set SSL certificate permissions - file: - path: "{{ fulcrum_ssl_cert_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) - -- name: Set SSL key permissions - file: - path: "{{ fulcrum_ssl_key_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0600' - when: fulcrum_ssl_enabled | default(false) - -- name: Check if Fulcrum binary already exists - stat: - path: "{{ fulcrum_binary_path }}" - register: fulcrum_binary_exists - changed_when: false - -- name: Download Fulcrum binary tarball - get_url: - url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - mode: '0644' - when: not fulcrum_binary_exists.stat.exists - -- name: Extract Fulcrum binary - unarchive: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp" - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - -- name: Install Fulcrum binary - copy: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" - dest: "{{ fulcrum_binary_path }}" - owner: root - group: root - mode: '0755' - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - -- name: Verify Fulcrum binary installation - command: "{{ fulcrum_binary_path }} --version" - register: fulcrum_version_check - changed_when: false - -- name: Display Fulcrum version - debug: - msg: "{{ fulcrum_version_check.stdout_lines }}" - diff --git a/ansible/roles/fulcrum/tasks/main.yml b/ansible/roles/fulcrum/tasks/main.yml deleted file mode 100644 index bc7ff05..0000000 --- a/ansible/roles/fulcrum/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/fulcrum/tasks/service.yml b/ansible/roles/fulcrum/tasks/service.yml deleted file mode 100644 index 716b44d..0000000 --- a/ansible/roles/fulcrum/tasks/service.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -- name: Create Fulcrum banner file - ansible.builtin.template: - src: banner.txt.j2 - dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - -- name: Create Fulcrum configuration file - ansible.builtin.template: - src: fulcrum.conf.j2 - dest: "{{ fulcrum_config_dir }}/fulcrum.conf" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0640' - notify: Restart fulcrum - -- name: Create systemd service file for Fulcrum - ansible.builtin.template: - src: fulcrum.service.j2 - dest: /etc/systemd/system/fulcrum.service - owner: root - group: root - mode: '0644' - notify: Restart fulcrum - -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start Fulcrum service - systemd: - name: fulcrum - enabled: yes - state: started - -- name: Wait for Fulcrum to start - wait_for: - port: "{{ fulcrum_tcp_port }}" - host: "{{ fulcrum_tcp_bind }}" - delay: 5 - timeout: 30 - ignore_errors: yes - -- name: Check Fulcrum service status - systemd: - name: fulcrum - register: fulcrum_service_status - changed_when: false - -- name: Display Fulcrum service status - debug: - msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" diff --git a/ansible/roles/fulcrum/templates/banner.txt.j2 b/ansible/roles/fulcrum/templates/banner.txt.j2 deleted file mode 100644 index 1ce4ced..0000000 --- a/ansible/roles/fulcrum/templates/banner.txt.j2 +++ /dev/null @@ -1,3 +0,0 @@ -counterinfra - -PER ASPERA AD ASTRA diff --git a/ansible/roles/fulcrum/templates/fulcrum.conf.j2 b/ansible/roles/fulcrum/templates/fulcrum.conf.j2 deleted file mode 100644 index 69f072c..0000000 --- a/ansible/roles/fulcrum/templates/fulcrum.conf.j2 +++ /dev/null @@ -1,29 +0,0 @@ -# Fulcrum Configuration -# Generated by Ansible - -# Bitcoin Core/Knots RPC settings -bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} -rpcuser = {{ bitcoin_rpc_user }} -rpcpassword = {{ bitcoin_rpc_password }} - -# Fulcrum server general settings -datadir = {{ fulcrum_db_dir }} -tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} -peering = {{ 'true' if fulcrum_peering else 'false' }} -zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} - -# SSL/TLS Configuration -{% if fulcrum_ssl_enabled | default(false) %} -ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} -cert = {{ fulcrum_ssl_cert_path }} -key = {{ fulcrum_ssl_key_path }} -{% endif %} - -# Anonymize client IP addresses and TxIDs in logs -anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} - -# Max RocksDB Memory in MiB -db_mem = {{ fulcrum_db_mem_mb }}.0 - -# Banner -banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt diff --git a/ansible/roles/fulcrum/templates/fulcrum.service.j2 b/ansible/roles/fulcrum/templates/fulcrum.service.j2 deleted file mode 100644 index 02f4aa5..0000000 --- a/ansible/roles/fulcrum/templates/fulcrum.service.j2 +++ /dev/null @@ -1,24 +0,0 @@ -# MiniBolt: systemd unit for Fulcrum -# /etc/systemd/system/fulcrum.service - -[Unit] -Description=Fulcrum -After=network.target - -StartLimitBurst=2 -StartLimitIntervalSec=20 - -[Service] -ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf - -User={{ fulcrum_user }} -Group={{ fulcrum_group }} - -# Process management -#################### -Type=simple -KillSignal=SIGINT -TimeoutStopSec=300 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.service.j2 b/ansible/roles/fulcrum/templates/healthcheck.service.j2 deleted file mode 100644 index b519808..0000000 --- a/ansible/roles/fulcrum/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Fulcrum Health Check -After=network.target fulcrum.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/fulcrum/templates/healthcheck.sh.j2 b/ansible/roles/fulcrum/templates/healthcheck.sh.j2 deleted file mode 100644 index 4449fed..0000000 --- a/ansible/roles/fulcrum/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Fulcrum health check — managed by Ansible (roles/fulcrum) -# -# Checks that Fulcrum's Electrum TCP port is accepting connections, and records -# the answer in the exit code, which systemd keeps: -# systemctl is-failed fulcrum-healthcheck.service -# That is a complete answer with no monitoring system involved. Reporting -# elsewhere is optional and generic — set healthcheck_push_url. - -FULCRUM_HOST="{{ fulcrum_tcp_bind }}" -FULCRUM_PORT={{ fulcrum_tcp_port }} -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -check_fulcrum() { - timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null -} - -report() { - local status=$1 msg=$2 - # No push URL is normal, not an error: the exit code below is still a - # complete answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true -} - -if check_fulcrum; then - report "up" "OK" - exit 0 -else - echo "Fulcrum TCP port ${FULCRUM_PORT} not responding" - report "down" "Fulcrum TCP port not responding" - exit 1 -fi diff --git a/ansible/roles/fulcrum/templates/healthcheck.timer.j2 b/ansible/roles/fulcrum/templates/healthcheck.timer.j2 deleted file mode 100644 index df16ff3..0000000 --- a/ansible/roles/fulcrum/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Fulcrum Health Check Timer -# NOTE: this deliberately does NOT carry `Requires=fulcrum.service`, which the -# hand-written unit had. Requires on a timer means the timer is stopped when the -# required unit stops — i.e. "if the thing I am watching goes down, stop -# watching it", which is backwards for a health check and leaves nothing to -# re-arm the timer when the service returns. The live timer had been `active` -# and `enabled` with NextElapseUSecMonotonic=infinity and a last trigger of -# 2026-02-17: seven months with no health check and no outward sign of it. - -[Timer] -OnBootSec=1min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/gatus/README.md b/ansible/roles/gatus/README.md deleted file mode 100644 index 129057c..0000000 --- a/ansible/roles/gatus/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# gatus - -Deploys [Gatus](https://github.com/TwiN/gatus) — health checks, a status page, -and alerting — **as the upstream container image**, on the -`observability` group. - -## Why the container - -Upstream publishes **no binary release assets**. The image is the only artefact -they ship and therefore the only one they test, so it is what `docker run` in -their README gets you, and it is what this role deploys. - -Building from source is entirely possible — their Dockerfile is a bare -`CGO_ENABLED=0 go build`, the Vue dashboard is compiled in via `//go:embed -static` in `web/static.go`, and the sqlite driver is pure-Go -`modernc.org/sqlite` so nothing needs linking. This role did that at first. The -reasons it doesn't now: - -- It produces a binary upstream never ran. -- It means compiling the AWS SDK, gRPC and the Google API libraries on the - smallest box in the estate. On this VPS that load was heavy enough that - unrelated Ansible tasks timed out while it ran. - -The cost of the container is a daemon on the machine whose job is to notice -when everything else breaks. That is a real trade, made deliberately. - -## Pinned by digest, not by tag - -```yaml -gatus_image_digest: "sha256:c5f210d0…" -gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" -``` - -A tag is mutable — `v5.36.0` can be repushed — so pinning the tag alone is a -weaker promise than it looks. The digest is a content address: if it resolves, -it is byte-for-byte the image reviewed here, and `docker compose pull` either -fetches exactly that or fails. `gatus_version` is kept alongside it purely so a -human can read which release it is; **the two must be updated together.** - -## What `FROM scratch` means for running it - -The image has no `/etc/passwd`, so there is no user to drop to by name and the -default is root. The compose file runs it by numeric id (`10001:10001`) and the -data directory on the host is owned to match. Everything else is locked down to -approximate what the systemd unit used to do natively: - -| systemd | compose | -|---|---| -| `ProtectSystem=strict` | `read_only: true` | -| `NoNewPrivileges=true` | `security_opt: [no-new-privileges:true]` | -| `CapabilityBoundingSet=` | `cap_drop: [ALL]` | -| `AmbientCapabilities=CAP_NET_RAW` | `cap_add: [NET_RAW]` (for `icmp://`) | - -## Configuration is a directory, not a file - -`GATUS_CONFIG_PATH` points at `/opt/gatus/config`, and Gatus merges every `*.yaml` -underneath it — maps deep-merge, lists append. This role owns exactly one file: - -``` -/opt/gatus/config/00-base.yaml web, storage, ui, alerting, security (this role) -/opt/gatus/config/endpoints/*.yaml one file per service (gatus_endpoint) -``` - -**A primitive defined in two files is ambiguous and upstream refuses it.** So -anything that is not a list belongs in `00-base.yaml` and nowhere else. Endpoints -are lists, so each service's file appends cleanly — the same shape as -`caddy_site`, where each service contributes its own vhost. - -## Pull and push - -Gatus polls. For anything with a reachable HTTP or TCP surface that is the -better check, because it tests the path a user actually takes. The monitoring -host joins the headscale mesh via `infra/920`, so internal boxes are reachable -by MagicDNS name and can be polled directly rather than having to report in. - -For state with no pollable surface — ZFS pool health, UPS mains status, disk -usage, backup freshness — Gatus has **external endpoints**, a push API: - -``` -POST /api/v1/endpoints/{group}_{name}/external?success=true&error=&duration= -Authorization: Bearer -``` - -with `heartbeat.interval` to alert when nothing reports in. That is the same -shape as the generic `healthcheck_push_url` already wired into every service -role, so those scripts need a URL, a POST, and an auth header — not a rewrite. - -## Variables - -See `defaults/main.yml`. The ones that matter: - -| Variable | Default | Note | -|---|---|---| -| `gatus_version` / `gatus_image_digest` | `v5.36.0` / `sha256:c5f210d0…` | must move together | -| `gatus_bind_address` | `127.0.0.1` | **never bind publicly** — the push API shares this listener | -| `gatus_storage_type` | `sqlite` | `memory` loses all history on restart | -| `gatus_alerting` | `{}` | pass-through; any provider Gatus supports | -| `gatus_allow_icmp` | `true` | adds back `NET_RAW` for `icmp://` checks | - -`gatus_alerting` empty is valid and is the current state: every condition is -still evaluated and recorded, there is just nowhere to shout yet. - -## Verifying - -```bash -docker ps --filter name=gatus -docker logs gatus --tail 50 -curl -s localhost:8080/health -ls /opt/gatus/config/endpoints/ -``` diff --git a/ansible/roles/gatus/defaults/main.yml b/ansible/roles/gatus/defaults/main.yml deleted file mode 100644 index 6f6ec09..0000000 --- a/ansible/roles/gatus/defaults/main.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- -# Gatus, deployed the way upstream distributes it: the container image. -# -# Upstream publishes NO binary release assets - the image is the only artefact -# they ship, and therefore the only artefact they test. Building from source is -# possible (`go build` alone is enough; the Vue dashboard is compiled in via -# `//go:embed static`, and CGO_ENABLED=0 works because the sqlite driver is -# pure-Go modernc.org/sqlite) but it produces a binary upstream never ran, and -# it means a full compile of the AWS SDK and gRPC on the smallest box in the -# estate. - -# ── Image ──────────────────────────────────────────────────────────────────── -gatus_version: "v5.36.0" -# Pinned by DIGEST, not by tag. A tag is mutable - `v5.36.0` can be repushed - -# so pinning the tag alone is a weaker promise than it looks. The digest is the -# content address: if it resolves, it is byte-for-byte the image reviewed here. -# Both must be updated together; the tag is kept only so humans can read it. -gatus_image_digest: "sha256:c5f210d095fa78e6efaa20ffeb14803f2ba4f10615e16a6d12087697149617f0" -gatus_image: "ghcr.io/twin/gatus@{{ gatus_image_digest }}" - -# ── Paths (host side) ──────────────────────────────────────────────────────── -gatus_dir: /opt/gatus -gatus_config_dir: "{{ gatus_dir }}/config" -gatus_data_dir: "{{ gatus_dir }}/data" - -# Gatus merges every *.yaml under GATUS_CONFIG_PATH and its subdirectories: -# maps deep-merge, lists append. That is why this role ships a config DIRECTORY -# rather than one file - each service contributes its own endpoint file, the -# same way each service contributes a vhost through `caddy_site`. -# -# Primitives must be defined exactly once across all files or the merge is -# ambiguous, so everything that is not a list lives in the base file and -# nowhere else. -gatus_base_config_file: "00-base.yaml" -gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" - -# ── Identity ───────────────────────────────────────────────────────────────── -# The image is FROM scratch, so it has no /etc/passwd and no user to drop to by -# name. Run it by numeric uid/gid instead, and own the data volume to match. -gatus_uid: 10001 -gatus_gid: 10001 - -# ── Web ────────────────────────────────────────────────────────────────────── -gatus_port: 8080 -# HOST-side address the container's port is published on. Gatus itself always -# binds 0.0.0.0 inside the container - see the note in config.yaml.j2. Never -# publish this on 0.0.0.0: the external-endpoint push API shares the dashboard's -# listener, and Caddy is what should be in front of both. -gatus_bind_address: "127.0.0.1" -# Pass-through, the same shape as gatus_alerting: whatever is set here is -# rendered verbatim under `ui:`, so every option upstream supports is reachable -# without adding a variable per key. See config/ui/ui.go for the full set - -# title, description, header, dashboard-heading/subheading, logo, link, favicon, -# buttons, custom-css, dark-mode, default-sort-by, default-filter-by. -gatus_ui: - title: "Status" - header: "Status" - # Group by default. The dashboard's own groupByGroup toggle starts OFF and - # remembers per browser in localStorage, so without this the ten groups render - # as one flat list for anyone who has not clicked it. - default-sort-by: group - -# ── Storage ────────────────────────────────────────────────────────────────── -# sqlite, not memory: history has to survive a restart, or the dashboard lies -# about uptime after every deploy. Path is INSIDE the container. -gatus_storage_type: sqlite -gatus_storage_path: "/data/gatus.db" -gatus_storage_caching: true - -# Per-endpoint row caps. Gatus bounds the database by COUNT, not by time, and -# trims inline on insert (storage/store/sql/sql.go, InsertEndpointResult) - so -# there is no retention job to write and no way for this to fill a disk. -# -# History depth is therefore a function of check frequency, not of days: -# 900 results is ~3 days of a 5-minute liveness check, and ~2.5 years of a daily -# disk check. Upstream's default is 100, which would have been 8 hours of -# liveness - not enough to still see a weekend incident on Monday. -# -# Note the uptime table is separate and its 30-day retention is hard-coded -# upstream (uptimeRetention), so uptime percentages top out at 30 days whatever -# this is set to. -gatus_storage_max_results: 900 -gatus_storage_max_events: 50 - -# ── Alerting ───────────────────────────────────────────────────────────────── -# Pass-through: rendered verbatim under `alerting:`, so any provider Gatus -# supports works without touching this role. Empty means "check and record, -# alert nowhere" - valid, and the default until a provider is chosen. -gatus_alerting: {} -gatus_default_alerts: [] - -# ── Security ───────────────────────────────────────────────────────────────── -# gatus_basic_auth: {username: admin, password-bcrypt-base64: "..."} -gatus_basic_auth: {} - -gatus_maintenance: {} - -# Keep serving the previous config if a contributed endpoint file is malformed, -# instead of panicking. See the note in config.yaml.j2. -gatus_skip_invalid_config_update: true -gatus_log_level: INFO - -# ── Self-check ─────────────────────────────────────────────────────────────── -# Gatus panics on a config with no endpoints, so the role always ships one. -# Turning this off is only safe once another file in endpoints/ provides one. -gatus_self_check: true - -# ── ICMP ───────────────────────────────────────────────────────────────────── -# Gatus supports icmp:// endpoints. Raw ICMP needs CAP_NET_RAW, which the -# container would get free only if it ran as root; it does not. Set false if -# you never use icmp:// checks and want the capability dropped entirely. -gatus_allow_icmp: true - -# ── Shared network ─────────────────────────────────────────────────────────── -# Gatus runs in a container, so the HOST's loopback is not reachable from it. -# Anything Gatus must talk to locally - the Signal API that sends its alerts - -# has to be on a shared docker network and addressed by service name. -gatus_network: monitoring diff --git a/ansible/roles/gatus/handlers/main.yml b/ansible/roles/gatus/handlers/main.yml deleted file mode 100644 index 25ad865..0000000 --- a/ansible/roles/gatus/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Restart gatus - ansible.builtin.command: - cmd: docker compose up -d --force-recreate - chdir: "{{ gatus_dir }}" diff --git a/ansible/roles/gatus/tasks/configure.yml b/ansible/roles/gatus/tasks/configure.yml deleted file mode 100644 index ef5d7e7..0000000 --- a/ansible/roles/gatus/tasks/configure.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- name: Assert Docker is available - ansible.builtin.command: docker --version - register: gatus_docker_check - changed_when: false - failed_when: gatus_docker_check.rc != 0 - -# Created explicitly rather than by either compose file, so neither the gatus -# stack nor the signal-api stack has to be deployed before the other. -- name: Ensure the shared monitoring network exists - ansible.builtin.command: "docker network create {{ gatus_network }}" - register: gatus_net - changed_when: "'already exists' not in gatus_net.stderr" - failed_when: - - gatus_net.rc != 0 - - "'already exists' not in gatus_net.stderr" - -- name: Create the gatus directories - ansible.builtin.file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode }}" - loop: - - {path: "{{ gatus_dir }}", owner: root, group: root, mode: "0755"} - # Config is root-owned and world-unreadable: it holds external-endpoint - # tokens. The container mounts it read-only and reads it as gatus_uid, so - # that id needs group access - hence the group ownership below. - - {path: "{{ gatus_config_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} - - {path: "{{ gatus_endpoints_dir }}", owner: root, group: "{{ gatus_gid }}", mode: "0750"} - # Data is the one path the container writes to, so it must be owned by the - # numeric id the container runs as. `read_only: true` makes everything else - # in the container immutable. - - {path: "{{ gatus_data_dir }}", owner: "{{ gatus_uid }}", group: "{{ gatus_gid }}", mode: "0750"} - -- name: Write the base gatus configuration - ansible.builtin.template: - src: config.yaml.j2 - dest: "{{ gatus_config_dir }}/{{ gatus_base_config_file }}" - owner: root - group: "{{ gatus_gid }}" - mode: "0640" - notify: Restart gatus - -# Without this the container crash-loops on an empty config. See the template. -- name: Write the gatus self-check endpoint - ansible.builtin.template: - src: endpoint-self.yaml.j2 - dest: "{{ gatus_endpoints_dir }}/00-self.yaml" - owner: root - group: "{{ gatus_gid }}" - mode: "0640" - when: gatus_self_check | bool - notify: Restart gatus - -- name: Write the docker compose file - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ gatus_dir }}/docker-compose.yml" - owner: root - group: root - mode: "0644" - notify: Restart gatus diff --git a/ansible/roles/gatus/tasks/main.yml b/ansible/roles/gatus/tasks/main.yml deleted file mode 100644 index e1b67bf..0000000 --- a/ansible/roles/gatus/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# import, never include: dynamic includes are opaque to --list-tasks, which is -# the primary verification tool in this repo. -- ansible.builtin.import_tasks: configure.yml -- ansible.builtin.import_tasks: service.yml diff --git a/ansible/roles/gatus/tasks/service.yml b/ansible/roles/gatus/tasks/service.yml deleted file mode 100644 index bfcca3b..0000000 --- a/ansible/roles/gatus/tasks/service.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Pulling by digest means this either fetches exactly the reviewed image or -# fails. There is no "latest wins" path. -- name: Pull the pinned gatus image - ansible.builtin.command: - cmd: docker compose pull - chdir: "{{ gatus_dir }}" - register: gatus_pull - changed_when: "'Downloaded newer image' in gatus_pull.stderr or 'Pull complete' in gatus_pull.stderr" - -- name: Start gatus - ansible.builtin.command: - cmd: docker compose up -d --remove-orphans - chdir: "{{ gatus_dir }}" - register: gatus_up - changed_when: "'Started' in gatus_up.stderr or 'Created' in gatus_up.stderr or 'Recreated' in gatus_up.stderr" - -- name: Flush handlers so a config change is live before it is verified - ansible.builtin.meta: flush_handlers - -- name: Wait for gatus to answer - ansible.builtin.uri: - url: "http://{{ gatus_bind_address }}:{{ gatus_port }}/health" - status_code: [200, 404] - register: gatus_health - until: gatus_health.status in [200, 404] - retries: 12 - delay: 5 - -- name: Assert gatus is running - ansible.builtin.assert: - that: - - gatus_health.status in [200, 404] - fail_msg: "gatus did not come up on {{ gatus_bind_address }}:{{ gatus_port }} - check `docker logs gatus`" - success_msg: "gatus is answering on {{ gatus_bind_address }}:{{ gatus_port }}" diff --git a/ansible/roles/gatus/templates/config.yaml.j2 b/ansible/roles/gatus/templates/config.yaml.j2 deleted file mode 100644 index faf61fb..0000000 --- a/ansible/roles/gatus/templates/config.yaml.j2 +++ /dev/null @@ -1,64 +0,0 @@ -# {{ gatus_base_config_file }} — managed by Ansible (roles/gatus) -# -# BASE CONFIGURATION ONLY. -# -# Gatus merges every *.yaml under GATUS_CONFIG_PATH: maps are deep-merged and -# lists are appended, but a primitive defined in two files is ambiguous and -# upstream refuses it. So `web`, `storage`, `ui`, `alerting` and `security` are -# set here and MUST NOT appear in any other file in this directory. -# -# Endpoints are lists, so they append cleanly. Each service drops its own file -# into endpoints/ via the gatus_endpoint role - the same shape as caddy_site. - -web: - # 0.0.0.0 is the CONTAINER's interface, not the host's. This must not be - # 127.0.0.1: that is the container's own loopback, which docker-proxy cannot - # reach, and gatus comes up healthy while the published port refuses every - # connection. - # - # The isolation comes from the port mapping in docker-compose.yml, which - # publishes to {{ gatus_bind_address }} on the host. Caddy fronts that, and it - # matters because the external-endpoint push API shares this listener with the - # dashboard. - address: 0.0.0.0 - port: {{ gatus_port }} - -# Gatus reloads when its config changes. If the NEW config fails to parse it -# calls panic() - unless this is set, in which case it logs the error and keeps -# running on the old config. Endpoint files are contributed by other playbooks, -# so one malformed file would otherwise take the monitor down, which is the -# worst possible time to lose it. -skip-invalid-config-update: {{ gatus_skip_invalid_config_update | bool | lower }} - -storage: - type: {{ gatus_storage_type }} -{% if gatus_storage_type != 'memory' %} - path: {{ gatus_storage_path }} -{% endif %} - caching: {{ gatus_storage_caching | bool | lower }} - maximum-number-of-results: {{ gatus_storage_max_results }} - maximum-number-of-events: {{ gatus_storage_max_events }} - -ui: -{{ gatus_ui | to_nice_yaml(indent=2) | indent(2, true) }} -{% if gatus_alerting %} - -alerting: -{{ gatus_alerting | to_nice_yaml(indent=2) | indent(2, true) }} -{% else %} - -# No alerting provider is configured yet. Gatus still evaluates every condition -# and records every result; it simply has nowhere to shout. Setting -# `gatus_alerting` is the single change needed to wire one up. -{% endif %} -{% if gatus_basic_auth %} - -security: - basic: -{{ gatus_basic_auth | to_nice_yaml(indent=2) | indent(4, true) }} -{% endif %} -{% if gatus_maintenance %} - -maintenance: -{{ gatus_maintenance | to_nice_yaml(indent=2) | indent(2, true) }} -{% endif %} diff --git a/ansible/roles/gatus/templates/docker-compose.yml.j2 b/ansible/roles/gatus/templates/docker-compose.yml.j2 deleted file mode 100644 index 512c9c5..0000000 --- a/ansible/roles/gatus/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,53 +0,0 @@ -# Managed by Ansible (roles/gatus) -services: - gatus: - image: {{ gatus_image }} - container_name: gatus - restart: unless-stopped - - # The image is FROM scratch: no /etc/passwd, so there is no user to drop to - # by name and the default is root. Run it by numeric id instead. - user: "{{ gatus_uid }}:{{ gatus_gid }}" - - ports: - # Loopback on purpose. Caddy fronts this, and the external-endpoint push - # API is served from the same listener as the dashboard - publishing - # 0.0.0.0 would put both straight on the public internet. - - "{{ gatus_bind_address }}:{{ gatus_port }}:{{ gatus_port }}" - - environment: - GATUS_CONFIG_PATH: /config - GATUS_LOG_LEVEL: "{{ gatus_log_level }}" - - volumes: - - {{ gatus_config_dir }}:/config:ro - - {{ gatus_data_dir }}:/data - - # Hardening. The systemd unit this replaced got most of it from - # ProtectSystem/NoNewPrivileges/etc; these are the container equivalents. - read_only: true - security_opt: - - no-new-privileges:true - cap_drop: - - ALL -{% if gatus_allow_icmp %} - cap_add: - # icmp:// endpoints need raw sockets. Dropped above with ALL, added back - # explicitly so the grant is visible rather than inherited from root. - - NET_RAW -{% endif %} - - networks: - # Shared with signal-api, so alerts can be delivered by service name. - # 127.0.0.1 inside this container is the container, not the host. - - {{ gatus_network }} - - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - -networks: - {{ gatus_network }}: - external: true diff --git a/ansible/roles/gatus/templates/endpoint-self.yaml.j2 b/ansible/roles/gatus/templates/endpoint-self.yaml.j2 deleted file mode 100644 index 197ae56..0000000 --- a/ansible/roles/gatus/templates/endpoint-self.yaml.j2 +++ /dev/null @@ -1,25 +0,0 @@ -# 00-self.yaml — managed by Ansible (roles/gatus) -# -# Gatus refuses to start with no endpoints at all: -# panic: error parsing config: configuration should contain at least one -# endpoint or suite -# -# So the role ships one. Checking its own listener is not as circular as it -# looks: it proves the config directory parsed, the container is serving, and -# the storage backend accepted a write. If this row is missing from the -# dashboard, the dashboard is not telling you the truth about anything else. -# -# It is also what keeps `gatus` deployable before any service has contributed -# an endpoint file of its own. - -endpoints: - - name: gatus - group: infrastructure - url: "http://localhost:{{ gatus_port }}/health" - interval: 60s - conditions: - - "[STATUS] == 200" -{% if gatus_default_alerts %} - alerts: -{{ gatus_default_alerts | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} diff --git a/ansible/roles/gatus_endpoint/defaults/main.yml b/ansible/roles/gatus_endpoint/defaults/main.yml deleted file mode 100644 index 90a2c19..0000000 --- a/ansible/roles/gatus_endpoint/defaults/main.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -# One invocation writes ONE file into Gatus's endpoints directory. Gatus merges -# every *.yaml under GATUS_CONFIG_PATH and appends lists, so each caller owns -# its own file and they compose without coordinating - the same shape as -# caddy_site, where each service contributes its own vhost. - -# Filename stem: .yaml -gatus_endpoint_name: "" - -# PULLED endpoints - Gatus makes the request and evaluates conditions. -# - {name, group, url, interval, conditions: [...], alerts: [...]} -# -# A DNS check adds `dns: {query-type, query-name}` - and note that for those, -# `url` is the RESOLVER to ask, not the name being looked up. -# A domain-expiry check is just `url: ` with a [DOMAIN_EXPIRATION] -# condition; it uses WHOIS/RDAP and needs no scheme. -gatus_endpoint_pulled: [] - -# EXTERNAL endpoints - the host pushes its own result. Gatus never reaches out, -# which is what makes this work for machines behind NAT and for state that has -# no pollable surface at all (disk usage, ZFS health, UPS mains). -# -# - {name, group, token, heartbeat, alerts: [...]} -# -# `heartbeat` is the important one: if nothing reports within that window Gatus -# alerts. That is what makes a push check detect its own failure - a dead timer -# looks exactly like a dead host, which is the correct reading. -gatus_endpoint_external: [] - -# Where the files live. Matches roles/gatus. -gatus_config_dir: /opt/gatus/config -gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" -gatus_gid: 10001 - -# Alerts attached to every endpoint in this file that does not specify its own. -# -# Gatus's provider-level `default-alert` only supplies DEFAULTS - an endpoint -# still has to opt in with `alerts: - type: signal` or it alerts on nothing at -# all. With ~90 endpoints that cannot be written by hand, so it is applied here. -# -# failure-threshold is set by the CALLER, because the right value depends on the -# check's cadence and there is no single correct default. See the note in -# infra/400_host_monitoring.yml. -gatus_endpoint_default_alerts: [] - -# UI options applied to every PULLED endpoint that does not set its own. -# -# Only pulled endpoints can carry this - an external (push) endpoint has no `ui` -# field at all, because it has no conditions to display. -# -# Note that every hide-* option already defaults to false upstream, so there is -# nothing to un-hide. The one setting that genuinely shows MORE is -# resolve-successful-conditions: by default a failing check displays the real -# value - "[STATUS] (502) == 200" - while a passing one drops it and shows only -# "[STATUS] == 200". Turning it on resolves both, so a healthy DNS check shows -# the IP it actually resolved rather than just the assertion. -gatus_endpoint_default_ui: - resolve-successful-conditions: true diff --git a/ansible/roles/gatus_endpoint/tasks/main.yml b/ansible/roles/gatus_endpoint/tasks/main.yml deleted file mode 100644 index 40f3448..0000000 --- a/ansible/roles/gatus_endpoint/tasks/main.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Assert gatus_endpoint parameters are sane - ansible.builtin.assert: - that: - - gatus_endpoint_name | length > 0 - - gatus_endpoint_pulled | length > 0 or gatus_endpoint_external | length > 0 - fail_msg: >- - gatus_endpoint needs a name and at least one of gatus_endpoint_pulled or - gatus_endpoint_external. An empty file would contribute nothing, and a - config with no endpoints at all makes Gatus refuse to start. - -- name: "Write Gatus endpoints '{{ gatus_endpoint_name }}'" - ansible.builtin.template: - src: endpoints.yaml.j2 - dest: "{{ gatus_endpoints_dir }}/{{ gatus_endpoint_name }}.yaml" - owner: root - group: "{{ gatus_gid }}" - mode: "0640" - # No handler. Gatus polls its own config every 30s - # (main.listenToConfigurationFileChanges) and reloads itself, so writing the - # file IS the deploy. A handler here would also fail whenever this role is - # used without roles/gatus loaded. diff --git a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 deleted file mode 100644 index 680691e..0000000 --- a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 +++ /dev/null @@ -1,60 +0,0 @@ -# {{ gatus_endpoint_name }}.yaml — managed by Ansible (roles/gatus_endpoint) -# -# Contributed by a playbook, not hand-edited. Gatus appends the lists in every -# *.yaml under its config directory, so this file adds to whatever else is -# registered without knowing about it. -{% if gatus_endpoint_external %} - -external-endpoints: -{% for e in gatus_endpoint_external %} - - name: {{ e.name }} - group: {{ e.group }} - token: "{{ e.token }}" -{% if e.heartbeat is defined %} - heartbeat: - interval: {{ e.heartbeat }} -{% endif %} -{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} -{% if _alerts %} - alerts: -{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} -{# external endpoints have no `ui` field upstream - they carry no conditions #} -{% endfor %} -{% endif %} -{% if gatus_endpoint_pulled %} - -endpoints: -{% for e in gatus_endpoint_pulled %} - - name: {{ e.name }} - group: {{ e.group }} - url: "{{ e.url }}" - interval: {{ e.interval | default('60s') }} -{% if e.dns is defined %} - # A DNS endpoint: `url` is the RESOLVER to ask, not the thing being asked - # about. The name being queried lives in query-name, and [BODY] holds the - # resolved record for the conditions below. - dns: - query-type: {{ e.dns['query-type'] }} - query-name: {{ e.dns['query-name'] }} -{% endif %} -{% if e.client is defined %} - client: -{{ e.client | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} - conditions: -{% for c in e.conditions %} - - "{{ c }}" -{% endfor %} -{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} -{% if _alerts %} - alerts: -{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} -{% set _ui = e.ui | default(gatus_endpoint_default_ui) %} -{% if _ui %} - ui: -{{ _ui | to_nice_yaml(indent=2) | indent(6, true) }} -{% endif %} -{% endfor %} -{% endif %} diff --git a/ansible/roles/healthcheck/defaults/main.yml b/ansible/roles/healthcheck/defaults/main.yml deleted file mode 100644 index 51534c2..0000000 --- a/ansible/roles/healthcheck/defaults/main.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -# One health check: a script, a systemd service, a timer, and an optional push. -# -# The exit code is the answer and systemd keeps it: -# systemctl is-failed -healthcheck.service -# Reporting anywhere else is optional and generic. Point healthcheck_push_url at -# Gatus, or at whatever replaces it, or at nothing. - -healthcheck_name: "" # e.g. disk-usage -> disk-usage-healthcheck -healthcheck_description: "" - -# The check itself. Pick ONE: -# healthcheck_check: a template under templates/checks/ (without .sh.j2) -# healthcheck_command: a shell one-liner that exits 0 for healthy -healthcheck_check: "" -healthcheck_command: "" - -# systemd timer. OnUnitActiveSec unless healthcheck_on_calendar is set. -healthcheck_interval: "5min" -healthcheck_on_calendar: "" -healthcheck_boot_delay: "2min" - -# ── Reporting ──────────────────────────────────────────────────────────────── -# Gatus external endpoints: -# POST {url}?success=true|false&error=... -# Authorization: Bearer {token} -# Empty url = check and log only, which is a valid state and not an error. -healthcheck_push_url: "" -healthcheck_push_token: "" - -# Some checks report MORE THAN ONE result - a host with four systemd services -# needs four endpoints, or a single red light cannot tell you which one died. -# Those set healthcheck_push_base to the endpoints COLLECTION and the check body -# appends each key itself, the same way check-backups.sh reports per source. -healthcheck_push_base: "" - -# Units for the systemd-units check. Each becomes its own Gatus endpoint. -healthcheck_units: [] -# Prefix for the per-unit endpoint keys, e.g. "services_vipy" -> services_vipy-caddy. -healthcheck_units_key_prefix: "" - -healthcheck_script_dir: /usr/local/bin -healthcheck_log_dir: /var/log/healthchecks - -# Per-check knobs, consumed by the templates under checks/ -healthcheck_disk_threshold: 85 # percent -healthcheck_cpu_temp_threshold: 80 # celsius -healthcheck_zfs_pool: "" -# ZFS degrades badly once a pool passes roughly 80% - allocation gets slow and -# fragmentation becomes hard to undo, and unlike a normal filesystem you cannot -# simply delete your way back to good performance. So this alarms well before -# the pool is actually out of space. -healthcheck_zfs_capacity_threshold: 80 -healthcheck_ups_name: "" diff --git a/ansible/roles/healthcheck/tasks/main.yml b/ansible/roles/healthcheck/tasks/main.yml deleted file mode 100644 index f37a0d5..0000000 --- a/ansible/roles/healthcheck/tasks/main.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -- name: "Assert healthcheck '{{ healthcheck_name }}' is fully specified" - ansible.builtin.assert: - that: - - healthcheck_name | length > 0 - - healthcheck_description | length > 0 - - (healthcheck_check | length > 0) != (healthcheck_command | length > 0) - - not (healthcheck_push_url | length > 0) or (healthcheck_push_token | length > 0) - fail_msg: >- - healthcheck needs a name, a description, exactly one of healthcheck_check - or healthcheck_command, and a token whenever a push URL is set. A push URL - with no token would report to Gatus and be rejected 401 on every run. - -# Deduplicated across the whole play run. This role is included once PER CHECK, -# and a host with several checks was otherwise running apt several times to -# install a curl that was already there - 29 apt transactions estate-wide, and -# the slowest thing in the deploy by a wide margin. The fact below remembers -# what has already been ensured on this host. -- name: Install healthcheck dependencies - ansible.builtin.package: - name: "{{ healthcheck_wanted_packages }}" - state: present - vars: - healthcheck_wanted_packages: >- - {{ (healthcheck_packages | default(['curl'])) - | difference(healthcheck_installed_packages | default([])) }} - when: healthcheck_wanted_packages | length > 0 - -- name: Remember which dependencies this host already has - ansible.builtin.set_fact: - healthcheck_installed_packages: >- - {{ (healthcheck_installed_packages | default([])) - | union(healthcheck_packages | default(['curl'])) }} - -- name: Create the healthcheck log directory - ansible.builtin.file: - path: "{{ healthcheck_log_dir }}" - state: directory - owner: root - group: root - mode: "0750" - -- name: "Install the {{ healthcheck_name }} check script" - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" - owner: root - group: root - mode: "0755" - -# The token is in this unit file, so it must not be world-readable. -- name: "Install the {{ healthcheck_name }} systemd service" - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.service" - owner: root - group: root - mode: "0600" - -- name: "Install the {{ healthcheck_name }} systemd timer" - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.timer" - owner: root - group: root - mode: "0644" - -# `restarted`, not `started`: started is a no-op on an already-active timer, so -# a changed interval or a stuck timer would never be picked up. -- name: "Enable and start the {{ healthcheck_name }} timer" - ansible.builtin.systemd: - name: "{{ healthcheck_name }}-healthcheck.timer" - enabled: yes - state: restarted - daemon_reload: yes - -- name: "Run the {{ healthcheck_name }} check once now" - ansible.builtin.command: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" - environment: - HEALTHCHECK_PUSH_URL: "{{ healthcheck_push_url }}" - HEALTHCHECK_PUSH_TOKEN: "{{ healthcheck_push_token }}" - register: healthcheck_first_run - changed_when: false - failed_when: false - -- name: "Report the first {{ healthcheck_name }} result" - ansible.builtin.debug: - msg: >- - {{ healthcheck_name }}: {{ 'HEALTHY' if healthcheck_first_run.rc == 0 - else 'UNHEALTHY (rc=' ~ healthcheck_first_run.rc ~ ')' }} diff --git a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 deleted file mode 100644 index cf57509..0000000 --- a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ - # Hottest core across every thermal zone and hwmon sensor lm-sensors knows - # about. Reading the hottest rather than an average is deliberate: one core - # throttling is a real problem that an average hides. - local threshold={{ healthcheck_cpu_temp_threshold }} - local hottest=0 label="" - - while read -r t; do - [ -z "$t" ] && continue - t=${t%.*} - if [ "$t" -gt "$hottest" ]; then hottest=$t; fi - done < <(sensors -u 2>/dev/null | awk '/_input:/ && /temp/ {print $2}') - - # Fall back to the kernel thermal zones if lm-sensors reports nothing. - if [ "$hottest" -eq 0 ]; then - for z in /sys/class/thermal/thermal_zone*/temp; do - [ -r "$z" ] || continue - local milli; milli=$(cat "$z" 2>/dev/null) || continue - local c=$((milli / 1000)) - if [ "$c" -gt "$hottest" ]; then hottest=$c; label=$(cat "${z%/temp}/type" 2>/dev/null); fi - done - fi - - if [ "$hottest" -eq 0 ]; then - MESSAGE="no temperature sensors readable" - return 1 - fi - MESSAGE="${hottest}C${label:+ (${label})}" - [ "$hottest" -lt "$threshold" ] diff --git a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 deleted file mode 100644 index b0e76ce..0000000 --- a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 +++ /dev/null @@ -1,19 +0,0 @@ - # Every real filesystem must be under the threshold. tmpfs, devtmpfs, - # squashfs and overlay are excluded: they are either RAM, read-only, or - # container layers, and none of them fills up in a way an operator can act on. - local threshold={{ healthcheck_disk_threshold }} - local worst=0 worst_mount="" over="" - - while read -r pct mount; do - pct=${pct%\%} - [ -z "$pct" ] && continue - if [ "$pct" -gt "$worst" ]; then worst=$pct; worst_mount=$mount; fi - if [ "$pct" -ge "$threshold" ]; then over="${over}${over:+, }${mount} ${pct}%"; fi - done < <(df -P -x tmpfs -x devtmpfs -x squashfs -x overlay --output=pcent,target 2>/dev/null | tail -n +2) - - if [ -n "$over" ]; then - MESSAGE="over ${threshold}%: ${over}" - return 1 - fi - MESSAGE="max ${worst}% on ${worst_mount:-/}" - return 0 diff --git a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 deleted file mode 100644 index 141e478..0000000 --- a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ - # Liveness has no test to run: the fact that this script executed at all is - # the signal. What proves the host is alive is the PUSH arriving at Gatus, - # and what detects the host being dead is the heartbeat window expiring with - # no push. So this always succeeds - the reporting is the check. - MESSAGE="up since $(uptime -p 2>/dev/null || echo unknown)" - return 0 diff --git a/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 b/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 deleted file mode 100644 index 90c6193..0000000 --- a/ansible/roles/healthcheck/templates/checks/systemd-units.sh.j2 +++ /dev/null @@ -1,33 +0,0 @@ - # One result PER UNIT, not one for the host. A host running four services - # needs four endpoints: a single red light would tell you "something on vipy - # is down" without saying which, and that is the question you actually have. - # - # Keys are host-qualified because unit names collide - caddy runs on four - # machines. Gatus builds the key as sanitize(group)_sanitize(name), so - # group "services" + name "vipy/caddy" becomes services_vipy-caddy. - local prefix="{{ healthcheck_units_key_prefix }}" - local down="" - - for unit in {{ healthcheck_units | join(' ') }}; do - local state substate - state=$(systemctl is-active "$unit" 2>/dev/null || true) - substate=$(systemctl show -p SubState --value "$unit" 2>/dev/null || true) - - if [ "$state" = "active" ]; then - report_key "${prefix}-${unit}" "true" "${unit} active (${substate:-running})" - else - # `failed` and `inactive` are different stories: one crashed, one was - # stopped. Both are down, but the message should say which. - local detail="${unit} is ${state:-unknown}" - [ "$state" = "failed" ] && detail="${detail} (${substate:-failed})" - report_key "${prefix}-${unit}" "false" "$detail" - down="${down}${down:+, }${unit}=${state:-unknown}" - fi - done - - if [ -n "$down" ]; then - MESSAGE="down: ${down}" - return 1 - fi - MESSAGE="all {{ healthcheck_units | length }} units active" - return 0 diff --git a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 deleted file mode 100644 index 508c860..0000000 --- a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 +++ /dev/null @@ -1,18 +0,0 @@ - # OL means on line power. Anything else - OB (on battery), LB (low battery), - # or no answer at all - is a failure worth waking up for, because the - # hypervisor has a finite number of minutes left. - local ups="{{ healthcheck_ups_name }}" - local status charge runtime load - - status=$(upsc "${ups}@localhost" ups.status 2>/dev/null) - if [ -z "$status" ]; then - MESSAGE="cannot reach UPS ${ups} via upsd" - return 1 - fi - - charge=$(upsc "${ups}@localhost" battery.charge 2>/dev/null) - runtime=$(upsc "${ups}@localhost" battery.runtime 2>/dev/null) - load=$(upsc "${ups}@localhost" ups.load 2>/dev/null) - - MESSAGE="status=${status} charge=${charge}% runtime=${runtime}s load=${load}%" - [[ "$status" == *"OL"* ]] diff --git a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 deleted file mode 100644 index ebe88b9..0000000 --- a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 +++ /dev/null @@ -1,63 +0,0 @@ - # Five conditions, all of which have to hold. Ported from the check that - # infra/nodito/32_zfs_pool_setup_playbook.yml deployed, which was correct - - # only its reporting was tied to Uptime Kuma. - local pool="{{ healthcheck_zfs_pool }}" - local json issues="" - - json=$(zpool status -j "$pool" 2>&1) || { MESSAGE="zpool status failed: ${json}"; return 1; } - - # 1. pool state - local state - state=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].state') - [ "$state" = "ONLINE" ] || issues="${issues}${issues:+; }pool ${state}" - - # 2. every vdev and device ONLINE - local bad - bad=$(echo "$json" | jq -r --arg p "$pool" ' - .pools[$p].vdevs[] | .. | objects - | select(.state? and .state != "ONLINE") - | "\(.name // "unknown"):\(.state)"' 2>/dev/null | paste -sd, -) - [ -z "$bad" ] || issues="${issues}${issues:+; }devices ${bad}" - - # 3. resilver in progress - local fn st - fn=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.function // "NONE"') - st=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.state // "NONE"') - if [ "$fn" = "RESILVER" ] && [ "$st" = "SCANNING" ]; then - issues="${issues}${issues:+; }resilvering" - fi - - # 4. read/write/checksum errors. ZFS reports these as strings. - local errs - errs=$(echo "$json" | jq -r --arg p "$pool" ' - .pools[$p].vdevs[] | .. | objects - | select(.name? and ((.read_errors // "0" | tonumber) > 0 - or (.write_errors // "0" | tonumber) > 0 - or (.checksum_errors // "0" | tonumber) > 0)) - | "\(.name) r=\(.read_errors) w=\(.write_errors) c=\(.checksum_errors)"' 2>/dev/null | paste -sd, -) - [ -z "$errs" ] || issues="${issues}${issues:+; }errors ${errs}" - - # 5. errors from the last scrub - local scan_err - scan_err=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.errors // "0"') - if [ -n "$scan_err" ] && [ "$scan_err" != "0" ] && [ "$scan_err" != "null" ]; then - issues="${issues}${issues:+; }scan errors ${scan_err}" - fi - - # 6. Capacity. Not an error condition in `zpool status` - a 95% full pool is - # reported perfectly ONLINE - so it has to be read separately, and it is - # the failure you get warning of rather than the one you discover. - local capacity - capacity=$(zpool list -H -o capacity "$pool" 2>/dev/null | tr -dc '0-9') - if [ -z "$capacity" ]; then - issues="${issues}${issues:+; }cannot read capacity" - elif [ "$capacity" -ge {{ healthcheck_zfs_capacity_threshold }} ]; then - issues="${issues}${issues:+; }pool ${capacity}% full (>={{ healthcheck_zfs_capacity_threshold }}%)" - fi - - if [ -n "$issues" ]; then MESSAGE="$issues"; return 1; fi - - local scrub - scrub=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.start_time // "never"') - MESSAGE="${pool} ONLINE, ${capacity}% full, last scrub ${scrub}" - return 0 diff --git a/ansible/roles/healthcheck/templates/healthcheck.service.j2 b/ansible/roles/healthcheck/templates/healthcheck.service.j2 deleted file mode 100644 index 3ddd8aa..0000000 --- a/ansible/roles/healthcheck/templates/healthcheck.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description={{ healthcheck_description }} -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -User=root -ExecStart={{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -Environment=HEALTHCHECK_PUSH_BASE={{ healthcheck_push_base }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 deleted file mode 100644 index 6667878..0000000 --- a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# {{ healthcheck_name }} — {{ healthcheck_description }} -# Managed by Ansible (roles/healthcheck). Do not edit on the host. -# -# The exit code is the real answer; systemd stores it: -# systemctl is-failed {{ healthcheck_name }}-healthcheck.service -# The push below is an optional extra, and having no URL is normal. - -set -uo pipefail - -LOG_FILE="{{ healthcheck_log_dir }}/{{ healthcheck_name }}.log" -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -# Set only by checks that report several results; see healthcheck_push_base. -PUSH_BASE="${HEALTHCHECK_PUSH_BASE:-}" - -log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $*" >> "$LOG_FILE"; } - -# Report to Gatus as an external endpoint. Note this is a POST with a bearer -# token, not a GET with a query string - it is not the shape Uptime Kuma used. -report() { - local success="$1" message="$2" - - # No push URL is normal, not an error: the exit code below is a complete - # answer for anything reading unit state. - [ -n "$PUSH_URL" ] || return 0 - - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - - local code - code=$(curl -s -o /dev/null -w '%{http_code}' -X POST \ - --max-time 15 --retry 2 --retry-delay 3 \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null) - - if [ "$code" = "200" ]; then - log "reported success=${success}" - else - log "ERROR: report failed (HTTP ${code})" - return 1 - fi -} - -# Report to an arbitrary endpoint key under PUSH_BASE. Used by checks that -# produce one result per item rather than a single verdict. -report_key() { - local key="$1" success="$2" message="$3" - [ -n "$PUSH_BASE" ] || return 0 - local encoded - encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') - curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_BASE}/${key}/external?success=${success}&error=${encoded}" 2>/dev/null || true -} - -# ── the check ──────────────────────────────────────────────────────────────── -# -# The blank line before the closing brace below is load-bearing. Jinja strips an -# included template's trailing newline, and trim_blocks (on by default in -# Ansible) then eats the newline after {% raw %}{% endif %}{% endraw %} - so without it the brace lands -# on the same line as the check body's last statement, producing `return 0}` and -# a script that dies with "syntax error: unexpected end of file". -check() { -{% if healthcheck_check %} -{% include 'checks/' ~ healthcheck_check ~ '.sh.j2' %} -{% else %} - {{ healthcheck_command }} -{% endif %} - -} - -MESSAGE="" -if check; then - log "OK${MESSAGE:+ - $MESSAGE}" - report "true" "${MESSAGE:-ok}" - exit 0 -else - log "FAILED${MESSAGE:+ - $MESSAGE}" - report "false" "${MESSAGE:-check failed}" - exit 1 -fi diff --git a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 deleted file mode 100644 index 0271735..0000000 --- a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=Run {{ healthcheck_description }} -Requires={{ healthcheck_name }}-healthcheck.service - -[Timer] -OnBootSec={{ healthcheck_boot_delay }} -{% if healthcheck_on_calendar %} -OnCalendar={{ healthcheck_on_calendar }} -{% else %} -OnUnitActiveSec={{ healthcheck_interval }} -{% endif %} -# Run a missed occurrence on the next boot rather than silently skipping it. -Persistent=true -# Spread the pushes out so twelve hosts do not all report in the same second. -RandomizedDelaySec={{ healthcheck_randomized_delay | default('30') }} - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/mempool/README.md b/ansible/roles/mempool/README.md deleted file mode 100644 index d8282cc..0000000 --- a/ansible/roles/mempool/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# `mempool` - -Deploys the [Mempool](https://mempool.space) block explorer as a three-container -Docker Compose stack — MariaDB, backend, frontend — on `mempool-box`, and keeps a -health check on each. - -Converted from `deploy_mempool_playbook.yml` (745 lines) under Plan 6. The -playbook is now 37 lines: this role, plus a second play that publishes the -frontend through Caddy on the edge host. - -## Phases - -| | | -|---|---| -| `docker.yml` | Docker engine: repo, key, packages, service | -| `deploy.yml` | directories, `docker-compose.yml`, pull, up, wait-for-healthy | -| `healthcheck.yml` | three check scripts, three services, three timers | - -## Three health checks, not one - -Mempool is three moving parts and knowing *which* one is down is the point, so -each gets its own check, unit and timer, driven by the `mempool_healthchecks` -list: - -| | checks | -|---|---| -| `mariadb` | `docker inspect` health status of `mempool-db` | -| `backend` | `GET /api/v1/backend-info` | -| `frontend` | `GET /` | - -Each records its answer in its exit code, which systemd keeps: -`systemctl is-failed mempool-backend-healthcheck.service`. Reporting elsewhere -is one field per check, `push_url`, and is the plug-in point for whatever -monitoring exists. Empty means check, exit honestly, report nowhere. The URLs -are credentials, so callers pass them from the vault. - -Nothing here is specific to a monitoring product. The embedded Python that -created monitors over the Uptime Kuma API, the `/tmp` credentials file, the -push-URL file read back and parsed, and three systemd `Environment=` rewrites -are gone. - -## MariaDB owns its own data directory - -`{{ mempool_mysql_dir }}` is bind-mounted into the container, which runs as uid -**999** and must create files there. The playbook this replaced declared -`owner: "{{ ansible_user }}"` (1000) on it, which had drifted from reality ever -since the containers were created — unnoticed, because the playbook had not been -run since. - -That was not academic. The first real run of this role pulled a newer -`mariadb:10.11` and recreated `mempool-db`; had the chown still been in place, -MariaDB would have come back to a directory it could not write. The role now -ensures the directory exists and leaves ownership to the container. - -## `mempool_frontend_port` lives in `services_config.yml` - -Two hosts need it: this role deploys the frontend on `mempool-box`, and the Caddy -play proxies to it from the edge host. A role default is invisible to the second -play, so the value lives in `service_settings.mempool.frontend_port` and the role -default derives from it. - -## Expect `changed=2` on a converged host - -`Pull Mempool images` and `Deploy Mempool containers with docker compose` are -bare `command:` tasks with no `changed_when`, so they always report changed. -That is the idempotent floor, not drift. Everything else reports `ok`. - -**`mariadb:10.11` is a moving tag**, so a run can pull a newer patch release and -recreate the database container. Pin it if that is not what you want. diff --git a/ansible/roles/mempool/defaults/main.yml b/ansible/roles/mempool/defaults/main.yml deleted file mode 100644 index 9ce8f6a..0000000 --- a/ansible/roles/mempool/defaults/main.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Mempool Configuration Variables - -# Version - Pinned to specific release -mempool_version: "v3.2.1" - -# Directories -mempool_dir: /opt/mempool -mempool_data_dir: "{{ mempool_dir }}/data" -mempool_mysql_dir: "{{ mempool_dir }}/mysql" - -# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) -bitcoin_host: "knots-box" -bitcoin_rpc_port: 8332 -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from group_vars/all/vault.yml - -# Network - Fulcrum Electrum server (via Tailnet Magic DNS) -fulcrum_host: "fulcrum-box" -fulcrum_port: 50001 -fulcrum_tls: "false" - -# Mempool network mode -mempool_network: "mainnet" - -# Container ports (internal) -# The edge host's socket-proxy/Caddy play needs this too, and a role default is -# invisible outside this role. The authoritative value for the live deployment is -# in host_vars/mempool_box_local/main.yml, which outranks this; the value here is the -# protocol standard, so the role still works standalone. -mempool_frontend_port: 8080 -mempool_backend_port: 8999 - -# MariaDB settings -mariadb_database: "mempool" -mariadb_user: "mempool" -# Note: mariadb_mempool_password is loaded from group_vars/all/vault.yml - - - -# --- Health checks ---------------------------------------------------------- -# Three independent checks, because Mempool is three moving parts and knowing -# WHICH one is down is the whole point. Each records its answer in its exit -# code, which systemd keeps: -# systemctl is-failed mempool-backend-healthcheck.service -# -# push_url is where to report, and is the single plug-in point for whatever -# monitoring exists. Empty means check, exit honestly, report nowhere. -# The URLs are credentials, so callers pass them from the vault. -healthcheck_push_token: "" - -mempool_healthchecks: - - name: mariadb - label: MariaDB - push_url: "" - - name: backend - label: Backend - push_url: "" - - name: frontend - label: Frontend - push_url: "" diff --git a/ansible/roles/mempool/tasks/deploy.yml b/ansible/roles/mempool/tasks/deploy.yml deleted file mode 100644 index d98df7e..0000000 --- a/ansible/roles/mempool/tasks/deploy.yml +++ /dev/null @@ -1,75 +0,0 @@ ---- -- name: Create mempool directories - file: - path: "{{ item }}" - state: directory - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0755' - loop: - - "{{ mempool_dir }}" - - "{{ mempool_data_dir }}" - -# MariaDB owns its own data directory. The container runs as uid 999 and has to -# create files in there; this playbook declared owner: {{ ansible_user }} (1000), -# which had been drifting from reality ever since the containers were created and -# would have broken MariaDB the first time it needed a new file. It went -# unnoticed only because the playbook had not been run since. -# -# So: ensure the directory exists, and let the container own it. On a fresh -# install the mariadb image's entrypoint sets ownership itself. -- name: Ensure the MariaDB data directory exists - file: - path: "{{ mempool_mysql_dir }}" - state: directory - -- name: Create docker-compose.yml for Mempool - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ mempool_dir }}/docker-compose.yml" - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0644' -- name: Pull Mempool images - command: docker compose pull - args: - chdir: "{{ mempool_dir }}" - -- name: Deploy Mempool containers with docker compose - command: docker compose up -d - args: - chdir: "{{ mempool_dir }}" - -- name: Wait for MariaDB to be healthy - command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db - register: mariadb_health - until: mariadb_health.stdout == 'healthy' - retries: 30 - delay: 10 - changed_when: false - -- name: Wait for Mempool backend to start - uri: - url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - method: GET - status_code: 200 - timeout: 10 - register: backend_check - until: backend_check.status == 200 - retries: 30 - delay: 10 - ignore_errors: yes - -- name: Wait for Mempool frontend to be available - uri: - url: "http://localhost:{{ mempool_frontend_port }}" - method: GET - status_code: 200 - timeout: 10 - register: frontend_check - until: frontend_check.status == 200 - retries: 20 - delay: 5 - ignore_errors: yes - -# ═════════════════════════════════════════════════════════════════════════ diff --git a/ansible/roles/mempool/tasks/docker.yml b/ansible/roles/mempool/tasks/docker.yml deleted file mode 100644 index b97e63e..0000000 --- a/ansible/roles/mempool/tasks/docker.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -- name: Remove old Docker-related packages - apt: - name: - - docker.io - - docker-doc - - docker-compose - - podman-docker - - containerd - - runc - state: absent - purge: yes - autoremove: yes - -- name: Update apt cache - apt: - update_cache: yes - -- name: Install prerequisites - apt: - name: - - ca-certificates - - curl - state: present - -- name: Create directory for Docker GPG key - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - -- name: Download Docker GPG key - get_url: - url: https://download.docker.com/linux/debian/gpg - dest: /etc/apt/keyrings/docker.asc - mode: '0644' - -- name: Get Debian architecture - command: dpkg --print-architecture - register: deb_arch - changed_when: false - -- name: Add Docker repository - apt_repository: - repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" - filename: docker - state: present - update_cache: yes - -- name: Install Docker packages - apt: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present - update_cache: yes - -- name: Ensure Docker is started and enabled - systemd: - name: docker - enabled: yes - state: started - -- name: Add user to docker group - user: - name: "{{ ansible_user }}" - groups: docker - append: yes - diff --git a/ansible/roles/mempool/tasks/healthcheck.yml b/ansible/roles/mempool/tasks/healthcheck.yml deleted file mode 100644 index 1e1ac28..0000000 --- a/ansible/roles/mempool/tasks/healthcheck.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -# Three checks, one per moving part. The Uptime Kuma specifics that used to -# follow — an embedded Python script creating monitors over the API, a /tmp -# credentials file, a push-URL file read back and parsed, and three systemd -# Environment= rewrites — are gone. Where each reports is now hc.push_url. -- name: Create Mempool health check scripts - ansible.builtin.template: - src: "healthcheck-{{ hc.name }}.sh.j2" - dest: "/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh" - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" - -- name: Create systemd services for health checks - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.service" - owner: root - group: root - mode: "0600" - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" - -- name: Create systemd timers for health checks - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/mempool-{{ hc.name }}-healthcheck.timer" - owner: root - group: root - mode: '0644' - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" - -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -# Ungated on purpose: enabling a timer is deployment, not monitoring. The -# deprecation banner swept this up with the push plumbing, so Ansible stopped -# managing three timers that are in fact running on the host. -- name: Enable and start health check timers - systemd: - name: "mempool-{{ hc.name }}-healthcheck.timer" - enabled: yes - state: started - loop: "{{ mempool_healthchecks }}" - loop_control: - loop_var: hc - label: "{{ hc.name }}" diff --git a/ansible/roles/mempool/tasks/main.yml b/ansible/roles/mempool/tasks/main.yml deleted file mode 100644 index 9feb652..0000000 --- a/ansible/roles/mempool/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: docker.yml -- ansible.builtin.import_tasks: deploy.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/mempool/templates/docker-compose.yml.j2 b/ansible/roles/mempool/templates/docker-compose.yml.j2 deleted file mode 100644 index fda2ad1..0000000 --- a/ansible/roles/mempool/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,75 +0,0 @@ -# All containers use host network for Tailscale MagicDNS resolution -services: - mariadb: - image: mariadb:10.11 - container_name: mempool-db - restart: unless-stopped - network_mode: host - environment: - MYSQL_DATABASE: "{{ mariadb_database }}" - MYSQL_USER: "{{ mariadb_user }}" - MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" - MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" - volumes: - - {{ mempool_mysql_dir }}:/var/lib/mysql - healthcheck: - test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - mempool-backend: - image: mempool/backend:{{ mempool_version }} - container_name: mempool-backend - restart: unless-stopped - network_mode: host - environment: - # Database (localhost since all containers share host network) - DATABASE_ENABLED: "true" - DATABASE_HOST: "127.0.0.1" - DATABASE_DATABASE: "{{ mariadb_database }}" - DATABASE_USERNAME: "{{ mariadb_user }}" - DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" - # Bitcoin Core/Knots (via Tailnet MagicDNS) - CORE_RPC_HOST: "{{ bitcoin_host }}" - CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" - CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" - CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" - # Electrum (Fulcrum via Tailnet MagicDNS) - ELECTRUM_HOST: "{{ fulcrum_host }}" - ELECTRUM_PORT: "{{ fulcrum_port }}" - ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" - # Mempool settings - MEMPOOL_NETWORK: "{{ mempool_network }}" - MEMPOOL_BACKEND: "electrum" - MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" - MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" - volumes: - - {{ mempool_data_dir }}:/backend/cache - depends_on: - mariadb: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - mempool-frontend: - image: mempool/frontend:{{ mempool_version }} - container_name: mempool-frontend - restart: unless-stopped - network_mode: host - environment: - FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" - BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" - depends_on: - - mempool-backend - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s diff --git a/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 deleted file mode 100644 index fe0f57c..0000000 --- a/ansible/roles/mempool/templates/healthcheck-backend.sh.j2 +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Mempool backend health check — managed by Ansible (roles/mempool) -# The exit code is the answer; systemd keeps it. Reporting is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -BACKEND_PORT="{{ mempool_backend_port }}" - -check() { - curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 -} - -report() { - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "$1" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true -} - -if check; then report up "OK"; exit 0 -else echo "Mempool backend not responding"; report down "Mempool backend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 b/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 deleted file mode 100644 index 8452202..0000000 --- a/ansible/roles/mempool/templates/healthcheck-frontend.sh.j2 +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Mempool frontend health check — managed by Ansible (roles/mempool) -# The exit code is the answer; systemd keeps it. Reporting is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -FRONTEND_PORT="{{ mempool_frontend_port }}" - -check() { - curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 -} - -report() { - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "$1" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true -} - -if check; then report up "OK"; exit 0 -else echo "Mempool frontend not responding"; report down "Mempool frontend not responding"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 b/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 deleted file mode 100644 index 922adab..0000000 --- a/ansible/roles/mempool/templates/healthcheck-mariadb.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Mempool MariaDB health check — managed by Ansible (roles/mempool) -# The exit code is the answer; systemd keeps it. Reporting is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" - -check() { -{% raw %} - [ "$(docker inspect --format='{{.State.Health.Status}}' mempool-db 2>/dev/null)" = "healthy" ] -{% endraw %} -} - -report() { - # No push URL is normal, not an error. The previous version logged - # "ERROR: UPTIME_KUMA_PUSH_URL not set" on every fire, once a minute. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "$1" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${2// /%20}" || true -} - -if check; then report up "OK"; exit 0 -else echo "MariaDB container unhealthy"; report down "MariaDB container unhealthy"; exit 1; fi diff --git a/ansible/roles/mempool/templates/healthcheck.service.j2 b/ansible/roles/mempool/templates/healthcheck.service.j2 deleted file mode 100644 index 76fa4e9..0000000 --- a/ansible/roles/mempool/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Mempool {{ hc.label }} Health Check -After=network.target docker.service - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/mempool-{{ hc.name }}-healthcheck-push.sh -Environment=HEALTHCHECK_PUSH_URL={{ hc.push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/mempool/templates/healthcheck.timer.j2 b/ansible/roles/mempool/templates/healthcheck.timer.j2 deleted file mode 100644 index 868cb7c..0000000 --- a/ansible/roles/mempool/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Mempool {{ hc.name }} Health Check Timer - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/phoenixd/README.md b/ansible/roles/phoenixd/README.md deleted file mode 100644 index 3c147e3..0000000 --- a/ansible/roles/phoenixd/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# `phoenixd` - -Deploys and runs [phoenixd](https://phoenix.acinq.co/server), an ACINQ Lightning -node, on the edge host. LNBits uses it as a wallet backend. The HTTP API stays on -loopback — phoenixd is never published through Caddy. - -Converted from `deploy_phoenixd_playbook.yml` (552 lines) under Plan 6. The -playbook is now 18 lines. - -## Phases - -| | | -|---|---| -| `install.yml` | packages, system user, directories, versioned download and install | -| `service.yml` | systemd unit, start, then first-boot checks (config written, seed created) | -| `healthcheck.yml` | check script, unit, timer | - -## The seed - -`{{ phoenixd_data_dir }}/seed.dat` **is** the funds. phoenixd is deliberately -excluded from the automated backups (Plan 5, Model C): the seed is twelve fixed -words that never change, so an automated job would only manufacture more copies -of a static secret on more machines. Write them down offline, once. - -Note the live file is mode `0644`. That is phoenixd's own doing, not this role's, -and it is worth tightening. - -## Monitoring: one variable, no product knowledge - -The check asks the node itself — the service must be active **and** -`phoenix-cli getinfo` must return a `nodeId` — and records the answer in its exit -code, which systemd keeps: - -```bash -systemctl is-failed phoenixd-healthcheck.service -``` - -That is a complete answer with no monitoring system involved. To report -elsewhere, set `healthcheck_push_url` to anything accepting an HTTP ping. Gone -from this role: the embedded Python that created monitors over the Uptime Kuma -API, the `/tmp` credentials file, the push-URL file written and parsed back, and -the systemd `Environment=` rewrite. - -### Two things the conversion fixed - -**The check used to log an error once a minute.** Its `Environment=` push URL had -been empty since the decommissioning, and the script printed -`ERROR: UPTIME_KUMA_PUSH_URL not set` on every fire — roughly 1,400 times a day. -The exit code was still correct, so nothing was broken; it was pure noise, and -noise that trains you to ignore the log. An unset push URL is now normal and -silent. - -**`Enable and start phoenixd health check timer` was guarded by -`uptime_kuma_enabled`** and so had not run since the decommissioning — while the -timer itself was still live on the host from before. Ansible had quietly stopped -managing something that was still running. Ungated. diff --git a/ansible/roles/phoenixd/handlers/main.yml b/ansible/roles/phoenixd/handlers/main.yml deleted file mode 100644 index 7779684..0000000 --- a/ansible/roles/phoenixd/handlers/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Restart phoenixd - systemd: - name: phoenixd - state: restarted - daemon_reload: yes - -- name: Restart phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - state: restarted - daemon_reload: yes diff --git a/ansible/roles/phoenixd/tasks/healthcheck.yml b/ansible/roles/phoenixd/tasks/healthcheck.yml deleted file mode 100644 index d439047..0000000 --- a/ansible/roles/phoenixd/tasks/healthcheck.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -# Everything here answers "is phoenixd healthy" and records the answer. The -# Uptime Kuma specifics that used to follow it — an embedded Python script -# creating monitors over the API, a /tmp credentials file, a push-URL file read -# back and parsed, and a systemd Environment= rewrite — are gone. What reports -# where is now one variable, healthcheck_push_url. See the role README. -- name: Create phoenixd health check script - ansible.builtin.template: - src: healthcheck.sh.j2 - dest: "{{ phoenixd_healthcheck_script_path }}" - owner: root - group: root - mode: "0755" - validate: "bash -n %s" - -- name: Create phoenixd health check systemd service - ansible.builtin.template: - src: healthcheck.service.j2 - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" - owner: root - group: root - mode: "0600" - notify: Restart phoenixd health check timer - -- name: Create phoenixd health check systemd timer - ansible.builtin.template: - src: healthcheck.timer.j2 - dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" - owner: root - group: root - mode: "0644" - notify: Restart phoenixd health check timer - -- name: Reload systemd daemon after health check units - systemd: - daemon_reload: yes - -# Ungated on purpose. This was guarded by `uptime_kuma_enabled`, but enabling a -# timer is deployment, not monitoring — the deprecation banner swept it up with -# the push plumbing. The timer is in fact running on the host, from before the -# decommissioning, so the guard meant Ansible had stopped managing something -# that was still live. -- name: Enable and start phoenixd health check timer - systemd: - name: "{{ phoenixd_healthcheck_service_name }}.timer" - enabled: yes - state: started - -- name: Display post-install information - debug: - msg: | - ✓ phoenixd {{ phoenixd_version }} deployed - - Status: systemctl status phoenixd - Logs: journalctl -u phoenixd -f - CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo - HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) - Data dir: {{ phoenixd_data_dir }} - Health: systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service - - API password (needed to wire LNBits up to this node): - sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf - - ⚠️ BACK UP THE SEED: {{ phoenixd_data_dir }}/seed.dat - Losing it means losing the funds. phoenixd is deliberately excluded - from the automated backups (Plan 5, Model C) because the seed is 12 - fixed words — write them down offline, once: - sudo cat {{ phoenixd_data_dir }}/seed.dat diff --git a/ansible/roles/phoenixd/tasks/install.yml b/ansible/roles/phoenixd/tasks/install.yml deleted file mode 100644 index 89a4fa0..0000000 --- a/ansible/roles/phoenixd/tasks/install.yml +++ /dev/null @@ -1,125 +0,0 @@ ---- -- name: Install phoenixd runtime dependencies - apt: - name: - - unzip - - curl - state: present - update_cache: yes - -# System User and Directories -- name: Create phoenixd system group - group: - name: "{{ phoenixd_group }}" - system: yes - -- name: Create phoenixd system user - user: - name: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ phoenixd_home }}" - create_home: yes - comment: "phoenixd Lightning node" - -- name: Create phoenixd home directory - file: - path: "{{ phoenixd_home }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0750" - -- name: Create phoenixd data directory - file: - path: "{{ phoenixd_data_dir }}" - state: directory - owner: "{{ phoenixd_user }}" - group: "{{ phoenixd_group }}" - mode: "0700" - -# Download and Install -- name: Check if phoenixd is already installed - stat: - path: "{{ phoenixd_bin_dir }}/phoenixd" - register: phoenixd_binary - -- name: Check installed phoenixd version - command: "{{ phoenixd_bin_dir }}/phoenixd --version" - register: phoenixd_installed_version - changed_when: false - failed_when: false - when: phoenixd_binary.stat.exists - -- name: Decide whether phoenixd needs installing - set_fact: - phoenixd_needs_install: >- - {{ not phoenixd_binary.stat.exists - or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} - -- name: Download phoenixd {{ phoenixd_version }} - get_url: - url: "{{ phoenixd_url }}" - dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - mode: "0644" - when: phoenixd_needs_install | bool - -- name: Create temporary extraction directory - file: - path: /tmp/phoenixd-extract - state: directory - mode: "0755" - when: phoenixd_needs_install | bool - -- name: Extract phoenixd archive - unarchive: - src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" - dest: /tmp/phoenixd-extract - remote_src: yes - when: phoenixd_needs_install | bool - -- name: Locate extracted binaries - find: - paths: /tmp/phoenixd-extract - patterns: "{{ item }}" - recurse: yes - file_type: file - register: phoenixd_extracted - loop: - - phoenixd - - phoenix-cli - when: phoenixd_needs_install | bool - -- name: Fail if the archive did not contain the expected binaries - assert: - that: - - item.files | length > 0 - fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - -- name: Install phoenixd and phoenix-cli binaries - copy: - src: "{{ item.files[0].path }}" - dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" - remote_src: yes - owner: root - group: root - mode: "0755" - loop: "{{ phoenixd_extracted.results }}" - loop_control: - label: "{{ item.item }}" - when: phoenixd_needs_install | bool - notify: Restart phoenixd - -- name: Clean up phoenixd download artifacts - file: - path: "{{ item }}" - state: absent - loop: - - "/tmp/phoenixd-{{ phoenixd_version }}.zip" - - /tmp/phoenixd-extract - diff --git a/ansible/roles/phoenixd/tasks/main.yml b/ansible/roles/phoenixd/tasks/main.yml deleted file mode 100644 index bc7ff05..0000000 --- a/ansible/roles/phoenixd/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# import_tasks, not include_tasks: static imports stay visible to --list-tasks, -# which is how this conversion was verified against the playbook it replaced. -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: service.yml -- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/phoenixd/tasks/service.yml b/ansible/roles/phoenixd/tasks/service.yml deleted file mode 100644 index 79bf6f5..0000000 --- a/ansible/roles/phoenixd/tasks/service.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -- name: Build phoenixd command line arguments - set_fact: - phoenixd_args: >- - {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) - + ['--chain', phoenixd_chain] - + ['--auto-liquidity', phoenixd_auto_liquidity] - + ['--http-bind-ip', phoenixd_http_bind_ip] - + ['--http-bind-port', phoenixd_http_bind_port | string] - + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) - + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) - + ['--silent'] }} - -- name: Create phoenixd systemd service - ansible.builtin.template: - src: phoenixd.service.j2 - dest: /etc/systemd/system/phoenixd.service - owner: root - group: root - mode: "0644" - notify: Restart phoenixd - -- name: Reload systemd daemon - systemd: - daemon_reload: yes - -- name: Enable and start phoenixd - systemd: - name: phoenixd - enabled: yes - state: started - -- name: Flush handlers so phoenixd is running before we inspect its data dir - meta: flush_handlers - -# --- First boot checks --- -- name: Wait for phoenixd to write its config file - wait_for: - path: "{{ phoenixd_data_dir }}/phoenix.conf" - state: present - timeout: 120 - -- name: Check that the seed file exists - stat: - path: "{{ phoenixd_data_dir }}/seed.dat" - register: phoenixd_seed_file - -- name: Fail if phoenixd did not create a seed - assert: - that: - - phoenixd_seed_file.stat.exists - fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" diff --git a/ansible/roles/phoenixd/templates/healthcheck.service.j2 b/ansible/roles/phoenixd/templates/healthcheck.service.j2 deleted file mode 100644 index 6ff21ab..0000000 --- a/ansible/roles/phoenixd/templates/healthcheck.service.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=phoenixd Health Check -After=network.target phoenixd.service - -[Service] -Type=oneshot -User=root -ExecStart={{ phoenixd_healthcheck_script_path }} -Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} -Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/phoenixd/templates/healthcheck.sh.j2 b/ansible/roles/phoenixd/templates/healthcheck.sh.j2 deleted file mode 100644 index a1717c5..0000000 --- a/ansible/roles/phoenixd/templates/healthcheck.sh.j2 +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# phoenixd health check — managed by Ansible (roles/phoenixd) -# -# Asks the node whether it is healthy and records the answer in the exit code, -# which systemd keeps: -# systemctl is-failed {{ phoenixd_healthcheck_service_name }}.service -# That is a complete answer on its own. Reporting anywhere else is optional. -PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" -PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" -export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" - -check_phoenixd() { - # Service must be active and the node must answer getinfo. - # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, - # but not the bind address, so pass it explicitly. - systemctl is-active --quiet phoenixd && \ - {{ phoenixd_bin_dir }}/phoenix-cli \ - --http-bind-ip {{ phoenixd_http_bind_ip }} \ - --http-bind-port {{ phoenixd_http_bind_port }} \ - getinfo 2>/dev/null | grep -q '"nodeId"' -} - -report() { - local status=$1 msg=$2 - # No push URL configured is NORMAL, not an error: the exit code below still - # answers the question. The previous version logged ERROR here on every - # single fire, once a minute, which is noise that trains you to ignore it. - [ -n "$PUSH_URL" ] || return 0 - # Gatus external endpoint: a POST with a bearer token, NOT Uptime Kuma's - # GET with ?status=up. The callers still pass up/down, so the mapping is - # done here rather than at every call site. - local _ok=false - [ "${status}" = "up" ] && _ok=true - curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ - -H "Authorization: Bearer ${PUSH_TOKEN}" \ - "${PUSH_URL}?success=${_ok}&error=${msg// /%20}" || true -} - -if check_phoenixd; then - report "up" "OK" - exit 0 -else - echo "phoenixd is not responding" - report "down" "phoenixd not responding" - exit 1 -fi diff --git a/ansible/roles/phoenixd/templates/healthcheck.timer.j2 b/ansible/roles/phoenixd/templates/healthcheck.timer.j2 deleted file mode 100644 index ee07257..0000000 --- a/ansible/roles/phoenixd/templates/healthcheck.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=phoenixd Health Check Timer - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/phoenixd/templates/phoenixd.service.j2 b/ansible/roles/phoenixd/templates/phoenixd.service.j2 deleted file mode 100644 index 44fdb94..0000000 --- a/ansible/roles/phoenixd/templates/phoenixd.service.j2 +++ /dev/null @@ -1,31 +0,0 @@ -[Unit] -Description=phoenixd - Lightning Network Node -Documentation=https://phoenix.acinq.co/server -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User={{ phoenixd_user }} -Group={{ phoenixd_group }} -WorkingDirectory={{ phoenixd_home }} -Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} -ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} -Restart=always -RestartSec=30 -TimeoutStartSec=120 -TimeoutStopSec=120 -StandardOutput=journal -StandardError=journal - -# Hardening: the node only ever writes to its own data directory -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths={{ phoenixd_data_dir }} - -LimitNOFILE=65535 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/signal_api/README.md b/ansible/roles/signal_api/README.md deleted file mode 100644 index eadcfb7..0000000 --- a/ansible/roles/signal_api/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# signal_api - -Runs [signal-cli-rest-api](https://github.com/bbernhard/signal-cli-rest-api) on -the `observability` host. Gatus uses it to deliver alerts over Signal. - -Gatus does not speak Signal — it POSTs JSON to this service, which holds the -Signal identity and does the protocol work. - -## It is never published, and that is not optional - -**This API has no authentication of any kind.** No key, no token, no basic auth. -Anything that can reach the port can send messages as your identity and read -your Signal. So the compose file publishes **no ports at all** and there is no -Caddy vhost. - -Gatus reaches it over a shared docker network (`monitoring`) by service name: -`http://signal-api:8080`. That is also *why* a shared network is needed rather -than a published port — Gatus runs in a container, so `127.0.0.1` for Gatus is -the Gatus container, not the host. - -The network is created by an explicit Ansible task in both this role and -`gatus`, so neither stack has to be deployed before the other. - -## MODE, and why `native` - -Upstream offers `normal`, `native`, `json-rpc` and `json-rpc-native`. The -json-rpc modes keep a resident JVM daemon and upstream describes them as -"increased memory". - -**This VPS has 464 MB of RAM**, already running Gatus and Caddy. A resident JVM -is not affordable. `native` runs a precompiled GraalVM binary per request — no -daemon, no resident cost — and alerts are rare enough that paying startup cost -per alert is the right trade. - -## Linking the device — a one-time manual step - -Ansible cannot scan a QR code, so this is manual. **Do not use -`/v1/qrcodelink`** — it is broken in `native` mode. - -### The trap - -`GET /v1/qrcodelink?device_name=...` returns: - -```json -{"error":"Couldn't create QR code: no data to encode"} -``` - -The linking itself is fine: running the binary directly inside the container -emits a perfectly good provisioning URI. - -``` -$ docker exec signal-api signal-cli-native link -n gatus -sgnl://linkdevice?uuid=...&pub_key=... -``` - -It is the REST wrapper that fails to capture that output in `native` mode. - -**Do not "fix" this by switching MODE to `normal` or `json-rpc`.** That puts a -JVM in the path of *every alert* on a 464 MB host, permanently degrading the -running system to work around a step performed once. Generate the QR yourself -instead. - -### The procedure - -**`docker exec` runs as root, but the service runs as uid 1000.** Without -`--config`, signal-cli writes the linked account to `/root/.local/share/signal-cli` -— the container's ephemeral layer, NOT the mounted volume. It looks like it -worked (`Associated with: +34…`), `/v1/accounts` keeps returning `[]`, and the -account is destroyed on the next `docker compose up`. Always pass `--config`. - -1. Start the link and capture the URI. It must keep running while you scan: - - docker exec signal-api sh -c "rm -f /tmp/link.uri; \ - nohup signal-cli-native --config /home/.local/share/signal-cli \ - link -n gatus > /tmp/link.uri 2>/tmp/link.log & echo started" - sleep 10 - docker exec signal-api cat /tmp/link.uri - - Do **not** add `setsid`, and do **not** background `docker exec` itself from - the host — the first stops the URI appearing, the second is killed when the - Ansible task returns. The output is block-buffered because stdout is a file, - so the URI appears only after several seconds; `stdbuf` does not help, as the - buffering is GraalVM's, not libc's. - -2. Render the QR on your own machine and scan it: - - qrencode -o /tmp/qr.png -s 12 -m 4 "sgnl://linkdevice?uuid=...&pub_key=..." - -3. Phone: Signal → Settings → Linked devices → **+** → scan. Provisioning links - expire in a couple of minutes, so generate and scan in one sitting. - -4. Confirm — this must list the number, not `[]`: - - docker exec signal-api curl -s http://localhost:8080/v1/accounts - -5. Send a test message: - - docker exec signal-api curl -s -X POST -H "Content-Type: application/json" \ - -d '{"message":"test","number":"+34…","recipients":["+34…"]}' \ - http://localhost:8080/v2/send - -Alerts are sent **from your own number**, so sending to yourself lands in Note -to Self. If the device is ever unlinked from the phone, alerts stop silently — -which is why this service is itself monitored. - -### If the phone says "network error" - -The phone is not the problem. `chat.signal.org` resolves to AWS Global -Accelerator **dualstack** addresses with the AAAA records first, this container -has no IPv6 address at all, and this host's IPv6 path is broken — the same edge -that returned a bogus 404 for the Go tarball. signal-cli reaches for an -unreachable IPv6 address and dies with `Link request error: Connection closed!`, -while the phone can only report a failed handshake. - -That is what `gai.conf` (mounted at `/etc/gai.conf`) fixes. If linking starts -failing again, check it is still mounted and that `getent ahosts chat.signal.org` -returns an IPv4 address first. - -## Backups - -Deliberately **not** backed up. The data directory holds Signal private keys, -and the recovery path is to link again from the phone — which takes a minute and -does not depend on any stored artefact. Backing it up would copy a credential -off the host to buy nothing. - -## Verifying - -```bash -docker ps --filter name=signal-api -docker exec signal-api curl -fsS http://localhost:8080/v1/health -docker exec signal-api curl -fsS http://localhost:8080/v1/accounts -docker logs signal-api --tail 50 -``` diff --git a/ansible/roles/signal_api/defaults/main.yml b/ansible/roles/signal_api/defaults/main.yml deleted file mode 100644 index 6bb7d4f..0000000 --- a/ansible/roles/signal_api/defaults/main.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# signal-cli-rest-api: the transport Gatus uses to send Signal messages. -# -# Gatus does not speak Signal. It POSTs JSON to this service, which holds the -# actual Signal identity and does the protocol work. - -# Pinned by digest for the same reason as Gatus: a tag is mutable. -# Upstream publishes no versioned tags worth pinning to, so this pins the -# DIGEST that `latest` resolved to when this was reviewed. `latest` is a moving -# target; a digest is a content address, and `docker compose pull` either -# fetches exactly this image or fails. -signal_api_image_digest: "sha256:2399d449123cdad56c4d859277e3b9127e1a00c4d2ab4601c239882609286cf8" -signal_api_image: "bbernhard/signal-cli-rest-api@{{ signal_api_image_digest }}" - -signal_api_dir: /opt/signal-api -signal_api_data_dir: "{{ signal_api_dir }}/data" - -# MODE matters on this host. Upstream offers normal / native / json-rpc / -# json-rpc-native. json-rpc keeps a resident JVM daemon and upstream describes it -# as "increased memory" - this VPS has 464MB total and already runs Gatus and -# Caddy, so a resident JVM is not affordable. `native` runs a precompiled -# GraalVM binary per request: no daemon, no resident cost, and alerts are rare -# enough that paying startup per alert is the right trade. -signal_api_mode: native - -# Port INSIDE the shared docker network. Never published to the host: this API -# has NO AUTHENTICATION of any kind. Anyone who can reach it can send messages -# as you and read your Signal. -signal_api_port: 8080 - -# Both this and Gatus join this network so Gatus can reach the API by service -# name. Gatus runs in a container, so the host's loopback is NOT reachable from -# it - this is why a shared network is required rather than a published port. -signal_api_network: monitoring -signal_api_service_name: signal-api - -# The uid the upstream image drops to (`setpriv --reuid=1000`). The data -# directory must be owned by it or signal-cli cannot write the account. -signal_api_uid: 1000 diff --git a/ansible/roles/signal_api/tasks/main.yml b/ansible/roles/signal_api/tasks/main.yml deleted file mode 100644 index 1aa92b4..0000000 --- a/ansible/roles/signal_api/tasks/main.yml +++ /dev/null @@ -1,93 +0,0 @@ ---- -- name: Assert Docker is available - ansible.builtin.command: docker --version - register: signal_docker_check - changed_when: false - -# Created explicitly rather than by either compose file, so neither stack has to -# be deployed before the other and neither owns it. -- name: Ensure the shared monitoring network exists - ansible.builtin.command: "docker network create {{ signal_api_network }}" - register: signal_net - changed_when: "'already exists' not in signal_net.stderr" - failed_when: - - signal_net.rc != 0 - - "'already exists' not in signal_net.stderr" - -- name: Create the signal-api directory - ansible.builtin.file: - path: "{{ signal_api_dir }}" - state: directory - owner: root - group: root - mode: "0755" - -# Owned by the container's uid, NOT root. -# -# The image drops to uid 1000 (`setpriv --reuid=1000`), and a root-owned 0700 -# directory cannot be traversed by uid 1000 - signal-cli then fails to write the -# account and linking silently never completes, leaving a 39-byte accounts.json -# with no accounts and the API returning "Failed to read local accounts list". -# -# 0700 on uid 1000 is still private: only that uid and root can read the Signal -# private keys, which is the property actually wanted. -- name: Create the signal-api data directory owned by the container user - ansible.builtin.file: - path: "{{ signal_api_data_dir }}" - state: directory - owner: "{{ signal_api_uid }}" - group: "{{ signal_api_uid }}" - mode: "0700" - -- name: Write the IPv4-preference resolver config - ansible.builtin.template: - src: gai.conf.j2 - dest: "{{ signal_api_dir }}/gai.conf" - owner: root - group: root - mode: "0644" - -- name: Write the docker compose file - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ signal_api_dir }}/docker-compose.yml" - owner: root - group: root - mode: "0644" - -- name: Pull the pinned signal-api image - ansible.builtin.command: - cmd: docker compose pull - chdir: "{{ signal_api_dir }}" - register: signal_pull - changed_when: "'Downloaded newer image' in signal_pull.stderr or 'Pull complete' in signal_pull.stderr" - -- name: Start signal-api - ansible.builtin.command: - cmd: docker compose up -d --remove-orphans - chdir: "{{ signal_api_dir }}" - register: signal_up - changed_when: "'Started' in signal_up.stderr or 'Created' in signal_up.stderr or 'Recreated' in signal_up.stderr" - -- name: Wait for the API to answer - ansible.builtin.command: - cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/health" - register: signal_health - until: signal_health.rc == 0 - retries: 12 - delay: 5 - changed_when: false - -- name: Report whether an account is linked yet - ansible.builtin.command: - cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/accounts" - register: signal_accounts - changed_when: false - failed_when: false - -- name: Show the linking status - ansible.builtin.debug: - msg: >- - {{ 'Linked account(s): ' ~ signal_accounts.stdout - if (signal_accounts.stdout | default('[]') | trim) not in ['[]', '', 'null'] - else 'NO ACCOUNT LINKED YET - this is a one-time manual step, see the role README.' }} diff --git a/ansible/roles/signal_api/templates/docker-compose.yml.j2 b/ansible/roles/signal_api/templates/docker-compose.yml.j2 deleted file mode 100644 index 9825299..0000000 --- a/ansible/roles/signal_api/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ -# Managed by Ansible (roles/signal_api) -services: - {{ signal_api_service_name }}: - image: {{ signal_api_image }} - container_name: {{ signal_api_service_name }} - restart: unless-stopped - - environment: - MODE: "{{ signal_api_mode }}" - - volumes: - # Prefer IPv4. See gai.conf.j2 - without this, signal-cli reaches for - # chat.signal.org's IPv6 address, which is unreachable from here, and - # linking fails with an opaque "network error" on the phone. - - {{ signal_api_dir }}/gai.conf:/etc/gai.conf:ro - # Holds the Signal identity: the linked-device keys and registration - # state. Lose this and the device must be linked again by scanning a new - # QR code from the phone. It is also the most sensitive thing on this - # host - anyone with these keys can send and read Signal as you. - - {{ signal_api_data_dir }}:/home/.local/share/signal-cli - - networks: - - {{ signal_api_network }} - - # NO PORTS. Deliberately. - # - # This API has no authentication whatsoever - no key, no token, nothing. - # Publishing it, even on 127.0.0.1, would expose "send a Signal message as - # this identity" to anything that can reach the host. Gatus talks to it over - # the shared docker network by service name instead, which is why no port is - # published and why there is no Caddy vhost. - - healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:8080/v1/health"] - interval: 60s - timeout: 5s - retries: 3 - - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - -networks: - {{ signal_api_network }}: - external: true diff --git a/ansible/roles/signal_api/templates/gai.conf.j2 b/ansible/roles/signal_api/templates/gai.conf.j2 deleted file mode 100644 index 9b2cca4..0000000 --- a/ansible/roles/signal_api/templates/gai.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -# Managed by Ansible (roles/signal_api) -# -# Prefer IPv4 over IPv6 in getaddrinfo. -# -# chat.signal.org resolves to AWS Global Accelerator dualstack addresses, and -# DNS returns the AAAA records first. This container has NO IPv6 address at all, -# and this host's IPv6 path is unreliable anyway - the same edge that made -# Google's IPv6 endpoint return a confident 404 for the Go tarball. -# -# signal-cli would connect to the AAAA address, fail, and report -# Link request error: Connection closed! -# while the phone showed a bare "network error" - a failure with no obvious -# cause on either end. -# -# This line flips the precedence so IPv4-mapped addresses sort first, which is -# the standard glibc fix. It does NOT disable IPv6; it only changes the order. -precedence ::ffff:0:0/96 100 diff --git a/ansible/roles/socket_proxy/README.md b/ansible/roles/socket_proxy/README.md deleted file mode 100644 index cebb95f..0000000 --- a/ansible/roles/socket_proxy/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# `socket_proxy` - -Exposes a service running on a private Tailscale host through a public TCP port -on an edge machine, using `systemd-socket-proxyd`. Writes a `.socket` and a -`.service` unit, enables the socket, opens the UFW port, and checks the upstream -is reachable. - -## Usage - -```yaml -- ansible.builtin.include_role: - name: socket_proxy - vars: - socket_proxy_name: fulcrum-ssl # -> fulcrum-ssl-proxy.{socket,service} - socket_proxy_description: "Fulcrum SSL" # -> "Fulcrum SSL Proxy Socket" - socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" - socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" -``` - -`socket_proxy_upstream_port` defaults to `socket_proxy_listen_port`, which is -what all three current callers want. - -## Optional unit settings - -These exist because the **live** `bitcoin-p2p-proxy` units on vipy carried -settings the playbook never wrote. Somebody added them by hand, so running -`deploy_bitcoin_knots_playbook.yml` would have silently removed them: - -| Variable | Emits | Why it matters | -|---|---|---| -| `socket_proxy_free_bind` | `FreeBind=true` in `[Socket]` | Lets the socket bind to an address that is not up yet. Without it the socket can fail to start on boot. | -| `socket_proxy_documentation` | `Documentation=` in both units | Cosmetic. | -| `socket_proxy_timeout_stop_sec` | `TimeoutStopSec=` | Bounds how long a stop can hang. | -| `socket_proxy_log_to_journal` | `StandardOutput=journal` + `StandardError=journal` | Cosmetic on modern systemd, which defaults to the journal anyway. | - -Only `bitcoin-p2p` passes any of them. - -## `socket_proxy_ufw_comment` - -Defaults to `" public access"`, which reproduces the live rule -comment for bitcoin-p2p and fulcrum-ssl. **datum-stratum must pass it -explicitly** — its live comment is `DATUM Gateway Stratum public access` while -the derived default would be `DATUM Stratum public access`, and UFW treats the -comment as part of the rule, so the mismatch rewrites the rule on every run. - -## The upstream check never fails the play - -`wait_for` on the upstream carries `failed_when: false`. The proxy is correctly -configured whether or not the backend happens to be up, and this is the one task -that depends on another machine. The original plays used `ignore_errors: yes`, -which prints a red "ignoring" line; `failed_when: false` is the quieter -equivalent. - -## Restarts - -The handler restarts the `.socket`, not the `.service` — that is what picks up a -changed unit; the service is started by the socket on the next connection. - -**Restarting a socket drops connections that are currently open through it.** -For bitcoin-p2p that means peers reconnect; for datum-stratum it means a mining -client has to reconnect and may lose in-flight shares. The handler only fires -when a unit file actually changes. diff --git a/ansible/roles/socket_proxy/defaults/main.yml b/ansible/roles/socket_proxy/defaults/main.yml deleted file mode 100644 index 6a512ea..0000000 --- a/ansible/roles/socket_proxy/defaults/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Required -socket_proxy_name: "" # "bitcoin-p2p" -> bitcoin-p2p-proxy.{socket,service} -socket_proxy_description: "" # "Bitcoin P2P" -> "Bitcoin P2P Proxy Socket" -socket_proxy_listen_port: 0 # public port on the edge host -socket_proxy_upstream_host: "" # Tailscale hostname, e.g. "knots-box" - -# Optional -socket_proxy_upstream_port: "" # defaults to socket_proxy_listen_port -socket_proxy_documentation: "" # Documentation= in both units -socket_proxy_free_bind: false # FreeBind=true: bind before the address is up -socket_proxy_timeout_stop_sec: "" # TimeoutStopSec= -socket_proxy_log_to_journal: false # StandardOutput/StandardError=journal - -# Firewall -socket_proxy_ufw_proto: tcp -socket_proxy_ufw_comment: "" # defaults to " public access" diff --git a/ansible/roles/socket_proxy/handlers/main.yml b/ansible/roles/socket_proxy/handlers/main.yml deleted file mode 100644 index 7fdf421..0000000 --- a/ansible/roles/socket_proxy/handlers/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Restarting the .socket is what picks up a changed unit; the .service is -# started by the socket on the next connection. -- name: Restart socket proxy - ansible.builtin.systemd: - name: "{{ socket_proxy_name }}-proxy.socket" - state: restarted - daemon_reload: yes diff --git a/ansible/roles/socket_proxy/tasks/main.yml b/ansible/roles/socket_proxy/tasks/main.yml deleted file mode 100644 index 4315555..0000000 --- a/ansible/roles/socket_proxy/tasks/main.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -- name: Assert socket_proxy parameters are sane - ansible.builtin.assert: - that: - - socket_proxy_name | length > 0 - - socket_proxy_description | length > 0 - - socket_proxy_listen_port | int > 0 - - socket_proxy_upstream_host | length > 0 - fail_msg: >- - socket_proxy: '{{ socket_proxy_name | default("") }}' needs a name, - a description, a listen port and an upstream host. - quiet: true - -- name: "Create the {{ socket_proxy_name }}-proxy socket unit" - ansible.builtin.template: - src: proxy.socket.j2 - dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.socket" - owner: root - group: root - mode: '0644' - notify: Restart socket proxy - -- name: "Create the {{ socket_proxy_name }}-proxy service unit" - ansible.builtin.template: - src: proxy.service.j2 - dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.service" - owner: root - group: root - mode: '0644' - notify: Restart socket proxy - -- name: "Enable and start the {{ socket_proxy_name }}-proxy socket" - ansible.builtin.systemd: - name: "{{ socket_proxy_name }}-proxy.socket" - enabled: yes - state: started - daemon_reload: yes - -- name: "Allow the {{ socket_proxy_name }} port through UFW" - community.general.ufw: - rule: allow - port: "{{ socket_proxy_listen_port | string }}" - proto: "{{ socket_proxy_ufw_proto }}" - comment: "{{ socket_proxy_ufw_comment | default(socket_proxy_description ~ ' public access', true) }}" - -# Reachability of the upstream over Tailscale. Deliberately non-fatal: the proxy -# is still correctly configured if the backend happens to be down, and this is -# the one check that depends on another machine being up. -- name: "Verify {{ socket_proxy_upstream_host }} is reachable over Tailscale" - ansible.builtin.wait_for: - host: "{{ socket_proxy_upstream_host }}" - port: "{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }}" - timeout: 10 - failed_when: false diff --git a/ansible/roles/socket_proxy/templates/proxy.service.j2 b/ansible/roles/socket_proxy/templates/proxy.service.j2 deleted file mode 100644 index e59d251..0000000 --- a/ansible/roles/socket_proxy/templates/proxy.service.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description={{ socket_proxy_description }} Proxy to {{ socket_proxy_upstream_host }} -{% if socket_proxy_documentation %} -Documentation={{ socket_proxy_documentation }} -{% endif %} -Requires={{ socket_proxy_name }}-proxy.socket -After=network.target - -[Service] -Type=notify -ExecStart=/lib/systemd/systemd-socket-proxyd {{ socket_proxy_upstream_host }}:{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }} -{% if socket_proxy_timeout_stop_sec %} -TimeoutStopSec={{ socket_proxy_timeout_stop_sec }} -{% endif %} -{% if socket_proxy_log_to_journal %} -StandardOutput=journal -StandardError=journal -{% endif %} diff --git a/ansible/roles/socket_proxy/templates/proxy.socket.j2 b/ansible/roles/socket_proxy/templates/proxy.socket.j2 deleted file mode 100644 index 0dc1721..0000000 --- a/ansible/roles/socket_proxy/templates/proxy.socket.j2 +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description={{ socket_proxy_description }} Proxy Socket -{% if socket_proxy_documentation %} -Documentation={{ socket_proxy_documentation }} -{% endif %} - -[Socket] -ListenStream={{ socket_proxy_listen_port }} -{% if socket_proxy_free_bind %} -FreeBind=true -{% endif %} - -[Install] -WantedBy=sockets.target diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml new file mode 100644 index 0000000..c9bd7ca --- /dev/null +++ b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml @@ -0,0 +1,38 @@ +# Bitcoin Knots Configuration Variables + +# Version - REQUIRED: Specify exact version/tag to build +bitcoin_knots_version: "v29.2.knots20251110" # Must specify exact version/tag +bitcoin_knots_version_short: "29.2.knots20251110" # Version without 'v' prefix (for tarball URLs) + +# Directories +bitcoin_knots_dir: /opt/bitcoin-knots +bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" +bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets +bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) +bitcoin_conf_dir: /etc/bitcoin + +# Network +bitcoin_rpc_port: 8332 +bitcoin_p2p_port: 8333 +bitcoin_rpc_bind: "0.0.0.0" + +# Build options +bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores +bitcoin_build_prefix: /usr/local + +# Configuration options +bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) +bitcoin_max_connections: 125 +# dbcache will be calculated as 90% of host RAM automatically in playbook + +# ZMQ Configuration +bitcoin_zmq_enabled: true +bitcoin_zmq_bind: "tcp://0.0.0.0" +bitcoin_zmq_port_rawblock: 28332 +bitcoin_zmq_port_rawtx: 28333 +bitcoin_zmq_port_hashblock: 28334 +bitcoin_zmq_port_hashtx: 28335 + +# Service user +bitcoin_user: bitcoin +bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index d67006b..d818945 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -1,35 +1,916 @@ ---- -# Bitcoin Knots: full node built from source, with PGP signature and SHA256 -# verification of the release tarball. The build is guarded by a binary-exists -# check, so a converged host skips the whole 30-60 minute compile. -# -# The chain lives in bitcoin_large_data_dir (/mnt/knots_data, ~875 GB). Nothing -# here touches it; the only `state: absent` tasks target the build tree under -# /opt/bitcoin-knots and run only when the binary is missing. - name: Build and Deploy Bitcoin Knots from Source - hosts: bitcoin + hosts: knots_box_local become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./bitcoin_knots_vars.yml vars: - # Preserves the push URL this check has been reporting to. The role knows - # nothing about Uptime Kuma — this is just "a URL that accepts a ping". - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_bitcoin-knots/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - bitcoin_knots + bitcoin_repo_url: "https://github.com/bitcoinknots/bitcoin.git" + bitcoin_sigs_base_url: "https://raw.githubusercontent.com/bitcoinknots/guix.sigs/knots" + bitcoin_version_major: "{{ bitcoin_knots_version_short | regex_replace('^(\\d+)\\..*', '\\1') }}" + bitcoin_source_tarball_url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" -- name: Setup public Bitcoin P2P forwarding on the edge host - hosts: edge - become: yes tasks: - - name: Expose Bitcoin P2P through a socket proxy - ansible.builtin.include_role: - name: socket_proxy - vars: - socket_proxy_name: bitcoin-p2p - socket_proxy_description: "Bitcoin P2P" - socket_proxy_listen_port: "{{ hostvars['knots_box_local'].bitcoin_p2p_port }}" - socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" - socket_proxy_documentation: "https://github.com/bitcoin/bitcoin" - socket_proxy_free_bind: true - socket_proxy_timeout_stop_sec: 5 - socket_proxy_log_to_journal: true + - name: Calculate 90% of system RAM for dbcache + set_fact: + bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" + changed_when: false + + - name: Display calculated dbcache value + debug: + msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" + + + - name: Install build dependencies + apt: + name: + - build-essential + - libtool + - autotools-dev + - automake + - pkg-config + - bsdmainutils + - python3 + - python3-pip + - libevent-dev + - libboost-system-dev + - libboost-filesystem-dev + - libboost-test-dev + - libboost-thread-dev + - libboost-chrono-dev + - libboost-program-options-dev + - libboost-dev + - libssl-dev + - libdb-dev + - libminiupnpc-dev + - libzmq3-dev + - libnatpmp-dev + - libsqlite3-dev + - git + - curl + - wget + - cmake + state: present + update_cache: yes + + - name: Create bitcoin group + group: + name: "{{ bitcoin_group }}" + system: yes + state: present + + - name: Create bitcoin user + user: + name: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ bitcoin_data_dir }}" + create_home: yes + state: present + + - name: Create bitcoin-knots directory + file: + path: "{{ bitcoin_knots_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create bitcoin-knots source directory + file: + path: "{{ bitcoin_knots_source_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create bitcoin data directory (for config, logs, wallets) + file: + path: "{{ bitcoin_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + + - name: Create bitcoin large data directory (for blockchain) + file: + path: "{{ bitcoin_large_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + + - name: Create bitcoin config directory + file: + path: "{{ bitcoin_conf_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Check if bitcoind binary already exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_binary_exists + changed_when: false + + - name: Install gnupg for signature verification + apt: + name: gnupg + state: present + when: not bitcoind_binary_exists.stat.exists + + - name: Import Luke Dashjr's Bitcoin Knots signing key + command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 + register: key_import + changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" + when: not bitcoind_binary_exists.stat.exists + failed_when: key_import.rc != 0 + + - name: Display imported key fingerprint + command: gpg --fingerprint 90C8019E36C2E964 + register: key_fingerprint + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Download SHA256SUMS file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + + - name: Download SHA256SUMS.asc signature file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + + - name: Verify PGP signature on SHA256SUMS file + command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS + register: sha256sums_verification + changed_when: false + failed_when: false # Don't fail here - check for 'Good signature' in next task + when: not bitcoind_binary_exists.stat.exists + + + - name: Display SHA256SUMS verification result + debug: + msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" + when: not bitcoind_binary_exists.stat.exists + + - name: Fail if SHA256SUMS signature verification failed + fail: + msg: "SHA256SUMS signature verification failed. Aborting build." + when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) + + - name: Remove any existing tarball to force fresh download + file: + path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Download Bitcoin Knots source tarball + get_url: + url: "{{ bitcoin_source_tarball_url }}" + dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + mode: '0644' + validate_certs: yes + force: yes + when: not bitcoind_binary_exists.stat.exists + + - name: Calculate SHA256 checksum of downloaded tarball + command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + register: tarball_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Extract expected checksum from SHA256SUMS file + shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' + register: expected_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + failed_when: expected_checksum.stdout == "" + + - name: Display checksum comparison + debug: + msg: + - "Expected: {{ expected_checksum.stdout | trim }}" + - "Actual: {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists + + - name: Verify tarball checksum matches SHA256SUMS + fail: + msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] + + - name: Remove existing source directory if it exists (to force fresh extraction) + file: + path: "{{ bitcoin_knots_source_dir }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Remove extracted directory if it exists (from previous runs) + file: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Extract verified source tarball + unarchive: + src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + dest: "{{ bitcoin_knots_dir }}" + remote_src: yes + when: not bitcoind_binary_exists.stat.exists + + - name: Check if extracted directory exists + stat: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + register: extracted_dir_stat + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Rename extracted directory to expected name + command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" + when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists + + - name: Check if CMakeLists.txt exists + stat: + path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" + register: cmake_exists + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Create CMake build directory + file: + path: "{{ bitcoin_knots_source_dir }}/build" + state: directory + mode: '0755' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + + - name: Configure Bitcoin Knots build with CMake + command: > + cmake + -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} + -DBUILD_BITCOIN_WALLET=OFF + -DCMAKE_BUILD_TYPE=Release + -DWITH_ZMQ=ON + .. + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: configure_result + changed_when: true + + - name: Verify CMake enabled ZMQ + shell: | + set -e + cd "{{ bitcoin_knots_source_dir }}/build" + cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: zmq_check + changed_when: false + + - name: Fail if CMakeLists.txt not found + fail: + msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." + when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) + + - name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) + command: cmake --build . -j{{ bitcoin_build_jobs }} + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + async: 3600 + poll: 0 + register: build_result + changed_when: true + + - name: Check build status + async_status: + jid: "{{ build_result.ansible_job_id }}" + register: build_job_result + until: build_job_result.finished + retries: 120 + delay: 60 + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined + + - name: Fail if build failed + fail: + msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) + + - name: Install Bitcoin Knots binaries + command: cmake --install . + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + changed_when: true + + - name: Verify bitcoind binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_installed + changed_when: false + + - name: Verify bitcoin-cli binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" + register: bitcoin_cli_installed + changed_when: false + + - name: Fail if binaries not found + fail: + msg: "Bitcoin Knots binaries not found after installation" + when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists + + - name: Create bitcoin.conf configuration file + copy: + dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" + content: | + # Bitcoin Knots Configuration + # Generated by Ansible + + # Data directory (blockchain storage) + datadir={{ bitcoin_large_data_dir }} + + # RPC Configuration + server=1 + rpcuser={{ bitcoin_rpc_user }} + rpcpassword={{ bitcoin_rpc_password }} + rpcbind={{ bitcoin_rpc_bind }} + rpcport={{ bitcoin_rpc_port }} + rpcallowip=0.0.0.0/0 + + # Network Configuration + listen=1 + port={{ bitcoin_p2p_port }} + maxconnections={{ bitcoin_max_connections }} + + # Performance + dbcache={{ bitcoin_dbcache_mb }} + + # Transaction Index (optional) + {% if bitcoin_enable_txindex %} + txindex=1 + {% endif %} + + # Logging (to journald via systemd) + logtimestamps=1 + printtoconsole=1 + + # ZMQ Configuration + {% if bitcoin_zmq_enabled | default(false) %} + zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} + zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} + zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} + zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} + {% endif %} + + # Security + disablewallet=1 + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0640' + notify: Restart bitcoind + + - name: Create systemd service file for bitcoind + copy: + dest: /etc/systemd/system/bitcoind.service + content: | + [Unit] + Description=Bitcoin Knots daemon + After=network.target + + [Service] + Type=simple + User={{ bitcoin_user }} + Group={{ bitcoin_group }} + ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf + Restart=always + RestartSec=10 + TimeoutStopSec=600 + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + notify: Restart bitcoind + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start bitcoind service + systemd: + name: bitcoind + enabled: yes + state: started + + - name: Wait for bitcoind RPC to be available + uri: + url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" + method: POST + body_format: json + body: + jsonrpc: "1.0" + id: "healthcheck" + method: "getblockchaininfo" + params: [] + user: "{{ bitcoin_rpc_user }}" + password: "{{ bitcoin_rpc_password }}" + status_code: 200 + timeout: 10 + register: rpc_check + until: rpc_check.status == 200 + retries: 30 + delay: 5 + ignore_errors: yes + + - name: Display RPC connection status + debug: + msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + + - name: Create Bitcoin Knots health check and push script + copy: + dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh + content: | + #!/bin/bash + # + # Bitcoin Knots Health Check and Push to Uptime Kuma + # Checks if bitcoind RPC is responding and pushes status to Uptime Kuma + # + + RPC_HOST="{{ bitcoin_rpc_bind }}" + RPC_PORT={{ bitcoin_rpc_port }} + RPC_USER="{{ bitcoin_rpc_user }}" + RPC_PASSWORD="{{ bitcoin_rpc_password }}" + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + # Check if bitcoind RPC is responding + check_bitcoind() { + local response + response=$(curl -s --max-time 30 \ + --user "${RPC_USER}:${RPC_PASSWORD}" \ + --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ + --header 'Content-Type: application/json' \ + "http://${RPC_HOST}:${RPC_PORT}" 2>&1) + + if [ $? -eq 0 ]; then + # Check if response contains a non-null error + # Successful responses have "error": null, failures have "error": {...} + if echo "$response" | grep -q '"error":null\|"error": null'; then + return 0 + else + return 1 + fi + else + return 1 + fi + } + + # Push status to Uptime Kuma + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + + # URL encode spaces in message + local encoded_msg="${msg// /%20}" + + if ! curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then + echo "ERROR: Failed to push to Uptime Kuma" + return 1 + fi + } + + # Main health check + if check_bitcoind; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "bitcoind RPC not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Install curl for health check script + apt: + name: curl + state: present + + - name: Create systemd timer for Bitcoin Knots health check + copy: + dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer + content: | + [Unit] + Description=Bitcoin Knots Health Check Timer + Requires=bitcoind.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for Bitcoin Knots health check + copy: + dest: /etc/systemd/system/bitcoin-knots-healthcheck.service + content: | + [Unit] + Description=Bitcoin Knots Health Check and Push to Uptime Kuma + After=network.target bitcoind.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + + - name: Enable and start Bitcoin Knots health check timer + systemd: + name: bitcoin-knots-healthcheck.timer + enabled: yes + state: started + + - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_bitcoin_knots_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for new monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "Bitcoin Knots" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_bitcoin_knots_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Extract push URL from monitor setup output + set_fact: + uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" + delegate_to: localhost + become: no + when: monitor_setup.stdout is defined + + - name: Display extracted push URL + debug: + msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" + when: uptime_kuma_push_url | default('') != '' + + - name: Set push URL in systemd service environment + lineinfile: + path: /etc/systemd/system/bitcoin-knots-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" + state: present + insertafter: '^\[Service\]' + when: uptime_kuma_push_url | default('') != '' + + - name: Reload systemd daemon after push URL update + systemd: + daemon_reload: yes + when: uptime_kuma_push_url | default('') != '' + + - name: Restart health check timer to pick up new environment + systemd: + name: bitcoin-knots-healthcheck.timer + state: restarted + when: uptime_kuma_push_url | default('') != '' + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_bitcoin_knots_monitor.py + - /tmp/ansible_config.yml + + handlers: + - name: Restart bitcoind + systemd: + name: bitcoind + state: restarted + + +- name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./bitcoin_knots_vars.yml + vars: + bitcoin_tailscale_hostname: "knots-box" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Create Bitcoin P2P proxy socket unit + copy: + dest: /etc/systemd/system/bitcoin-p2p-proxy.socket + content: | + [Unit] + Description=Bitcoin P2P Proxy Socket + + [Socket] + ListenStream={{ bitcoin_p2p_port }} + + [Install] + WantedBy=sockets.target + owner: root + group: root + mode: '0644' + notify: Restart bitcoin-p2p-proxy socket + + - name: Create Bitcoin P2P proxy service unit + copy: + dest: /etc/systemd/system/bitcoin-p2p-proxy.service + content: | + [Unit] + Description=Bitcoin P2P Proxy to {{ bitcoin_tailscale_hostname }} + Requires=bitcoin-p2p-proxy.socket + After=network.target + + [Service] + Type=notify + ExecStart=/lib/systemd/systemd-socket-proxyd {{ bitcoin_tailscale_hostname }}:{{ bitcoin_p2p_port }} + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Bitcoin P2P proxy socket + systemd: + name: bitcoin-p2p-proxy.socket + enabled: yes + state: started + + - name: Allow Bitcoin P2P port through UFW + ufw: + rule: allow + port: "{{ bitcoin_p2p_port | string }}" + proto: tcp + comment: "Bitcoin P2P public access" + + - name: Verify connectivity to knots-box via Tailscale + wait_for: + host: "{{ bitcoin_tailscale_hostname }}" + port: "{{ bitcoin_p2p_port }}" + timeout: 10 + ignore_errors: yes + + - name: Display public endpoint + debug: + msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public P2P + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_bitcoin_p2p_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_bitcoin_p2p_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for TCP monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_bitcoin_p2p_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ bitcoin_p2p_port }} + monitor_name: "Bitcoin Knots P2P Public" + mode: '0644' + + - name: Run Uptime Kuma TCP monitor setup + command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up TCP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_bitcoin_p2p_tcp_monitor.py + - /tmp/ansible_bitcoin_p2p_config.yml + + handlers: + - name: Restart bitcoin-p2p-proxy socket + systemd: + name: bitcoin-p2p-proxy.socket + state: restarted diff --git a/ansible/services/caddy_playbook.yml b/ansible/services/caddy_playbook.yml index 29e74b2..de98c8f 100644 --- a/ansible/services/caddy_playbook.yml +++ b/ansible/services/caddy_playbook.yml @@ -1,5 +1,5 @@ - name: Install and configure Caddy on Debian 12 - hosts: caddy + hosts: vps become: yes tasks: diff --git a/ansible/services/datum-gateway/datum_gateway_vars.yml b/ansible/services/datum-gateway/datum_gateway_vars.yml new file mode 100644 index 0000000..48bb5b7 --- /dev/null +++ b/ansible/services/datum-gateway/datum_gateway_vars.yml @@ -0,0 +1,43 @@ +# DATUM Gateway Configuration Variables +# https://github.com/OCEAN-xyz/datum_gateway + +# Version - pin to a specific tag +datum_gateway_version: "v0.4.1beta" + +# Directories +datum_gateway_dir: /opt/datum-gateway +datum_gateway_source_dir: "{{ datum_gateway_dir }}/source" +datum_gateway_config_dir: /etc/datum-gateway +datum_gateway_log_dir: /var/log/datum-gateway + +# Binary +datum_gateway_bin_path: /usr/local/bin/datum_gateway + +# Ports +datum_gateway_stratum_port: 23334 # Miners connect here via Stratum v1 +datum_gateway_api_port: 7152 # Web dashboard / API + +# Stratum settings +datum_vardiff_min: 524288 # Minimum share difficulty (must be power of 2; OCEAN floor overrides if higher) + +# Service user +datum_gateway_user: datum +datum_gateway_group: datum + +# Build options +datum_gateway_build_jobs: 4 + +# Bitcoin node connection +# The gateway runs on the same host as Bitcoin Knots so localhost RPC works. +# datum_bitcoin_rpc_url should include http:// and port. +datum_bitcoin_rpc_url: "http://127.0.0.1:8332" +# Note: bitcoin_rpc_user and bitcoin_rpc_password come from infra_secrets.yml + +# Mining config +datum_coinbase_tag_primary: "DATUM" +datum_coinbase_tag_secondary: "BY ORDER OF BIP110" + +datum_pool_pass_workers: true +datum_pool_pass_full_users: true +datum_pooled_mining_only: true + diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 2e88a27..f33ded7 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -1,58 +1,859 @@ --- -# DATUM Gateway: solo/pooled mining gateway, built from source on knots-box. +# DATUM Gateway Deployment Playbook # -# This is HALF OF A SYSTEM. The Bitcoin Knots node on the same host feeds it via -# `blocknotify=killall -USR1 datum_gateway` in bitcoin.conf (see -# roles/bitcoin_knots/README.md). Changing either config means thinking about -# both. Interrupting Stratum costs mining shares, so check for connected miners -# before restarting: -# ss -tn state established '( sport = :23334 )' -- name: Deploy DATUM Gateway on the bitcoin host - hosts: bitcoin +# Deploys DATUM Gateway (https://github.com/OCEAN-xyz/datum_gateway) on the +# Bitcoin Knots host so it has direct localhost RPC access to bitcoind. +# +# What this does: +# 1. Installs build deps and compiles datum_gateway from source +# 2. Creates a dedicated system user and config/log directories +# 3. Writes /etc/datum-gateway/config.json from vars/secrets +# 4. Patches bitcoin.conf with the required blockmaxsize/blocknotify lines +# 5. Creates and enables a systemd service +# 6. Creates a push-monitor health check script + systemd timer +# 7. Registers a push monitor in Uptime Kuma +# +# Separate play: adds a Caddy reverse proxy on vipy for the dashboard. +# +# Stratum port (23334) is bound on knots_box_local. Expose it to miners via +# a firewall rule, Tailscale, or a socket proxy on vipy — not handled here. +# +# Required secrets in infra_secrets.yml: +# datum_mining_address - Bitcoin address for block rewards +# datum_gateway_admin_password - Password for the /api admin endpoint +# bitcoin_rpc_user - Shared with the bitcoin-knots deployment +# bitcoin_rpc_password - Shared with the bitcoin-knots deployment + +- name: Deploy DATUM Gateway on knots_box_local + hosts: knots_box_local become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./datum_gateway_vars.yml vars: - # Preserves the push URL this check reports to. The role knows nothing about - # Uptime Kuma — this is just "a URL that accepts a ping". - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_datum-gateway/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - datum_gateway + datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" + datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" -- name: Configure Caddy reverse proxy for the DATUM Gateway dashboard on the edge host - hosts: edge - become: yes tasks: - - name: Publish the DATUM Gateway dashboard through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: datum-gateway - caddy_site_domain: "{{ subdomains.datum_gateway }}.{{ root_domain }}" - caddy_site_upstream: "{{ hostvars['knots_box_local'].ansible_host }}:{{ hostvars['knots_box_local'].datum_gateway_api_port }}" - caddy_site_resolvers: "100.100.100.100" - caddy_site_basic_auth: - - user: "{{ datum_dashboard_username }}" - hash: "{{ datum_dashboard_password_hash }}" + # =========================================== + # Build Dependencies + # =========================================== + - name: Install DATUM Gateway build dependencies + apt: + name: + - cmake + - build-essential + - git + - libjansson-dev + - libmicrohttpd-dev + - libsodium-dev + - libcurl4-openssl-dev + # Runtime-only (netcat for health check) + - netcat-openbsd + state: present + update_cache: yes - # The role validates its own site fragment; this re-validates the whole - # assembled Caddyfile, which is the only thing that catches a conflict - # between two sites. - - name: Validate the assembled Caddyfile - ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + # =========================================== + # System User and Directories + # =========================================== + - name: Create datum system user + user: + name: "{{ datum_gateway_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ datum_gateway_dir }}" + create_home: no + comment: "DATUM Gateway" + + - name: Create DATUM Gateway directories + file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ datum_gateway_group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ datum_gateway_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_source_dir }}", owner: root, mode: "0755" } + - { path: "{{ datum_gateway_config_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + - { path: "{{ datum_gateway_log_dir }}", owner: "{{ datum_gateway_user }}", mode: "0750" } + + # =========================================== + # Build from Source + # =========================================== + - name: Clone DATUM Gateway repository at {{ datum_gateway_version }} + git: + repo: https://github.com/OCEAN-xyz/datum_gateway.git + dest: "{{ datum_gateway_source_dir }}" + version: "{{ datum_gateway_version }}" + force: yes + register: git_clone + + - name: Configure cmake build + command: cmake . -DCMAKE_BUILD_TYPE=Release + args: + chdir: "{{ datum_gateway_source_dir }}" + + - name: Compile datum_gateway + command: make -j{{ datum_gateway_build_jobs }} + args: + chdir: "{{ datum_gateway_source_dir }}" + + - name: Install datum_gateway binary + copy: + src: "{{ datum_gateway_source_dir }}/datum_gateway" + dest: "{{ datum_gateway_bin_path }}" + remote_src: yes + owner: root + group: root + mode: "0755" + notify: Restart datum-gateway + + # =========================================== + # Configuration + # =========================================== + - name: Write DATUM Gateway config.json + copy: + dest: "{{ datum_gateway_config_dir }}/config.json" + content: | + { + "bitcoind": { + "rpcuser": "{{ bitcoin_rpc_user }}", + "rpcpassword": "{{ bitcoin_rpc_password }}", + "rpcurl": "{{ datum_bitcoin_rpc_url }}", + "notify_fallback": true + }, + "stratum": { + "listen_port": {{ datum_gateway_stratum_port }}, + "vardiff_min": {{ datum_vardiff_min }} + }, + "mining": { + "pool_address": "{{ datum_mining_address }}", + "coinbase_tag_primary": "{{ datum_coinbase_tag_primary }}", + "coinbase_tag_secondary": "{{ datum_coinbase_tag_secondary }}" + }, + "api": { + "admin_password": "{{ datum_gateway_admin_password }}", + "listen_port": {{ datum_gateway_api_port }}, + "modify_conf": false + }, + "logger": { + "log_to_console": true, + "log_to_file": true, + "log_file": "{{ datum_gateway_log_dir }}/datum_gateway.log", + "log_rotate_daily": true, + "log_level_console": 2, + "log_level_file": 1 + }, + "datum": { + "pool_pass_workers": {{ datum_pool_pass_workers | lower }}, + "pool_pass_full_users": {{ datum_pool_pass_full_users | lower }}, + "pooled_mining_only": {{ datum_pooled_mining_only | lower }} + } + } + owner: "{{ datum_gateway_user }}" + group: "{{ datum_gateway_group }}" + mode: "0640" + notify: Restart datum-gateway + + # =========================================== + # Systemd Service + # =========================================== + - name: Create datum-gateway systemd service + copy: + dest: /etc/systemd/system/datum-gateway.service + content: | + [Unit] + Description=DATUM Gateway - Bitcoin Mining Gateway + Documentation=https://github.com/OCEAN-xyz/datum_gateway + After=network.target bitcoind.service + Wants=bitcoind.service + + [Service] + User={{ datum_gateway_user }} + Group={{ datum_gateway_group }} + Type=simple + ExecStart={{ datum_gateway_bin_path }} --config {{ datum_gateway_config_dir }}/config.json + Restart=on-failure + RestartSec=10 + StandardOutput=journal + StandardError=journal + + # Prevent config from being read by other users + ReadWritePaths={{ datum_gateway_log_dir }} + ReadOnlyPaths={{ datum_gateway_config_dir }} + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + notify: Restart datum-gateway + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start datum-gateway + systemd: + name: datum-gateway + enabled: yes + state: started + + # =========================================== + # Health Check Script + Systemd Timer + # =========================================== + - name: Create DATUM Gateway health check script + copy: + dest: /usr/local/bin/datum-gateway-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + STRATUM_PORT={{ datum_gateway_stratum_port }} + + check_datum() { + # Service must be active and stratum port must be listening + systemctl is-active --quiet datum-gateway && \ + nc -z 127.0.0.1 "${STRATUM_PORT}" + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_datum; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "DATUM Gateway not responding" + exit 1 + fi + owner: root + group: root + mode: "0755" + + - name: Create datum-gateway health check systemd service + copy: + dest: /etc/systemd/system/datum-gateway-healthcheck.service + content: | + [Unit] + Description=DATUM Gateway Health Check + After=network.target datum-gateway.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/datum-gateway-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + + - name: Create datum-gateway health check systemd timer + copy: + dest: /etc/systemd/system/datum-gateway-healthcheck.timer + content: | + [Unit] + Description=DATUM Gateway Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: "0644" + + - name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + + - name: Enable and start datum-gateway health check timer + systemd: + name: datum-gateway-healthcheck.timer + enabled: yes + state: started + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for DATUM Gateway + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_datum_gateway_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + + try: + import socketio.exceptions + except ImportError: + pass + + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_datum_gateway_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + push_url = None + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + else: + print(f"Creating push monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=90, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + # socketio timeout: add_monitor may have succeeded server-side + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + + api.disconnect() + + if push_url: + print(f"PUSH_URL={push_url}") + with open('/tmp/datum_gateway_push_url.txt', 'w') as f: + f.write(push_url) + + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for push monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_datum_gateway_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "DATUM Gateway" + mode: "0644" + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_datum_gateway_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URL from file + slurp: + src: /tmp/datum_gateway_push_url.txt + delegate_to: localhost + become: no + register: push_url_file + ignore_errors: yes + + - name: Parse push URL + set_fact: + datum_push_url: "{{ push_url_file.content | b64decode | trim }}" + when: push_url_file.content is defined + + - name: Update health check service with push URL + lineinfile: + path: /etc/systemd/system/datum-gateway-healthcheck.service + regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" + line: "Environment=UPTIME_KUMA_PUSH_URL={{ datum_push_url }}" + when: datum_push_url is defined + notify: Restart datum-gateway health check timer + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_datum_gateway_monitor.py + - /tmp/ansible_datum_gateway_config.yml + - /tmp/datum_gateway_push_url.txt + + handlers: + - name: Restart datum-gateway + systemd: + name: datum-gateway + state: restarted + daemon_reload: yes + + - name: Restart datum-gateway health check timer + systemd: + name: datum-gateway-healthcheck.timer + state: restarted + daemon_reload: yes + + +# =========================================== +# Caddy Reverse Proxy for DATUM Dashboard (on vipy) +# =========================================== +- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on vipy + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./datum_gateway_vars.yml + vars: + datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" + datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: "import sites-enabled/*" + insertafter: EOF + state: present + backup: yes + create: yes + mode: "0644" + + - name: Create Caddy reverse proxy config for DATUM Gateway dashboard + copy: + dest: "{{ caddy_sites_dir }}/datum-gateway.conf" + content: | + {{ datum_gateway_domain }} { + basic_auth { + {{ datum_dashboard_username }} {{ datum_dashboard_password_hash }} + } + reverse_proxy knots-box:{{ datum_gateway_api_port }} { + # Resolve via Tailscale MagicDNS + transport http { + resolvers 100.100.100.100 + } + } + } + owner: root + group: root + mode: "0644" + + - name: Validate Caddy config + command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + register: caddy_validate changed_when: false -- name: Setup public Stratum port forwarding on the edge host - hosts: edge + - name: Display Caddy validation output + debug: + msg: "{{ caddy_validate.stdout_lines + caddy_validate.stderr_lines }}" + + - name: Reload Caddy + command: systemctl reload caddy + register: caddy_reload + + - name: Display Caddy reload output + debug: + msg: "{{ caddy_reload.stdout_lines + caddy_reload.stderr_lines }}" + + - name: Display DATUM Gateway dashboard URL + debug: + msg: "DATUM Gateway dashboard: https://{{ datum_gateway_domain }}" + + # =========================================== + # Uptime Kuma HTTP Monitor for Public Dashboard + # =========================================== + - name: Create Uptime Kuma HTTP monitor setup script for DATUM dashboard + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_datum_http_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_datum_http_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + else: + print(f"Creating HTTP monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for HTTP monitor + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_datum_http_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ datum_gateway_domain }}" + monitor_name: "DATUM Gateway Dashboard" + mode: "0644" + + - name: Run Uptime Kuma HTTP monitor setup + command: python3 /tmp/setup_datum_http_monitor.py + delegate_to: localhost + become: no + register: http_monitor_setup + changed_when: "'SUCCESS' in http_monitor_setup.stdout" + ignore_errors: yes + + - name: Display HTTP monitor setup output + debug: + msg: "{{ http_monitor_setup.stdout_lines }}" + when: http_monitor_setup.stdout is defined + + - name: Clean up HTTP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_datum_http_monitor.py + - /tmp/ansible_datum_http_config.yml + + +# =========================================== +# Stratum Port Forwarding on vipy via systemd-socket-proxyd +# Miners connect to vipy:23334; traffic is forwarded to knots-box:23334 +# over the Tailscale network, matching the Bitcoin P2P proxy pattern. +# =========================================== +- name: Setup public Stratum port forwarding on vipy via systemd-socket-proxyd + hosts: vipy become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./datum_gateway_vars.yml + vars: + datum_tailscale_hostname: "knots-box" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + tasks: - - name: Expose the DATUM Stratum port through a socket proxy - ansible.builtin.include_role: - name: socket_proxy - vars: - socket_proxy_name: datum-stratum - socket_proxy_description: "DATUM Stratum" - socket_proxy_listen_port: "{{ hostvars['knots_box_local'].datum_gateway_stratum_port }}" - socket_proxy_upstream_host: "{{ hostvars['knots_box_local'].ansible_host }}" - # Matches the UFW comment already on the edge host; the derived default - # would say "DATUM Stratum" and rewrite the rule. - socket_proxy_ufw_comment: "DATUM Gateway Stratum public access" + - name: Create Stratum proxy socket unit + copy: + dest: /etc/systemd/system/datum-stratum-proxy.socket + content: | + [Unit] + Description=DATUM Stratum Proxy Socket + + [Socket] + ListenStream={{ datum_gateway_stratum_port }} + + [Install] + WantedBy=sockets.target + owner: root + group: root + mode: "0644" + notify: Restart datum-stratum-proxy socket + + - name: Create Stratum proxy service unit + copy: + dest: /etc/systemd/system/datum-stratum-proxy.service + content: | + [Unit] + Description=DATUM Stratum Proxy to {{ datum_tailscale_hostname }} + Requires=datum-stratum-proxy.socket + After=network.target + + [Service] + Type=notify + ExecStart=/lib/systemd/systemd-socket-proxyd {{ datum_tailscale_hostname }}:{{ datum_gateway_stratum_port }} + owner: root + group: root + mode: "0644" + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Stratum proxy socket + systemd: + name: datum-stratum-proxy.socket + enabled: yes + state: started + + - name: Allow Stratum port through UFW + ufw: + rule: allow + port: "{{ datum_gateway_stratum_port | string }}" + proto: tcp + comment: "DATUM Gateway Stratum public access" + + - name: Verify connectivity to knots-box Stratum via Tailscale + wait_for: + host: "{{ datum_tailscale_hostname }}" + port: "{{ datum_gateway_stratum_port }}" + timeout: 10 + ignore_errors: yes + + - name: Display public Stratum endpoint + debug: + msg: "DATUM Stratum public endpoint: {{ ansible_host }}:{{ datum_gateway_stratum_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public Stratum + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Stratum + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_datum_stratum_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_datum_stratum_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for Stratum TCP monitor + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_datum_stratum_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ datum_gateway_stratum_port }} + monitor_name: "DATUM Stratum (public)" + mode: "0644" + + - name: Run Uptime Kuma TCP monitor setup + command: python3 /tmp/setup_datum_stratum_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up Stratum TCP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_datum_stratum_tcp_monitor.py + - /tmp/ansible_datum_stratum_config.yml + + handlers: + - name: Restart datum-stratum-proxy socket + systemd: + name: datum-stratum-proxy.socket + state: restarted diff --git a/ansible/services/forgejo-runner/SETUP.md b/ansible/services/forgejo-runner/SETUP.md index 5cb2a3f..a66d295 100644 --- a/ansible/services/forgejo-runner/SETUP.md +++ b/ansible/services/forgejo-runner/SETUP.md @@ -25,6 +25,4 @@ ansible-playbook ansible/services/forgejo-runner/deploy_forgejo_runner_playbook. 1. On the VM: `systemctl status forgejo-runner` should show active 2. In Forgejo: **Site Administration** > **Actions** > **Runners** should show the runner as online -3. ~~In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings~~ - *(Uptime Kuma was decommissioned 2026-09-11 — this check no longer applies. The - healthcheck timer still runs on the host but pushes nowhere. See `archive/uptime_kuma/`.)* +3. In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index 04081ff..a194178 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -1,13 +1,392 @@ ---- - name: Install Forgejo Runner on Debian 13 - hosts: ci_runner + hosts: forgejo_runner_local become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./forgejo_runner_vars.yml vars: - # Preserves the push URL this host has been reporting to all along, so the - # move to a role changes no behaviour. The role itself knows nothing about - # Uptime Kuma — this is just "a URL that accepts a ping", and whatever - # replaces it sets the same variable. - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_forgejo-runner/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - forgejo_runner + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + healthcheck_interval_seconds: 60 + healthcheck_timeout_seconds: 90 + healthcheck_retries: 1 + healthcheck_script_dir: /opt/forgejo-runner-healthcheck + healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" + healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" + healthcheck_service_name: forgejo-runner-healthcheck + + tasks: + # ── 1. Assert Docker is available ────────────────────────────────── + - name: Check if Docker is installed + command: docker --version + register: docker_check + changed_when: false + failed_when: docker_check.rc != 0 + + - name: Fail if Docker is not available + assert: + that: + - docker_check.rc == 0 + fail_msg: > + Docker is not installed or not in PATH. + Please install Docker before running this playbook. + + # ── 2. Download forgejo-runner binary ────────────────────────────── + - name: Download forgejo-runner binary + get_url: + url: "{{ forgejo_runner_url }}" + dest: "{{ forgejo_runner_bin_path }}" + mode: '0755' + + # ── 3. Create runner system user ─────────────────────────────────── + - name: Create runner system user + user: + name: "{{ forgejo_runner_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ forgejo_runner_dir }}" + create_home: no + groups: docker + append: yes + comment: 'Forgejo Runner' + + # ── 4. Create working directory ──────────────────────────────────── + - name: Create forgejo-runner working directory + file: + path: "{{ forgejo_runner_dir }}" + state: directory + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + mode: '0750' + + # ── 5. Generate default config ───────────────────────────────────── + - name: Check if config already exists + stat: + path: "{{ forgejo_runner_config_path }}" + register: config_stat + + - name: Generate default config + shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not config_stat.stat.exists + + - name: Set config file ownership + file: + path: "{{ forgejo_runner_config_path }}" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not config_stat.stat.exists + + # ── 6. Register runner ───────────────────────────────────────────── + - name: Check if runner is already registered + stat: + path: "{{ forgejo_runner_dir }}/.runner" + register: runner_stat + + - name: Register runner with Forgejo instance + command: > + {{ forgejo_runner_bin_path }} register --no-interactive + --instance {{ forgejo_instance_url }} + --token {{ forgejo_runner_registration_token }} + --name forgejo-runner-box + --labels "{{ forgejo_runner_labels }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not runner_stat.stat.exists + + - name: Set runner registration file ownership + file: + path: "{{ forgejo_runner_dir }}/.runner" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not runner_stat.stat.exists + + # ── 7. Create systemd service ────────────────────────────────────── + - name: Create forgejo-runner systemd service + copy: + dest: /etc/systemd/system/forgejo-runner.service + content: | + [Unit] + Description=Forgejo Runner + Documentation=https://forgejo.org/docs/latest/admin/actions/ + After=docker.service + Requires=docker.service + + [Service] + Type=simple + User={{ forgejo_runner_user }} + Group={{ forgejo_runner_user }} + WorkingDirectory={{ forgejo_runner_dir }} + ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} + Restart=on-failure + RestartSec=10 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + # ── 8. Reload systemd, enable and start ──────────────────────────── + - name: Reload systemd + systemd: + daemon_reload: yes + + - name: Enable and start forgejo-runner service + systemd: + name: forgejo-runner + enabled: yes + state: started + + # ── 9. Verify runner is active ───────────────────────────────────── + - name: Verify forgejo-runner is active + command: systemctl is-active forgejo-runner + register: runner_active + changed_when: false + + - name: Assert runner is running + assert: + that: + - runner_active.stdout == "active" + fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" + + # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── + - name: Create Uptime Kuma push monitor setup script + copy: + dest: /tmp/setup_forgejo_runner_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma push monitor setup + command: > + {{ ansible_playbook_python }} + /tmp/setup_forgejo_runner_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "services" + "forgejo-runner-healthcheck" + "Forgejo Runner healthcheck - ping every {{ healthcheck_interval_seconds }}s" + "{{ healthcheck_timeout_seconds }}" + "{{ healthcheck_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL + set_fact: + uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Create healthcheck script directory + file: + path: "{{ healthcheck_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create forgejo-runner healthcheck script + copy: + dest: "{{ healthcheck_script_path }}" + content: | + #!/bin/bash + + # Forgejo Runner Healthcheck Script + # Checks if forgejo-runner is active and pings Uptime Kuma on success + + LOG_FILE="{{ healthcheck_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_push_url }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + main() { + if systemctl is-active --quiet forgejo-runner; then + log_message "forgejo-runner is active, sending ping" + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=forgejo-runner%20is%20active" 2>&1) + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Ping sent successfully (HTTP $http_code)" + else + log_message "ERROR: Failed to send ping (HTTP $http_code)" + exit 1 + fi + else + log_message "ERROR: forgejo-runner is not active" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create healthcheck systemd service + copy: + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" + content: | + [Unit] + Description=Forgejo Runner Healthcheck + After=network.target + + [Service] + Type=oneshot + ExecStart={{ healthcheck_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create healthcheck systemd timer + copy: + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" + content: | + [Unit] + Description=Run Forgejo Runner Healthcheck every minute + Requires={{ healthcheck_service_name }}.service + + [Timer] + OnBootSec=30sec + OnUnitActiveSec={{ healthcheck_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd for healthcheck units + systemd: + daemon_reload: yes + + - name: Enable and start healthcheck timer + systemd: + name: "{{ healthcheck_service_name }}.timer" + enabled: yes + state: started + + - name: Test healthcheck script + command: "{{ healthcheck_script_path }}" + register: healthcheck_test + changed_when: false + + - name: Verify healthcheck script works + assert: + that: + - healthcheck_test.rc == 0 + fail_msg: "Healthcheck script failed to execute properly" + + - name: Display deployment summary + debug: + msg: | + Forgejo Runner deployed successfully! + + Runner Name: forgejo-runner-box + Instance: {{ forgejo_instance_url }} + Working Directory: {{ forgejo_runner_dir }} + Service: forgejo-runner.service ({{ runner_active.stdout }}) + + Healthcheck Monitor: forgejo-runner-healthcheck + Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s + Timeout: {{ healthcheck_timeout_seconds }}s + + - name: Clean up temporary monitor setup script + file: + path: /tmp/setup_forgejo_runner_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/services/forgejo-runner/forgejo_runner_vars.yml b/ansible/services/forgejo-runner/forgejo_runner_vars.yml new file mode 100644 index 0000000..e618fca --- /dev/null +++ b/ansible/services/forgejo-runner/forgejo_runner_vars.yml @@ -0,0 +1,9 @@ +forgejo_runner_version: "6.3.1" +forgejo_runner_arch: "linux-amd64" +forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" +forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" +forgejo_runner_user: "runner" +forgejo_runner_dir: "/opt/forgejo-runner" +forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" +forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" +forgejo_instance_url: "https://forgejo.contrapeso.xyz" diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index f908586..e17d08f 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -1,11 +1,16 @@ - name: Install Forgejo on Debian 12 with Caddy reverse proxy - hosts: edge + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./forgejo_vars.yml vars: forgejo_subdomain: "{{ subdomains.forgejo }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" forgejo_domain: "{{ forgejo_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure required packages are installed @@ -83,10 +88,138 @@ enabled: yes state: started - - name: Publish Forgejo through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: forgejo - caddy_site_domain: "{{ forgejo_domain }}" - caddy_site_upstream: "localhost:{{ forgejo_port }}" + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy reverse proxy configuration for forgejo + copy: + dest: "{{ caddy_sites_dir }}/forgejo.conf" + content: | + {{ forgejo_domain }} { + reverse_proxy localhost:{{ forgejo_port }} + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for Forgejo + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_forgejo_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ forgejo_domain }}/api/healthz" + monitor_name: "Forgejo" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_forgejo_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_forgejo_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/forgejo/forgejo_vars.yml b/ansible/services/forgejo/forgejo_vars.yml index 7bba0ed..0bbb5a5 100644 --- a/ansible/services/forgejo/forgejo_vars.yml +++ b/ansible/services/forgejo/forgejo_vars.yml @@ -9,12 +9,15 @@ forgejo_url: "https://codeberg.org/forgejo/forgejo/releases/download/v{{ forgejo forgejo_bin_path: "/usr/local/bin/forgejo" forgejo_user: "git" -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "{{ groups['edge'] | first }}" +remote_host_name: "vipy" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/forgejo-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/forgejo_backup.sh" diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml deleted file mode 100644 index 5329e2a..0000000 --- a/ansible/services/forgejo/setup_backup_forgejo.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -# Forgejo backup: dumps locally on vipy, encrypted with age. -# -# The biggest artefact in the estate (~2.7 G) and the reason retention here is -# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14. -# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the -# old job did the same. -- name: Configure the Forgejo backup on the edge host - hosts: edge - become: yes - vars_files: - - ./forgejo_vars.yml - - tasks: - - name: Ensure Forgejo dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: forgejo - backup_source_description: "Forgejo" - backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo" - backup_source_stop_service: forgejo - backup_source_retention_days: 2 - backup_source_on_calendar: "*-*-* 02:30:00" - # Reported to Gatus as backup-dump_forgejo. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_forgejo/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml new file mode 100644 index 0000000..b90f0fb --- /dev/null +++ b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml @@ -0,0 +1,122 @@ +--- +- name: Configure local backup for Forgejo from remote + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./forgejo_vars.yml + vars: + remote_data_path: "{{ forgejo_data_dir }}" + remote_config_path: "{{ forgejo_config_dir }}" + forgejo_service_name: "forgejo" + gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" + gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" + + tasks: + - name: Debug Forgejo backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "remote_config_path='{{ remote_config_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + - "gpg_recipient={{ gpg_recipient }}" + - "gpg_key_id={{ gpg_key_id }}" + + - name: Ensure local backup directory exists + ansible.builtin.file: + path: "{{ local_backup_dir }}" + state: directory + mode: "0755" + + - name: Ensure ~/.local/bin exists + ansible.builtin.file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: "0755" + + - name: Create Forgejo backup script + ansible.builtin.copy: + dest: "{{ backup_script_path }}" + mode: "0750" + content: | + #!/bin/bash + set -euo pipefail + + if [ -z "{{ gpg_recipient }}" ]; then + echo "GPG recipient is not configured. Aborting." + exit 1 + fi + + TIMESTAMP=$(date +'%Y-%m-%d') + ENCRYPTED_BACKUP="{{ local_backup_dir }}/forgejo-backup-$TIMESTAMP.tar.gz.gpg" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + echo "Stopping Forgejo service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop {{ forgejo_service_name }}" + + echo "Creating encrypted backup archive..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo tar -czf - {{ remote_data_path }} {{ remote_config_path }}" | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" + + echo "Starting Forgejo service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start {{ forgejo_service_name }}" + + # Rotate old backups (keep 3 days) + # Calculate cutoff date (3 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '3 days ago' +'%Y-%m-%d') + for backup_file in "{{ local_backup_dir }}"/forgejo-backup-*.tar.gz.gpg; do + if [ -f "$backup_file" ]; then + # Extract date from filename: forgejo-backup-YYYY-MM-DD.tar.gz.gpg + file_date=$(basename "$backup_file" | sed -n 's/forgejo-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$backup_file" + fi + fi + done + + echo "Backup completed successfully" + + - name: Ensure cronjob for Forgejo backup exists + ansible.builtin.cron: + name: "Forgejo backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run Forgejo backup script to create initial backup + ansible.builtin.command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check if backup file exists + stat: + path: "{{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg" + register: backup_file_stat + + - name: Verify backup file exists + assert: + that: + - backup_file_stat.stat.exists + - backup_file_stat.stat.isreg + fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" + success_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists" + + - name: Verify backup file is not empty + assert: + that: + - backup_file_stat.stat.size > 0 + fail_msg: "Backup file {{ local_backup_dir }}/forgejo-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" + success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 1b7b963..1255cd6 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -1,27 +1,716 @@ ---- -# Fulcrum: Electrum server indexing the Bitcoin Knots node. -# The index takes days to rebuild, so nothing here touches its data directory -# beyond asserting that it exists. - name: Deploy Fulcrum Electrum Server - hosts: electrum + hosts: fulcrum_box_local become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./fulcrum_vars.yml vars: - # Preserves the push URL this check has been configured with. The role knows - # nothing about Uptime Kuma — this is just "a URL that accepts a ping". - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_fulcrum/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - fulcrum + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" -- name: Setup public Fulcrum SSL forwarding on the edge host - hosts: edge - become: yes tasks: - - name: Expose Fulcrum SSL through a socket proxy - ansible.builtin.include_role: - name: socket_proxy - vars: - socket_proxy_name: fulcrum-ssl - socket_proxy_description: "Fulcrum SSL" - socket_proxy_listen_port: "{{ hostvars['fulcrum_box_local'].fulcrum_ssl_port }}" - socket_proxy_upstream_host: "{{ hostvars['fulcrum_box_local'].ansible_host }}" + - name: Calculate 75% of system RAM for db_mem + set_fact: + fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" + changed_when: false + + - name: Display calculated db_mem value + debug: + msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" + + - name: Display Fulcrum version to install + debug: + msg: "Installing Fulcrum version {{ fulcrum_version }}" + + - name: Install required packages + apt: + name: + - curl + - wget + - openssl + state: present + update_cache: yes + + - name: Create fulcrum group + group: + name: "{{ fulcrum_group }}" + system: yes + state: present + + - name: Create fulcrum user + user: + name: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + system: yes + shell: /usr/sbin/nologin + home: /home/{{ fulcrum_user }} + create_home: yes + state: present + + - name: Create Fulcrum database directory (heavy data on special mount) + file: + path: "{{ fulcrum_db_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Create Fulcrum config directory + file: + path: "{{ fulcrum_config_dir }}" + state: directory + owner: root + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Create Fulcrum lib directory (for banner and other data files) + file: + path: "{{ fulcrum_lib_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + + # =========================================== + # SSL Certificate Generation + # =========================================== + - name: Check if SSL certificate already exists + stat: + path: "{{ fulcrum_ssl_cert_path }}" + register: fulcrum_ssl_cert_exists + when: fulcrum_ssl_enabled | default(false) + + - name: Generate self-signed SSL certificate for Fulcrum + command: > + openssl req -x509 -newkey rsa:4096 + -keyout {{ fulcrum_ssl_key_path }} + -out {{ fulcrum_ssl_cert_path }} + -sha256 -days {{ fulcrum_ssl_cert_days }} + -nodes + -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" + args: + creates: "{{ fulcrum_ssl_cert_path }}" + when: fulcrum_ssl_enabled | default(false) + notify: Restart fulcrum + + - name: Set SSL certificate permissions + file: + path: "{{ fulcrum_ssl_cert_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) + + - name: Set SSL key permissions + file: + path: "{{ fulcrum_ssl_key_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0600' + when: fulcrum_ssl_enabled | default(false) + + - name: Check if Fulcrum binary already exists + stat: + path: "{{ fulcrum_binary_path }}" + register: fulcrum_binary_exists + changed_when: false + + - name: Download Fulcrum binary tarball + get_url: + url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + mode: '0644' + when: not fulcrum_binary_exists.stat.exists + + - name: Extract Fulcrum binary + unarchive: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp" + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + + - name: Install Fulcrum binary + copy: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" + dest: "{{ fulcrum_binary_path }}" + owner: root + group: root + mode: '0755' + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + + - name: Verify Fulcrum binary installation + command: "{{ fulcrum_binary_path }} --version" + register: fulcrum_version_check + changed_when: false + + - name: Display Fulcrum version + debug: + msg: "{{ fulcrum_version_check.stdout_lines }}" + + - name: Create Fulcrum banner file + copy: + dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" + content: | + counterinfra + + PER ASPERA AD ASTRA + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + + - name: Create Fulcrum configuration file + copy: + dest: "{{ fulcrum_config_dir }}/fulcrum.conf" + content: | + # Fulcrum Configuration + # Generated by Ansible + + # Bitcoin Core/Knots RPC settings + bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} + rpcuser = {{ bitcoin_rpc_user }} + rpcpassword = {{ bitcoin_rpc_password }} + + # Fulcrum server general settings + datadir = {{ fulcrum_db_dir }} + tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} + peering = {{ 'true' if fulcrum_peering else 'false' }} + zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} + + # SSL/TLS Configuration + {% if fulcrum_ssl_enabled | default(false) %} + ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} + cert = {{ fulcrum_ssl_cert_path }} + key = {{ fulcrum_ssl_key_path }} + {% endif %} + + # Anonymize client IP addresses and TxIDs in logs + anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} + + # Max RocksDB Memory in MiB + db_mem = {{ fulcrum_db_mem_mb }}.0 + + # Banner + banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0640' + notify: Restart fulcrum + + - name: Create systemd service file for Fulcrum + copy: + dest: /etc/systemd/system/fulcrum.service + content: | + # MiniBolt: systemd unit for Fulcrum + # /etc/systemd/system/fulcrum.service + + [Unit] + Description=Fulcrum + After=network.target + + StartLimitBurst=2 + StartLimitIntervalSec=20 + + [Service] + ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf + + User={{ fulcrum_user }} + Group={{ fulcrum_group }} + + # Process management + #################### + Type=simple + KillSignal=SIGINT + TimeoutStopSec=300 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + notify: Restart fulcrum + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum service + systemd: + name: fulcrum + enabled: yes + state: started + + - name: Wait for Fulcrum to start + wait_for: + port: "{{ fulcrum_tcp_port }}" + host: "{{ fulcrum_tcp_bind }}" + delay: 5 + timeout: 30 + ignore_errors: yes + + - name: Check Fulcrum service status + systemd: + name: fulcrum + register: fulcrum_service_status + changed_when: false + + - name: Display Fulcrum service status + debug: + msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" + + - name: Create Fulcrum health check and push script + copy: + dest: /usr/local/bin/fulcrum-healthcheck-push.sh + content: | + #!/bin/bash + # + # Fulcrum Health Check and Push to Uptime Kuma + # Checks if Fulcrum TCP port is responding and pushes status to Uptime Kuma + # + + FULCRUM_HOST="{{ fulcrum_tcp_bind }}" + FULCRUM_PORT={{ fulcrum_tcp_port }} + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + # Check if Fulcrum TCP port is responding + check_fulcrum() { + # Try to connect to TCP port + timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null + return $? + } + + # Push status to Uptime Kuma + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + + # URL encode spaces in message + local encoded_msg="${msg// /%20}" + + if ! curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then + echo "ERROR: Failed to push to Uptime Kuma" + return 1 + fi + } + + # Main health check + if check_fulcrum; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Fulcrum TCP port not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create systemd timer for Fulcrum health check + copy: + dest: /etc/systemd/system/fulcrum-healthcheck.timer + content: | + [Unit] + Description=Fulcrum Health Check Timer + Requires=fulcrum.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for Fulcrum health check + copy: + dest: /etc/systemd/system/fulcrum-healthcheck.service + content: | + [Unit] + Description=Fulcrum Health Check and Push to Uptime Kuma + After=network.target fulcrum.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum health check timer + systemd: + name: fulcrum-healthcheck.timer + enabled: yes + state: started + + - name: Create Uptime Kuma push monitor setup script for Fulcrum + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_fulcrum_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for new monitor") + push_url = f"{url}/api/push/{push_token}" + print(f"Push URL: {push_url}") + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "Fulcrum" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_fulcrum_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Extract push URL from monitor setup output + set_fact: + uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" + delegate_to: localhost + become: no + when: monitor_setup.stdout is defined + + - name: Display extracted push URL + debug: + msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" + when: uptime_kuma_push_url | default('') != '' + + - name: Set push URL in systemd service environment + lineinfile: + path: /etc/systemd/system/fulcrum-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" + state: present + insertafter: '^\[Service\]' + when: uptime_kuma_push_url | default('') != '' + + - name: Reload systemd daemon after push URL update + systemd: + daemon_reload: yes + when: uptime_kuma_push_url | default('') != '' + + - name: Restart health check timer to pick up new environment + systemd: + name: fulcrum-healthcheck.timer + state: restarted + when: uptime_kuma_push_url | default('') != '' + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_fulcrum_monitor.py + - /tmp/ansible_config.yml + - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz + - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux + + handlers: + - name: Restart fulcrum + systemd: + name: fulcrum + state: restarted + + +- name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./fulcrum_vars.yml + vars: + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Create Fulcrum SSL proxy socket unit + copy: + dest: /etc/systemd/system/fulcrum-ssl-proxy.socket + content: | + [Unit] + Description=Fulcrum SSL Proxy Socket + + [Socket] + ListenStream={{ fulcrum_ssl_port }} + + [Install] + WantedBy=sockets.target + owner: root + group: root + mode: '0644' + notify: Restart fulcrum-ssl-proxy socket + + - name: Create Fulcrum SSL proxy service unit + copy: + dest: /etc/systemd/system/fulcrum-ssl-proxy.service + content: | + [Unit] + Description=Fulcrum SSL Proxy to {{ fulcrum_tailscale_hostname }} + Requires=fulcrum-ssl-proxy.socket + After=network.target + + [Service] + Type=notify + ExecStart=/lib/systemd/systemd-socket-proxyd {{ fulcrum_tailscale_hostname }}:{{ fulcrum_ssl_port }} + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum SSL proxy socket + systemd: + name: fulcrum-ssl-proxy.socket + enabled: yes + state: started + + - name: Allow Fulcrum SSL port through UFW + ufw: + rule: allow + port: "{{ fulcrum_ssl_port | string }}" + proto: tcp + comment: "Fulcrum SSL public access" + + - name: Verify connectivity to fulcrum-box via Tailscale + wait_for: + host: "{{ fulcrum_tailscale_hostname }}" + port: "{{ fulcrum_ssl_port }}" + timeout: 10 + ignore_errors: yes + + - name: Display public endpoint + debug: + msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public SSL Port + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_fulcrum_ssl_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_fulcrum_ssl_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for TCP monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_fulcrum_ssl_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ fulcrum_ssl_port }} + monitor_name: "Fulcrum SSL Public" + mode: '0644' + + - name: Run Uptime Kuma TCP monitor setup + command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up TCP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_fulcrum_ssl_tcp_monitor.py + - /tmp/ansible_fulcrum_ssl_config.yml + + handlers: + - name: Restart fulcrum-ssl-proxy socket + systemd: + name: fulcrum-ssl-proxy.socket + state: restarted + diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml new file mode 100644 index 0000000..8486b14 --- /dev/null +++ b/ansible/services/fulcrum/fulcrum_vars.yml @@ -0,0 +1,51 @@ +# Fulcrum Configuration Variables + +# Version - Pinned to specific release +fulcrum_version: "2.1.0" # Fulcrum version to install + +# Directories +fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) +fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) +fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk +fulcrum_binary_path: /usr/local/bin/Fulcrum + +# Network - Bitcoin RPC connection +# Bitcoin Knots is on a different host (knots_box_local) +# Using RPC user/password authentication (credentials from infra_secrets.yml) +bitcoin_rpc_host: "192.168.1.140" # Bitcoin Knots RPC host (IP of knots_box_local) +bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml + +# Network - Fulcrum server +fulcrum_tcp_port: 50001 +fulcrum_ssl_port: 50002 +# Binding address for Fulcrum TCP/SSL server: +# - "127.0.0.1" = localhost only (use when Caddy is on the same box) +# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) +# - Specific IP = bind to specific network interface +fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) +fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port +# If Caddy is on a different box, set this to the IP address that Caddy will use to connect + +# SSL/TLS Configuration +fulcrum_ssl_enabled: true +fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" +fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" +fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert + +# Port forwarding configuration (for public access via VPS) +fulcrum_tailscale_hostname: "fulcrum-box" + +# Performance +# db_mem will be calculated as 75% of available RAM automatically in playbook +fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache + +# Configuration options +fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs +fulcrum_peering: false # Disable peering with other Fulcrum servers +fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications + +# Service user +fulcrum_user: fulcrum +fulcrum_group: fulcrum + diff --git a/ansible/services/gatus/deploy_gatus_playbook.yml b/ansible/services/gatus/deploy_gatus_playbook.yml deleted file mode 100644 index d6a5ac1..0000000 --- a/ansible/services/gatus/deploy_gatus_playbook.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -# Gatus: health checks, status page and alerting for the whole estate. -# -# Built from source and run under systemd - upstream publishes no binaries, and -# their Dockerfile shows the runtime needs nothing but the static binary and a -# CA bundle. See roles/gatus/README.md. -- name: Deploy Gatus on the observability host - hosts: observability - become: yes - vars: - gatus_alerting: - signal: - # NOTE: the key is `api-url`, not `url` as upstream's own README table - # says - see alerting/provider/signal/signal.go. Gatus appends /v2/send - # itself if the suffix is missing. - # - # Reached by service name over the shared docker network. Gatus runs in - # a container, so 127.0.0.1 here would be the Gatus container, not the - # host - and the Signal API deliberately publishes no ports because it - # has no authentication. - api-url: "http://signal-api:8080" - number: "{{ signal_number }}" - recipients: "{{ signal_recipients }}" - default-alert: - # Overridden per group by the registration playbooks; these are the - # values that apply if a caller sets nothing. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - # An ongoing outage should not become an ongoing phone buzz. - minimum-reminder-interval: 6h - - roles: - - gatus - -# The dashboard is bound to loopback; Caddy publishes it. -# -# Auth is done HERE, at the edge, and not with Gatus's own `security.basic`. -# Gatus's security middleware protects exactly four routes (api/api.go): -# -# /api/v1/endpoints/statuses -# /api/v1/endpoints/:key/statuses -# /api/v1/suites/statuses -# /api/v1/suites/:key/statuses -# -# Everything else is registered on the UNPROTECTED router, including -# /api/v1/config, every badge, and - the part that matters - -# /api/v1/endpoints/:key/uptimes/:duration and .../response-times/:duration/history, -# which return real per-endpoint data to anyone who can guess a key. Keys are -# just "_". So Gatus's own auth makes the dashboard render empty -# while leaving the data readable, which is worse than it looks. -# -# The one route that must NOT sit behind basic auth is the external-endpoint -# push API. It authenticates with `Authorization: Bearer `, and basic -# auth wants `Authorization: Basic <...>` - same header, two schemes, and the -# push clients lose. It is not actually unauthenticated: the handler 401s on a -# missing prefix, an empty token, or a token that does not match that endpoint's -# own. Upstream's comment on the route says exactly that. -- name: Publish the Gatus status page through Caddy - hosts: observability - become: yes - tasks: - - name: Require the dashboard credentials to be set - ansible.builtin.assert: - that: - - gatus_dashboard_username is defined - - gatus_dashboard_username | length > 0 - - gatus_dashboard_password_hash is defined - - gatus_dashboard_password_hash.startswith('$2') - fail_msg: >- - gatus_dashboard_username and gatus_dashboard_password_hash must be in - the vault. Generate the hash on the observability host, which runs - Caddy natively, so the bcrypt cost and format match what verifies it: - caddy hash-password --plaintext 'your-password' - then: ansible-vault edit group_vars/all/vault.yml - - - name: Configure the Caddy vhost for Gatus - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: gatus - caddy_site_domain: "{{ subdomains.gatus }}.{{ root_domain }}" - # caddy_site_body rather than caddy_site_upstream + caddy_site_basic_auth, - # because that pair applies auth to the whole site with no way to carve - # out the push path. `handle` blocks are mutually exclusive and first - # match wins, so the push API gets a route of its own. - caddy_site_body: | - @push { - path /api/v1/endpoints/*/external - method POST - } - - # Push API: Bearer-authenticated by Gatus itself. No basic auth here, - # or the Authorization header collides. - handle @push { - reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} - } - - # Everything else: the dashboard, the config endpoint, the badges and - # the uptime/response-time history. - handle { - basic_auth { - {{ gatus_dashboard_username }} {{ gatus_dashboard_password_hash }} - } - reverse_proxy 127.0.0.1:{{ gatus_port | default(8080) }} - } diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index e181f24..1bcf5bf 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -1,12 +1,18 @@ - name: Deploy headscale and configure Caddy reverse proxy - hosts: vpn_control + hosts: spacey become: no vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./headscale_vars.yml vars: headscale_subdomain: "{{ subdomains.headscale }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" headscale_base_domain: "tailnet.{{ root_domain }}" + headscale_namespace: "{{ service_settings.headscale.namespace }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Install required packages @@ -224,34 +230,144 @@ port: '3478' proto: udp - - name: Publish headscale through Caddy - ansible.builtin.include_role: - name: caddy_site - # This play is become: no and elevates per task. `apply` is how an - # include_role passes become down to the role's tasks - `become:` on - # the include itself is rejected. The role's handler sets its own. - apply: - become: yes - vars: - caddy_site_name: headscale - caddy_site_domain: "{{ headscale_domain }}" - # Raw body, and it must stay raw: the /admin* route in front of - # Headplane is not expressible as a plain reverse_proxy. The previous - # version of this task wrote only `reverse_proxy localhost:8080`, which - # would have deleted the admin route and its auth on the next run. - caddy_site_body: | - @headplane { - path /admin* + - name: Ensure Caddy sites-enabled directory exists + become: yes + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + become: yes + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy reverse proxy configuration for headscale + become: yes + copy: + dest: "{{ caddy_sites_dir }}/headscale.conf" + content: | + {{ headscale_domain }} { + reverse_proxy localhost:{{ headscale_port }} } - handle @headplane { - basicauth { - {{ headscale_ui_username }} {{ headscale_ui_password_hash }} - } - reverse_proxy http://localhost:{{ headplane_port }} - } - # Headscale API is protected by its own API key authentication - # All API operations require a valid Bearer token in the Authorization header - reverse_proxy * http://localhost:{{ headscale_port }} + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + become: yes + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for Headscale + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_headscale_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ headscale_domain }}/health" + monitor_name: "Headscale" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_headscale_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_headscale_monitor.py + - /tmp/ansible_config.yml handlers: - name: Restart headscale diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index c3fb948..653c175 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -1,5 +1,5 @@ # Headscale service configuration -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (subdomain and caddy_sites_dir now in services_config.yml) headscale_port: 8080 headscale_grpc_port: 50443 @@ -10,14 +10,15 @@ headscale_version: "0.26.1" # Data directory headscale_data_dir: /var/lib/headscale -# Namespace is headscale_namespace in group_vars/all/main.yml +# Namespace now configured in services_config.yml under service_settings.headscale.namespace # Remote access -remote_host_name: "{{ groups['vpn_control'] | first }}" +remote_host_name: "spacey" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" -# Headplane (headscale admin UI), proxied at /admin* behind Caddy basic auth -headplane_port: 3000 +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/headscale-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/headscale_backup.sh" diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml deleted file mode 100644 index 15ba5b7..0000000 --- a/ansible/services/headscale/setup_backup_headscale.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Configure the Headscale backup on the vpn_control host - hosts: vpn_control - become: yes - vars_files: - - ./headscale_vars.yml - - tasks: - - name: Ensure Headscale dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: headscale - backup_source_description: "Headscale" - # -C / with relative paths: avoids tar's "removing leading /" and makes - # the restore target explicit. - backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" - backup_source_stop_service: headscale - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:00:00" - # Reported to Gatus as backup-dump_headscale. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_headscale/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml new file mode 100644 index 0000000..5f9a764 --- /dev/null +++ b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml @@ -0,0 +1,75 @@ +- name: Configure local backup for Headscale from remote + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./headscale_vars.yml + vars: + remote_data_path: "{{ headscale_data_dir }}" + remote_config_path: "/etc/headscale" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "remote_config_path='{{ remote_config_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + # Stop headscale service for consistent backup + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop headscale" + + # Backup data directory + rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/data/" + + # Backup config directory + rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_config_path }}/ "$BACKUP_DIR/config/" + + # Start headscale service again + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start headscale" + + # Rotate old backups (keep 14 days) + find "{{ local_backup_dir }}" -maxdepth 1 -type d -name '20*' -mtime +13 -exec rm -rf {} \; + + - name: Ensure cronjob for backup exists + cron: + name: "Headscale backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index 7b89f33..e5d546b 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -1,11 +1,16 @@ - name: Deploy LNBits with Poetry and configure Caddy reverse proxy - hosts: edge + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./lnbits_vars.yml vars: lnbits_subdomain: "{{ subdomains.lnbits }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" lnbits_domain: "{{ lnbits_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create lnbits directory @@ -142,12 +147,142 @@ enabled: yes state: started - - name: Publish LNBits through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: lnbits - caddy_site_domain: "{{ lnbits_domain }}" - caddy_site_upstream: "localhost:{{ lnbits_port }}" - caddy_site_headers_up: - X-Forwarded-Host: "{{ lnbits_domain }}" + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + create: yes + mode: '0644' + + - name: Create Caddy reverse proxy configuration for lnbits + copy: + dest: "{{ caddy_sites_dir }}/lnbits.conf" + content: | + {{ lnbits_domain }} { + reverse_proxy localhost:{{ lnbits_port }} { + header_up X-Forwarded-Host {{ lnbits_domain }} + } + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for LNBits + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_lnbits_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ lnbits_domain }}/api/v1/health" + monitor_name: "LNBits" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_lnbits_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_lnbits_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/lnbits/lnbits_vars.yml b/ansible/services/lnbits/lnbits_vars.yml index 466ce78..bdb97df 100644 --- a/ansible/services/lnbits/lnbits_vars.yml +++ b/ansible/services/lnbits/lnbits_vars.yml @@ -3,12 +3,15 @@ lnbits_dir: /opt/lnbits lnbits_data_dir: "{{ lnbits_dir }}/data" lnbits_port: 8765 -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "{{ groups['edge'] | first }}" +remote_host_name: "vipy" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/lnbits-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/lnbits_backup.sh" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml deleted file mode 100644 index 2da0ac0..0000000 --- a/ansible/services/lnbits/setup_backup_lnbits.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -# LNBits backup: dumps locally on vipy, encrypted with age. -# The old job produced TWO gpg artefacts (data, then .env separately). They are -# folded into one tar here so the wallet database and the .env that configures -# it are always the same point in time; two artefacts written by two runs can -# drift. Pulling one file back out needs no unpacking: -# age -d -i | tar -xzO opt/lnbits/lnbits/.env -- name: Configure the LNBits backup on the edge host - hosts: edge - become: yes - vars_files: - - ./lnbits_vars.yml - - tasks: - - name: Ensure LNBits dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: lnbits - backup_source_description: "LNBits" - backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env" - backup_source_stop_service: lnbits - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:20:00" - # Reported to Gatus as backup-dump_lnbits. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_lnbits/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml new file mode 100644 index 0000000..5d10dec --- /dev/null +++ b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml @@ -0,0 +1,104 @@ +- name: Configure local backup for LNBits from remote + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./lnbits_vars.yml + vars: + remote_data_path: "{{ lnbits_data_dir }}" + remote_lnbits_dir: "{{ lnbits_dir }}/lnbits" + gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" + gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + - "gpg_recipient={{ gpg_recipient }}" + - "gpg_key_id={{ gpg_key_id }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + ENCRYPTED_BACKUP="{{ local_backup_dir }}/lnbits-backup-$TIMESTAMP.tar.gz.gpg" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + # Stop LNBits service before backup + echo "Stopping LNBits service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl stop lnbits.service" + + # Create encrypted backup on the fly + # First, create a tar archive of the data directory and pipe it through gpg + echo "Creating backup..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "cd {{ remote_data_path }} && tar -czf - ." | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" + + # Also backup the .env file separately (smaller, might need quick access) + $SSH_CMD {{ remote_user }}@{{ remote_host }} "cat {{ remote_lnbits_dir }}/.env" | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "{{ local_backup_dir }}/lnbits-env-$TIMESTAMP.gpg" + + # Start LNBits service after backup + echo "Starting LNBits service..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} "sudo systemctl start lnbits.service" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for backup_file in "{{ local_backup_dir }}"/lnbits-backup-*.tar.gz.gpg; do + if [ -f "$backup_file" ]; then + # Extract date from filename: lnbits-backup-YYYY-MM-DD.tar.gz.gpg + file_date=$(basename "$backup_file" | sed -n 's/lnbits-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$backup_file" + fi + fi + done + for env_file in "{{ local_backup_dir }}"/lnbits-env-*.gpg; do + if [ -f "$env_file" ]; then + # Extract date from filename: lnbits-env-YYYY-MM-DD.gpg + file_date=$(basename "$env_file" | sed -n 's/lnbits-env-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$env_file" + fi + fi + done + + echo "Backup completed successfully" + + - name: Ensure cronjob for backup exists + cron: + name: "LNBits backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index 8b21b85..da56bd6 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -1,7 +1,10 @@ - name: Deploy Memos on memos-box - hosts: memos + hosts: memos_box_local become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" @@ -62,6 +65,14 @@ owner: root group: root + - name: Clean up temporary files + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/memos.tar.gz + - /tmp/memos + - name: Create memos environment file copy: dest: "{{ memos_config_dir }}/memos.env" @@ -134,21 +145,162 @@ state: restarted -- name: Configure Caddy reverse proxy for Memos on the edge host (via Tailscale) - hosts: edge +- name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Publish Memos through Caddy (via Tailscale) - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: memos - caddy_site_domain: "{{ memos_domain }}" - caddy_site_upstream: "{{ memos_tailscale_hostname }}:{{ memos_port }}" - caddy_site_resolvers: "100.100.100.100" + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy reverse proxy configuration for memos (via Tailscale) + copy: + dest: "{{ caddy_sites_dir }}/memos.conf" + content: | + {{ memos_domain }} { + reverse_proxy {{ memos_tailscale_hostname }}:{{ memos_port }} { + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers 100.100.100.100 + } + } + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for Memos + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_memos_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_memos_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_memos_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ memos_domain }}/healthz" + monitor_name: "Memos" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_memos_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_memos_monitor.py + - /tmp/ansible_memos_config.yml diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index 94c6de7..99618db 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -12,7 +12,15 @@ memos_url: "https://github.com/usememos/memos/releases/download/v{{ memos_versio memos_tailscale_hostname: "memos-box" memos_tailscale_ip: "100.64.0.4" -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain in services_config.yml) # Remote access (for backup from lapy via Tailscale) +backup_host: "{{ memos_tailscale_hostname }}" +backup_user: "counterweight" +backup_key_file: "~/.ssh/counterganzua" +backup_port: 22 + +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/memos_backup.sh" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml deleted file mode 100644 index 53ab85d..0000000 --- a/ansible/services/memos/setup_backup_memos.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# Memos backup: dumps locally on memos-box, encrypted with age. -# Replaces the lapy pull, which had been writing EMPTY directories since -# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned -# to a different machine that has no rsync. -- name: Configure the Memos backup on its own host - hosts: memos - become: yes - vars_files: - - ./memos_vars.yml - - tasks: - - name: Ensure Memos dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: memos - backup_source_description: "Memos" - backup_source_dump_command: "tar -czf - -C / var/lib/memos" - # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a - # consistent database rather than a torn mid-write copy. The old rsync - # job did not stop it. - backup_source_stop_service: memos - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:00:00" - # Reported to Gatus as backup-dump_memos. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_memos/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml new file mode 100644 index 0000000..6d9c161 --- /dev/null +++ b/ansible/services/memos/setup_backup_memos_to_lapy.yml @@ -0,0 +1,106 @@ +- name: Configure local backup for Memos from memos-box + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./memos_vars.yml + vars: + backup_data_path: "{{ memos_data_dir }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "backup_host={{ backup_host }}" + - "backup_user={{ backup_user }}" + - "backup_data_path='{{ backup_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if backup_key_file %} + SSH_CMD="ssh -i {{ backup_key_file }} -p {{ backup_port }}" + {% else %} + SSH_CMD="ssh -p {{ backup_port }}" + {% endif %} + + rsync -az -e "$SSH_CMD" --rsync-path="sudo rsync" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for dir in "{{ local_backup_dir }}"/20*; do + if [ -d "$dir" ]; then + dir_date=$(basename "$dir") + if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then + rm -rf "$dir" + fi + fi + done + + - name: Ensure cronjob for backup exists + cron: + name: "Memos backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 15 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check backup directory exists and contains files + stat: + path: "{{ local_backup_dir }}/{{ today_date.stdout }}" + register: backup_dir_stat + + - name: Verify backup directory exists + assert: + that: + - backup_dir_stat.stat.exists + - backup_dir_stat.stat.isdir + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" + success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" + + - name: Check if backup directory contains files + find: + paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" + recurse: yes + register: backup_files + + - name: Verify backup directory is not empty + assert: + that: + - backup_files.files | length > 0 + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" + success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" + diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 0042fb0..658180a 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -1,33 +1,751 @@ ---- - name: Deploy Mempool Block Explorer with Docker - hosts: mempool + hosts: mempool_box_local become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./mempool_vars.yml vars: - # Preserves the three push URLs these checks have been reporting to all - # along, so the move to a role changes no behaviour. The role knows nothing - # about Uptime Kuma — these are just "URLs that accept a ping", and whatever - # replaces it sets the same values. - mempool_healthchecks: - - {name: mariadb, label: MariaDB, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-mariadb/external"} - - {name: backend, label: Backend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-backend/external"} - - {name: frontend, label: Frontend, push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_mempool-frontend/external"} - # One token for all three components: they run on the same host, so the - # blast radius is already that host. - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - mempool + mempool_subdomain: "{{ subdomains.mempool }}" + mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" -- name: Configure Caddy reverse proxy for Mempool on the edge host - hosts: edge - become: yes - vars: - mempool_domain: "{{ subdomains.mempool }}.{{ root_domain }}" tasks: - - name: Publish Mempool through Caddy (via Tailscale) - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: mempool - caddy_site_domain: "{{ mempool_domain }}" - caddy_site_upstream: "{{ hostvars['mempool_box_local'].ansible_host }}:{{ hostvars['mempool_box_local'].mempool_frontend_port }}" - caddy_site_resolvers: "100.100.100.100" + # =========================================== + # Docker Installation (from 910_docker_playbook.yml) + # =========================================== + - name: Remove old Docker-related packages + apt: + name: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + purge: yes + autoremove: yes + + - name: Update apt cache + apt: + update_cache: yes + + - name: Install prerequisites + apt: + name: + - ca-certificates + - curl + state: present + + - name: Create directory for Docker GPG key + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Download Docker GPG key + get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + + - name: Get Debian architecture + command: dpkg --print-architecture + register: deb_arch + changed_when: false + + - name: Add Docker repository + apt_repository: + repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + filename: docker + state: present + update_cache: yes + + - name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + + - name: Ensure Docker is started and enabled + systemd: + name: docker + enabled: yes + state: started + + - name: Add user to docker group + user: + name: "{{ ansible_user }}" + groups: docker + append: yes + + # =========================================== + # Mempool Deployment + # =========================================== + - name: Create mempool directories + file: + path: "{{ item }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + loop: + - "{{ mempool_dir }}" + - "{{ mempool_data_dir }}" + - "{{ mempool_mysql_dir }}" + + - name: Create docker-compose.yml for Mempool + copy: + dest: "{{ mempool_dir }}/docker-compose.yml" + content: | + # All containers use host network for Tailscale MagicDNS resolution + services: + mariadb: + image: mariadb:10.11 + container_name: mempool-db + restart: unless-stopped + network_mode: host + environment: + MYSQL_DATABASE: "{{ mariadb_database }}" + MYSQL_USER: "{{ mariadb_user }}" + MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" + MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" + volumes: + - {{ mempool_mysql_dir }}:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + mempool-backend: + image: mempool/backend:{{ mempool_version }} + container_name: mempool-backend + restart: unless-stopped + network_mode: host + environment: + # Database (localhost since all containers share host network) + DATABASE_ENABLED: "true" + DATABASE_HOST: "127.0.0.1" + DATABASE_DATABASE: "{{ mariadb_database }}" + DATABASE_USERNAME: "{{ mariadb_user }}" + DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" + # Bitcoin Core/Knots (via Tailnet MagicDNS) + CORE_RPC_HOST: "{{ bitcoin_host }}" + CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" + CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" + CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" + # Electrum (Fulcrum via Tailnet MagicDNS) + ELECTRUM_HOST: "{{ fulcrum_host }}" + ELECTRUM_PORT: "{{ fulcrum_port }}" + ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" + # Mempool settings + MEMPOOL_NETWORK: "{{ mempool_network }}" + MEMPOOL_BACKEND: "electrum" + MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" + volumes: + - {{ mempool_data_dir }}:/backend/cache + depends_on: + mariadb: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + mempool-frontend: + image: mempool/frontend:{{ mempool_version }} + container_name: mempool-frontend + restart: unless-stopped + network_mode: host + environment: + FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" + BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" + depends_on: + - mempool-backend + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' + + - name: Pull Mempool images + command: docker compose pull + args: + chdir: "{{ mempool_dir }}" + + - name: Deploy Mempool containers with docker compose + command: docker compose up -d + args: + chdir: "{{ mempool_dir }}" + + - name: Wait for MariaDB to be healthy + command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db + register: mariadb_health + until: mariadb_health.stdout == 'healthy' + retries: 30 + delay: 10 + changed_when: false + + - name: Wait for Mempool backend to start + uri: + url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + method: GET + status_code: 200 + timeout: 10 + register: backend_check + until: backend_check.status == 200 + retries: 30 + delay: 10 + ignore_errors: yes + + - name: Wait for Mempool frontend to be available + uri: + url: "http://localhost:{{ mempool_frontend_port }}" + method: GET + status_code: 200 + timeout: 10 + register: frontend_check + until: frontend_check.status == 200 + retries: 20 + delay: 5 + ignore_errors: yes + + - name: Display deployment status + debug: + msg: + - "Mempool deployment complete!" + - "Frontend: http://localhost:{{ mempool_frontend_port }}" + - "Backend API: http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + - "Backend check: {{ 'OK' if backend_check.status == 200 else 'Still initializing...' }}" + - "Frontend check: {{ 'OK' if frontend_check.status == 200 else 'Still initializing...' }}" + + # =========================================== + # Health Check Scripts for Uptime Kuma Push Monitors + # =========================================== + - name: Create Mempool MariaDB health check script + copy: + dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + check_container() { + local status=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db 2>/dev/null) + [ "$status" = "healthy" ] + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_container; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "MariaDB container unhealthy" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create Mempool backend health check script + copy: + dest: /usr/local/bin/mempool-backend-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + BACKEND_PORT={{ mempool_backend_port }} + + check_backend() { + curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_backend; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Backend API not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create Mempool frontend health check script + copy: + dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + FRONTEND_PORT={{ mempool_frontend_port }} + + check_frontend() { + curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_frontend; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Frontend not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + # =========================================== + # Systemd Timers for Health Checks + # =========================================== + - name: Create systemd services for health checks + copy: + dest: "/etc/systemd/system/mempool-{{ item.name }}-healthcheck.service" + content: | + [Unit] + Description=Mempool {{ item.label }} Health Check + After=network.target docker.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/mempool-{{ item.name }}-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + loop: + - { name: "mariadb", label: "MariaDB" } + - { name: "backend", label: "Backend" } + - { name: "frontend", label: "Frontend" } + + - name: Create systemd timers for health checks + copy: + dest: "/etc/systemd/system/mempool-{{ item }}-healthcheck.timer" + content: | + [Unit] + Description=Mempool {{ item }} Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + loop: + - mariadb + - backend + - frontend + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start health check timers + systemd: + name: "mempool-{{ item }}-healthcheck.timer" + enabled: yes + state: started + loop: + - mariadb + - backend + - frontend + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for Mempool + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_mempool_monitors.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_mempool_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitors_to_create = config['monitors'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + results = {} + for monitor_name in monitors_to_create: + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + results[monitor_name] = f"{url}/api/push/{push_token}" + print(f"Push URL ({monitor_name}): {results[monitor_name]}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=90, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + results[monitor_name] = f"{url}/api/push/{push_token}" + print(f"Push URL ({monitor_name}): {results[monitor_name]}") + + api.disconnect() + print("SUCCESS") + + # Write results to file for Ansible to read + with open('/tmp/mempool_push_urls.yml', 'w') as f: + yaml.dump(results, f) + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_mempool_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitors: + - "Mempool MariaDB" + - "Mempool Backend" + - "Mempool Frontend" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_mempool_monitors.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URLs from file + slurp: + src: /tmp/mempool_push_urls.yml + delegate_to: localhost + become: no + register: push_urls_file + ignore_errors: yes + + - name: Parse push URLs + set_fact: + push_urls: "{{ push_urls_file.content | b64decode | from_yaml }}" + when: push_urls_file.content is defined + ignore_errors: yes + + - name: Update MariaDB health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-mariadb-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool MariaDB'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool MariaDB'] is defined + + - name: Update Backend health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-backend-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Backend'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool Backend'] is defined + + - name: Update Frontend health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-frontend-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Frontend'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool Frontend'] is defined + + - name: Reload systemd after push URL updates + systemd: + daemon_reload: yes + when: push_urls is defined + + - name: Restart health check timers + systemd: + name: "mempool-{{ item }}-healthcheck.timer" + state: restarted + loop: + - mariadb + - backend + - frontend + when: push_urls is defined + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_mempool_monitors.py + - /tmp/ansible_mempool_config.yml + - /tmp/mempool_push_urls.yml + +- name: Configure Caddy reverse proxy for Mempool on vipy + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./mempool_vars.yml + vars: + mempool_subdomain: "{{ subdomains.mempool }}" + mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" + + tasks: + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + create: yes + mode: '0644' + + - name: Create Caddy reverse proxy configuration for Mempool + copy: + dest: "{{ caddy_sites_dir }}/mempool.conf" + content: | + {{ mempool_domain }} { + reverse_proxy mempool-box:{{ mempool_frontend_port }} { + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers 100.100.100.100 + } + } + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + systemd: + name: caddy + state: reloaded + + - name: Display Mempool URL + debug: + msg: "Mempool is now available at https://{{ mempool_domain }}" + + # =========================================== + # Uptime Kuma HTTP Monitor for Public Endpoint + # =========================================== + - name: Create Uptime Kuma HTTP monitor setup script for Mempool + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_mempool_http_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_mempool_http_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating HTTP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for HTTP monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_mempool_http_config.yml + content: | + uptime_kuma_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ mempool_domain }}" + monitor_name: "Mempool Public Access" + mode: '0644' + + - name: Run Uptime Kuma HTTP monitor setup + command: python3 /tmp/setup_mempool_http_monitor.py + delegate_to: localhost + become: no + register: http_monitor_setup + changed_when: "'SUCCESS' in http_monitor_setup.stdout" + ignore_errors: yes + + - name: Display HTTP monitor setup output + debug: + msg: "{{ http_monitor_setup.stdout_lines }}" + when: http_monitor_setup.stdout is defined + + - name: Clean up HTTP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_mempool_http_monitor.py + - /tmp/ansible_mempool_http_config.yml + diff --git a/ansible/services/mempool/mempool_vars.yml b/ansible/services/mempool/mempool_vars.yml new file mode 100644 index 0000000..d3051c3 --- /dev/null +++ b/ansible/services/mempool/mempool_vars.yml @@ -0,0 +1,33 @@ +# Mempool Configuration Variables + +# Version - Pinned to specific release +mempool_version: "v3.2.1" + +# Directories +mempool_dir: /opt/mempool +mempool_data_dir: "{{ mempool_dir }}/data" +mempool_mysql_dir: "{{ mempool_dir }}/mysql" + +# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) +bitcoin_host: "knots-box" +bitcoin_rpc_port: 8332 +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml + +# Network - Fulcrum Electrum server (via Tailnet Magic DNS) +fulcrum_host: "fulcrum-box" +fulcrum_port: 50001 +fulcrum_tls: "false" + +# Mempool network mode +mempool_network: "mainnet" + +# Container ports (internal) +mempool_frontend_port: 8080 +mempool_backend_port: 8999 + +# MariaDB settings +mariadb_database: "mempool" +mariadb_user: "mempool" +# Note: mariadb_mempool_password is loaded from infra_secrets.yml + + diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index 58f0348..b8c0064 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -1,24 +1,20 @@ - name: Deploy ntfy-emergency-app with Docker Compose and configure Caddy reverse proxy - hosts: edge + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml + - ../../infra_secrets.yml + - ../../services_config.yml - ./ntfy_emergency_app_vars.yml vars: ntfy_emergency_app_subdomain: "{{ subdomains.ntfy_emergency_app }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" ntfy_emergency_app_domain: "{{ ntfy_emergency_app_subdomain }}.{{ root_domain }}" - # ⚠ UNRESOLVED: this app sends its notifications to an ntfy server, and the - # server it points at ran on watchtower, which is being destroyed. This was - # derived from subdomains.ntfy, which is now gone with it. - # - # Until an ntfy server exists again this URL is dead, and the app fails - # silently at exactly the moment it matters - it is an EMERGENCY notifier. - # Three ways out, none of them automatic: - # * stand ntfy up somewhere else (the monitoring VPS, or vipy) - # * point this at the public ntfy.sh - # * retire the app - ntfy_emergency_app_ntfy_url: "{{ ntfy_service_url }}" + ntfy_service_domain: "{{ subdomains.ntfy }}.{{ root_domain }}" + ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" ntfy_emergency_app_ntfy_user: "{{ ntfy_username | default('') }}" ntfy_emergency_app_ntfy_password: "{{ ntfy_password | default('') }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create ntfy-emergency-app directory @@ -53,10 +49,142 @@ args: chdir: "{{ ntfy_emergency_app_dir }}" - - name: Publish ntfy-emergency-app through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: ntfy-emergency-app - caddy_site_domain: "{{ ntfy_emergency_app_domain }}" - caddy_site_upstream: "localhost:{{ ntfy_emergency_app_port }}" + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy reverse proxy configuration for ntfy-emergency-app + copy: + dest: "{{ caddy_sites_dir }}/ntfy-emergency-app.conf" + content: | + {{ ntfy_emergency_app_domain }} { + reverse_proxy localhost:{{ ntfy_emergency_app_port }} + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_ntfy_emergency_app_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ ntfy_emergency_app_domain }}" + monitor_name: "ntfy-emergency-app" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_ntfy_emergency_app_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_ntfy_emergency_app_monitor.py + - /tmp/ansible_config.yml diff --git a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml index bfa4bd1..415bc4d 100644 --- a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml +++ b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml @@ -2,21 +2,15 @@ ntfy_emergency_app_dir: /opt/ntfy-emergency-app ntfy_emergency_app_port: 3000 -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # ntfy configuration ntfy_emergency_app_topic: "emergencia" ntfy_emergency_app_ui_message: "Leave Pablo a message, he will respond as soon as possible" # Remote access -remote_host_name: "{{ groups['edge'] | first }}" +remote_host_name: "vipy" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" - -# Where the emergency notifications are sent. This pointed at ntfy.contrapeso.xyz -# on watchtower; that host is being destroyed, so this MUST be repointed before -# the app can work again. Left at the old value so the break is visible rather -# than silently defaulted to something plausible. -ntfy_service_url: "https://ntfy.contrapeso.xyz" diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml new file mode 100644 index 0000000..0729baa --- /dev/null +++ b/ansible/services/ntfy/deploy_ntfy_playbook.yml @@ -0,0 +1,120 @@ +- name: Deploy ntfy and configure Caddy reverse proxy + hosts: watchtower + become: yes + vars_files: + - ../../infra_vars.yml + - ../../infra_secrets.yml + - ../../services_config.yml + - ./ntfy_vars.yml + vars: + ntfy_subdomain: "{{ subdomains.ntfy }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" + ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" + + tasks: + - name: Ensure /etc/apt/keyrings exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Download and dearmor ntfy GPG key + shell: curl -fsSL https://archive.heckel.io/apt/pubkey.txt | gpg --dearmor -o /etc/apt/keyrings/archive.heckel.io.gpg + args: + creates: /etc/apt/keyrings/archive.heckel.io.gpg + + - name: Add ntfy APT repository + copy: + dest: /etc/apt/sources.list.d/archive.heckel.io.list + content: | + deb [arch=amd64 signed-by=/etc/apt/keyrings/archive.heckel.io.gpg] https://archive.heckel.io/apt debian main + mode: '0644' + + - name: Update APT cache + apt: + update_cache: yes + + - name: Install ntfy + apt: + name: ntfy + state: present + + - name: Ensure ntfy cache directories exist + file: + path: "{{ item }}" + state: directory + owner: ntfy + group: ntfy + mode: '0755' + loop: + - /var/cache/ntfy + - /var/cache/ntfy/attachments + + - name: Deploy ntfy configuration file + copy: + dest: /etc/ntfy/server.yml + content: | + base-url: "http://{{ ntfy_domain }}" + listen-http: ":{{ ntfy_port }}" + cache-file: "/var/cache/ntfy/cache.db" + attachment-cache-dir: "/var/cache/ntfy/attachments" + behind-proxy: true + auth-file: "/var/lib/ntfy/user.db" + auth-default-access: "deny-all" + owner: root + group: root + mode: '0644' + notify: Restart ntfy + + - name: Enable and start ntfy service + systemd: + name: ntfy + enabled: yes + state: started + + - name: Create ntfy admin user + shell: | + (echo "{{ ntfy_password }}"; echo "{{ ntfy_password }}") | ntfy user add --role=admin "{{ ntfy_username }}" + + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy reverse proxy configuration for ntfy + copy: + dest: "{{ caddy_sites_dir }}/ntfy.conf" + content: | + {{ ntfy_domain }}, http://{{ ntfy_domain }} { + reverse_proxy 127.0.0.1:{{ ntfy_port }} + + @httpget { + protocol http + method GET + path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) + } + redir @httpget https://{host}{uri} + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + handlers: + - name: Restart ntfy + systemd: + name: ntfy + state: restarted \ No newline at end of file diff --git a/ansible/services/ntfy/ntfy_vars.yml b/ansible/services/ntfy/ntfy_vars.yml new file mode 100644 index 0000000..5ebec37 --- /dev/null +++ b/ansible/services/ntfy/ntfy_vars.yml @@ -0,0 +1,3 @@ +ntfy_port: 6674 + +# ntfy_topic now lives in services_config.yml under service_settings.ntfy.topic \ No newline at end of file diff --git a/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml similarity index 83% rename from archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml rename to ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml index 97d6c9c..5ba03f1 100644 --- a/archive/uptime_kuma/setup_ntfy_uptime_kuma_notification.yml +++ b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml @@ -1,24 +1,15 @@ -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. -# -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. -# -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ - name: Setup ntfy as Uptime Kuma Notification Channel - hosts: observability + hosts: watchtower become: no vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./ntfy_vars.yml vars: ntfy_subdomain: "{{ subdomains.ntfy }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" uptime_kuma_subdomain: "{{ subdomains.uptime_kuma }}" ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" ntfy_server_url: "https://{{ ntfy_domain }}" diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index 21d8b43..f4ee8ec 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -1,11 +1,16 @@ - name: Deploy personal blog static site with Caddy file server - hosts: edge + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./personal_blog_vars.yml vars: personal_blog_subdomain: "{{ subdomains.personal_blog }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" personal_blog_domain: "{{ personal_blog_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Ensure user is in www-data group @@ -46,10 +51,139 @@ group: www-data mode: '0664' - - name: Publish the blog through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: personal-blog - caddy_site_domain: "{{ personal_blog_domain }}" - caddy_site_root: "{{ personal_blog_web_root }}" + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy file server configuration for personal blog + copy: + dest: "{{ caddy_sites_dir }}/personal-blog.conf" + content: | + {{ personal_blog_domain }} { + root * {{ personal_blog_web_root }} + file_server + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for Personal Blog + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_personal_blog_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ personal_blog_domain }}" + monitor_name: "Personal Blog" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_personal_blog_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_personal_blog_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/personal-blog/personal_blog_vars.yml b/ansible/services/personal-blog/personal_blog_vars.yml index a1f34e3..59e0921 100644 --- a/ansible/services/personal-blog/personal_blog_vars.yml +++ b/ansible/services/personal-blog/personal_blog_vars.yml @@ -4,7 +4,7 @@ personal_blog_web_root: "/var/www/pablohere.contrapeso.xyz" # Remote access for deployment -remote_host_name: "{{ groups['edge'] | first }}" +remote_host_name: "vipy" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml index bd9b715..2b90d68 100644 --- a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml +++ b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml @@ -1,7 +1,8 @@ - name: Configure deployment alias for personal blog in lapy .bashrc - hosts: control + hosts: lapy gather_facts: no vars_files: + - ../../infra_vars.yml - ./personal_blog_vars.yml vars: bashrc_path: "{{ lookup('env', 'HOME') }}/.bashrc" diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index b9cf243..65ad7c9 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -1,15 +1,535 @@ --- -# phoenixd: Lightning node on the edge host, used by LNBits as a wallet backend. -# Never exposed through Caddy — the HTTP API stays on loopback. -- name: Deploy phoenixd on the edge host - hosts: edge +# phoenixd Deployment Playbook +# +# Deploys phoenixd (https://phoenix.acinq.co/server), the server version of the +# Phoenix Lightning wallet, on vipy so LNBits can use it as a wallet backend +# over loopback. +# +# What this does: +# 1. Downloads the pinned phoenixd release and installs phoenixd + phoenix-cli +# 2. Creates a dedicated system user and a 0700 data directory +# 3. Creates and enables a systemd service +# 4. Creates a push-monitor health check script + systemd timer +# 5. Registers a push monitor in Uptime Kuma +# +# The HTTP API stays bound to 127.0.0.1 and is NOT proxied by Caddy: phoenixd +# holds funds and its API is protected by a single password. Anything that needs +# it either runs on this host or reaches it over the Tailscale mesh. +# +# ⚠️ After the first run, back up {{ phoenixd_data_dir }}/seed.dat. Losing it +# means losing the funds. See setup_backup_phoenixd_to_lapy.yml. + +- name: Deploy phoenixd on vipy + hosts: vipy become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./phoenixd_vars.yml vars: - # phoenixd's health check has never reported anywhere since the Uptime Kuma - # decommissioning — its systemd Environment= was left empty. Leaving it empty - # preserves that; the check still runs and its exit code is still the answer. - # Set this to plug in whatever monitoring replaces it. - healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_phoenixd/external" - healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" - roles: - - phoenixd + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + # =========================================== + # Prerequisites + # =========================================== + - name: Install phoenixd runtime dependencies + apt: + name: + - unzip + - curl + state: present + update_cache: yes + + # =========================================== + # System User and Directories + # =========================================== + - name: Create phoenixd system group + group: + name: "{{ phoenixd_group }}" + system: yes + + - name: Create phoenixd system user + user: + name: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ phoenixd_home }}" + create_home: yes + comment: "phoenixd Lightning node" + + - name: Create phoenixd home directory + file: + path: "{{ phoenixd_home }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0750" + + - name: Create phoenixd data directory + file: + path: "{{ phoenixd_data_dir }}" + state: directory + owner: "{{ phoenixd_user }}" + group: "{{ phoenixd_group }}" + mode: "0700" + + # =========================================== + # Download and Install + # =========================================== + - name: Check if phoenixd is already installed + stat: + path: "{{ phoenixd_bin_dir }}/phoenixd" + register: phoenixd_binary + + - name: Check installed phoenixd version + command: "{{ phoenixd_bin_dir }}/phoenixd --version" + register: phoenixd_installed_version + changed_when: false + failed_when: false + when: phoenixd_binary.stat.exists + + - name: Decide whether phoenixd needs installing + set_fact: + phoenixd_needs_install: >- + {{ not phoenixd_binary.stat.exists + or phoenixd_version not in (phoenixd_installed_version.stdout | default('')) }} + + - name: Download phoenixd {{ phoenixd_version }} + get_url: + url: "{{ phoenixd_url }}" + dest: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + mode: "0644" + when: phoenixd_needs_install | bool + + - name: Create temporary extraction directory + file: + path: /tmp/phoenixd-extract + state: directory + mode: "0755" + when: phoenixd_needs_install | bool + + - name: Extract phoenixd archive + unarchive: + src: "/tmp/phoenixd-{{ phoenixd_version }}.zip" + dest: /tmp/phoenixd-extract + remote_src: yes + when: phoenixd_needs_install | bool + + - name: Locate extracted binaries + find: + paths: /tmp/phoenixd-extract + patterns: "{{ item }}" + recurse: yes + file_type: file + register: phoenixd_extracted + loop: + - phoenixd + - phoenix-cli + when: phoenixd_needs_install | bool + + - name: Fail if the archive did not contain the expected binaries + assert: + that: + - item.files | length > 0 + fail_msg: "Could not find '{{ item.item }}' in the phoenixd {{ phoenixd_version }} archive" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + + - name: Install phoenixd and phoenix-cli binaries + copy: + src: "{{ item.files[0].path }}" + dest: "{{ phoenixd_bin_dir }}/{{ item.item }}" + remote_src: yes + owner: root + group: root + mode: "0755" + loop: "{{ phoenixd_extracted.results }}" + loop_control: + label: "{{ item.item }}" + when: phoenixd_needs_install | bool + notify: Restart phoenixd + + - name: Clean up phoenixd download artifacts + file: + path: "{{ item }}" + state: absent + loop: + - "/tmp/phoenixd-{{ phoenixd_version }}.zip" + - /tmp/phoenixd-extract + + # =========================================== + # Systemd Service + # =========================================== + - name: Build phoenixd command line arguments + set_fact: + phoenixd_args: >- + {{ (['--agree-to-terms-of-service'] if phoenixd_agree_tos else []) + + ['--chain', phoenixd_chain] + + ['--auto-liquidity', phoenixd_auto_liquidity] + + ['--http-bind-ip', phoenixd_http_bind_ip] + + ['--http-bind-port', phoenixd_http_bind_port | string] + + (['--max-mining-fee', phoenixd_max_mining_fee | string] if phoenixd_max_mining_fee else []) + + (['--webhook', phoenixd_webhook_url] if phoenixd_webhook_url else []) + + ['--silent'] }} + + - name: Create phoenixd systemd service + copy: + dest: /etc/systemd/system/phoenixd.service + content: | + [Unit] + Description=phoenixd - Lightning Network Node + Documentation=https://phoenix.acinq.co/server + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User={{ phoenixd_user }} + Group={{ phoenixd_group }} + WorkingDirectory={{ phoenixd_home }} + Environment=PHOENIX_DATADIR={{ phoenixd_data_dir }} + ExecStart={{ phoenixd_bin_dir }}/phoenixd {{ phoenixd_args | join(' ') }} + Restart=always + RestartSec=30 + TimeoutStartSec=120 + TimeoutStopSec=120 + StandardOutput=journal + StandardError=journal + + # Hardening: the node only ever writes to its own data directory + NoNewPrivileges=true + PrivateTmp=true + ProtectSystem=strict + ProtectHome=read-only + ReadWritePaths={{ phoenixd_data_dir }} + + LimitNOFILE=65535 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + notify: Restart phoenixd + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start phoenixd + systemd: + name: phoenixd + enabled: yes + state: started + + - name: Flush handlers so phoenixd is running before we inspect its data dir + meta: flush_handlers + + # =========================================== + # First Boot Checks + # =========================================== + - name: Wait for phoenixd to write its config file + wait_for: + path: "{{ phoenixd_data_dir }}/phoenix.conf" + state: present + timeout: 120 + + - name: Check that the seed file exists + stat: + path: "{{ phoenixd_data_dir }}/seed.dat" + register: phoenixd_seed_file + + - name: Fail if phoenixd did not create a seed + assert: + that: + - phoenixd_seed_file.stat.exists + fail_msg: "phoenixd started but {{ phoenixd_data_dir }}/seed.dat is missing - check 'journalctl -u phoenixd'" + + # =========================================== + # Health Check Script + Systemd Timer + # =========================================== + - name: Create phoenixd health check script + copy: + dest: "{{ phoenixd_healthcheck_script_path }}" + content: | + #!/bin/bash + # Checks phoenixd and pushes the result to Uptime Kuma. + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + export PHOENIX_DATADIR="{{ phoenixd_data_dir }}" + + check_phoenixd() { + # Service must be active and the node must answer getinfo. + # phoenix-cli reads the api password from $PHOENIX_DATADIR/phoenix.conf, + # but not the bind address, so pass it explicitly. + systemctl is-active --quiet phoenixd && \ + {{ phoenixd_bin_dir }}/phoenix-cli \ + --http-bind-ip {{ phoenixd_http_bind_ip }} \ + --http-bind-port {{ phoenixd_http_bind_port }} \ + getinfo 2>/dev/null | grep -q '"nodeId"' + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_phoenixd; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "phoenixd not responding" + exit 1 + fi + owner: root + group: root + mode: "0755" + + - name: Create phoenixd health check systemd service + copy: + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" + content: | + [Unit] + Description=phoenixd Health Check + After=network.target phoenixd.service + + [Service] + Type=oneshot + User=root + ExecStart={{ phoenixd_healthcheck_script_path }} + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + + - name: Create phoenixd health check systemd timer + copy: + dest: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.timer" + content: | + [Unit] + Description=phoenixd Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: "0644" + + - name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + + - name: Enable and start phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + enabled: yes + state: started + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for phoenixd + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_phoenixd_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import time + import traceback + import yaml + + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_phoenixd_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + try: + api.add_monitor(type='group', name='services') + except Exception: + time.sleep(2) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + push_url = None + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + else: + print(f"Creating push monitor '{monitor_name}'...") + try: + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=90, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + except Exception as e: + # socketio timeout: add_monitor may have succeeded server-side + print(f"add_monitor raised (possibly timeout): {e}", file=sys.stderr) + time.sleep(2) + + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + push_url = f"{url}/api/push/{push_token}" + + api.disconnect() + + if push_url: + print(f"PUSH_URL={push_url}") + with open('/tmp/phoenixd_push_url.txt', 'w') as f: + f.write(push_url) + + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: "0755" + + - name: Create temporary config for push monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_phoenixd_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "{{ phoenixd_monitor_name }}" + mode: "0644" + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_phoenixd_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URL from file + slurp: + src: /tmp/phoenixd_push_url.txt + delegate_to: localhost + become: no + register: push_url_file + ignore_errors: yes + + - name: Parse push URL + set_fact: + phoenixd_push_url: "{{ push_url_file.content | b64decode | trim }}" + when: push_url_file.content is defined + + - name: Update health check service with push URL + lineinfile: + path: "/etc/systemd/system/{{ phoenixd_healthcheck_service_name }}.service" + regexp: "^Environment=UPTIME_KUMA_PUSH_URL=" + line: "Environment=UPTIME_KUMA_PUSH_URL={{ phoenixd_push_url }}" + when: phoenixd_push_url is defined + notify: Restart phoenixd health check timer + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_phoenixd_monitor.py + - /tmp/ansible_phoenixd_config.yml + - /tmp/phoenixd_push_url.txt + + # =========================================== + # Post-install Notes + # =========================================== + - name: Display post-install information + debug: + msg: | + ✓ phoenixd {{ phoenixd_version }} deployed + + Status: systemctl status phoenixd + Logs: journalctl -u phoenixd -f + CLI: sudo PHOENIX_DATADIR={{ phoenixd_data_dir }} phoenix-cli --http-bind-port {{ phoenixd_http_bind_port }} getinfo + HTTP API: http://{{ phoenixd_http_bind_ip }}:{{ phoenixd_http_bind_port }} (loopback only) + Data dir: {{ phoenixd_data_dir }} + + API password (needed to wire LNBits up to this node): + sudo grep '^http-password=' {{ phoenixd_data_dir }}/phoenix.conf + + ⚠️ BACK UP THE SEED NOW: {{ phoenixd_data_dir }}/seed.dat + Losing it means losing the funds. Run + services/phoenixd/setup_backup_phoenixd_to_lapy.yml and also keep + the 12 words somewhere offline. + + handlers: + - name: Restart phoenixd + systemd: + name: phoenixd + state: restarted + daemon_reload: yes + + - name: Restart phoenixd health check timer + systemd: + name: "{{ phoenixd_healthcheck_service_name }}.timer" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/phoenixd/defaults/main.yml b/ansible/services/phoenixd/phoenixd_vars.yml similarity index 63% rename from ansible/roles/phoenixd/defaults/main.yml rename to ansible/services/phoenixd/phoenixd_vars.yml index 2fcf1ea..57218dc 100644 --- a/ansible/roles/phoenixd/defaults/main.yml +++ b/ansible/services/phoenixd/phoenixd_vars.yml @@ -35,22 +35,18 @@ phoenixd_http_bind_port: 9740 # Optional webhook for payment events. Leave empty to disable. phoenixd_webhook_url: "" - - - -# --- Health check ----------------------------------------------------------- -# The check asks phoenixd itself whether it is healthy (service active AND the -# node answers getinfo with a nodeId) and records the answer in its exit code, -# which systemd keeps: -# systemctl is-failed phoenixd-healthcheck.service -# That is a complete answer with no monitoring system involved. +# Monitoring phoenixd_healthcheck_script_path: /usr/local/bin/phoenixd-healthcheck-push.sh phoenixd_healthcheck_service_name: phoenixd-healthcheck +phoenixd_monitor_name: "Phoenixd" -# WHERE TO REPORT HEALTH — the one place to plug in monitoring. -# Empty means check, log, exit honestly, report nowhere. Any endpoint that -# accepts an HTTP ping works; nothing here is specific to a monitoring product. -healthcheck_push_url: "" -# Bearer token for the Gatus external endpoint. Required whenever a push URL -# is set: Gatus rejects an unauthenticated push with 401. -healthcheck_push_token: "" +# Remote access +remote_host_name: "vipy" +remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" +remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" +remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" +remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" + +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/phoenixd-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/phoenixd_backup.sh" diff --git a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml new file mode 100644 index 0000000..cc008e6 --- /dev/null +++ b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml @@ -0,0 +1,136 @@ +--- +# Backs up the phoenixd seed and config from vipy to Lapy, gpg encrypted. +# +# Only seed.dat and phoenix.conf are backed up, on purpose: +# - seed.dat is what actually recovers the funds. phoenixd keeps its channel +# state with the ACINQ peer, so a wallet is restored from the seed alone. +# - restoring a *stale* channel database to a running Lightning node is +# dangerous (it can trigger a force close and a penalty), so we do not keep +# copies of phoenix.db around to be tempted by. +# - phoenix.conf holds the http api password, which is what LNBits and any +# other local consumer authenticate with. +# +# Because both files are static, phoenixd does not need to be stopped. + +- name: Configure local backup for phoenixd from remote + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./phoenixd_vars.yml + vars: + remote_data_path: "{{ phoenixd_data_dir }}" + gpg_recipient: "{{ hostvars['localhost']['gpg_recipient'] | default('') }}" + gpg_key_id: "{{ hostvars['localhost']['gpg_key_id'] | default('') }}" + + tasks: + - name: Debug phoenixd backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + - "gpg_recipient={{ gpg_recipient }}" + - "gpg_key_id={{ gpg_key_id }}" + + - name: Ensure local backup directory exists + ansible.builtin.file: + path: "{{ local_backup_dir }}" + state: directory + mode: "0700" + + - name: Ensure ~/.local/bin exists + ansible.builtin.file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: "0755" + + - name: Create phoenixd backup script + ansible.builtin.copy: + dest: "{{ backup_script_path }}" + mode: "0750" + content: | + #!/bin/bash + set -euo pipefail + + if [ -z "{{ gpg_recipient }}" ]; then + echo "GPG recipient is not configured. Aborting." + exit 1 + fi + + TIMESTAMP=$(date +'%Y-%m-%d') + ENCRYPTED_BACKUP="{{ local_backup_dir }}/phoenixd-backup-$TIMESTAMP.tar.gz.gpg" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + # seed.dat + phoenix.conf only, see the header of the playbook. + echo "Creating encrypted backup archive..." + $SSH_CMD {{ remote_user }}@{{ remote_host }} \ + "sudo tar -czf - -C {{ remote_data_path }} seed.dat phoenix.conf" | \ + gpg --batch --yes --encrypt --recipient "{{ gpg_recipient }}" --output "$ENCRYPTED_BACKUP" + + chmod 600 "$ENCRYPTED_BACKUP" + + # Rotate old backups (keep 14 days) + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for backup_file in "{{ local_backup_dir }}"/phoenixd-backup-*.tar.gz.gpg; do + if [ -f "$backup_file" ]; then + # Extract date from filename: phoenixd-backup-YYYY-MM-DD.tar.gz.gpg + file_date=$(basename "$backup_file" | sed -n 's/phoenixd-backup-\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\)\.tar\.gz\.gpg/\1/p') + if [ -n "$file_date" ] && [ "$file_date" != "$TIMESTAMP" ] && [ "$file_date" \< "$CUTOFF_DATE" ]; then + rm -f "$backup_file" + fi + fi + done + + echo "Backup completed successfully" + + - name: Ensure cronjob for phoenixd backup exists + ansible.builtin.cron: + name: "phoenixd backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 15 + hour: "9" + + - name: Run phoenixd backup script to create initial backup + ansible.builtin.command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check if backup file exists + stat: + path: "{{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg" + register: backup_file_stat + + - name: Verify backup file exists + assert: + that: + - backup_file_stat.stat.exists + - backup_file_stat.stat.isreg + fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg was not created" + success_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists" + + - name: Verify backup file is not empty + assert: + that: + - backup_file_stat.stat.size > 0 + fail_msg: "Backup file {{ local_backup_dir }}/phoenixd-backup-{{ today_date.stdout }}.tar.gz.gpg exists but is empty" + success_msg: "Backup file size is {{ backup_file_stat.stat.size }} bytes" + + - name: Remind about the offline seed copy + debug: + msg: | + These encrypted backups are only as safe as your GPG key. + Also write the 12 words down offline once: + ssh {{ remote_user }}@{{ remote_host }} "sudo cat {{ remote_data_path }}/seed.dat" diff --git a/ansible/services/signal-api/deploy_signal_api_playbook.yml b/ansible/services/signal-api/deploy_signal_api_playbook.yml deleted file mode 100644 index d4690f9..0000000 --- a/ansible/services/signal-api/deploy_signal_api_playbook.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -# The Signal transport for Gatus alerts. -# -# Deliberately NOT published and NOT fronted by Caddy: the API has no -# authentication of any kind, so it is reachable only from the shared docker -# network that Gatus is on. See roles/signal_api/README.md, including the -# one-time manual step to link the device. -- name: Deploy the Signal API on the observability host - hosts: observability - become: yes - roles: - - signal_api - - # post_tasks, not a second play: the registration below needs the role's - # defaults (service name, port) in scope, and a separate play would not have - # them. - post_tasks: - # Monitored, because a dead alert transport is the worst kind of dead: every - # check could be failing and nothing would tell you. Gatus polls it over the - # shared network - the same path the alerts take - so this proves the actual - # delivery route rather than merely that a container is running. - # - # Deliberately NOT backed up: the data directory holds Signal private keys, - # and the recovery path is to link the device again from the phone. Backing - # it up would copy a credential off the host to buy nothing. - - name: Register the signal-api health endpoint with Gatus - ansible.builtin.include_role: - name: gatus_endpoint - vars: - gatus_endpoint_name: signal-api - gatus_endpoint_pulled: - - name: signal-api - group: infrastructure - url: "http://{{ signal_api_service_name }}:{{ signal_api_port }}/v1/health" - interval: 5m - # Deliberately NOT alerted via Signal: if this endpoint is down, - # Signal is exactly what cannot deliver the alert. It is visible on - # the dashboard, and its failure shows up indirectly as every other - # alert going missing. - conditions: - # /v1/health answers 204 No Content, not 200 - checked live. Any - # 2xx is asserted rather than the exact code, so an upstream - # change from 204 to 200 does not read as an outage. - - "[STATUS] < 300" diff --git a/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml b/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml similarity index 99% rename from archive/uptime_kuma/deploy_uptime_kuma_playbook.yml rename to ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml index 8754dce..4af3858 100644 --- a/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml +++ b/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Uptime Kuma with Docker Compose and configure Caddy reverse proxy - hosts: monitoring + hosts: watchtower become: yes vars_files: - ../../infra_vars.yml diff --git a/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml b/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml similarity index 99% rename from archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml rename to ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml index 54c0b79..9ae9713 100644 --- a/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml +++ b/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Uptime Kuma from remote - hosts: control + hosts: lapy gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/archive/uptime_kuma/uptime_kuma_vars.yml b/ansible/services/uptime_kuma/uptime_kuma_vars.yml similarity index 92% rename from archive/uptime_kuma/uptime_kuma_vars.yml rename to ansible/services/uptime_kuma/uptime_kuma_vars.yml index 33ffb6d..3263f49 100644 --- a/archive/uptime_kuma/uptime_kuma_vars.yml +++ b/ansible/services/uptime_kuma/uptime_kuma_vars.yml @@ -4,7 +4,7 @@ uptime_kuma_data_dir: "{{ uptime_kuma_dir }}/data" uptime_kuma_port: 3001 # Remote access -remote_host_name: "{{ groups['monitoring'] | first }}" +remote_host_name: "watchtower" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 282fc2e..0340538 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -1,11 +1,16 @@ - name: Deploy Vaultwarden with Docker Compose and configure Caddy reverse proxy - hosts: edge + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml - ./vaultwarden_vars.yml vars: vaultwarden_subdomain: "{{ subdomains.vaultwarden }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" vaultwarden_domain: "{{ vaultwarden_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create vaultwarden directory @@ -76,10 +81,143 @@ name: fail2ban state: restarted - - name: Publish Vaultwarden through Caddy - ansible.builtin.include_role: - name: caddy_site - vars: - caddy_site_name: vaultwarden - caddy_site_domain: "{{ vaultwarden_domain }}" - caddy_site_upstream: "localhost:{{ vaultwarden_port }}" + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Create Caddy reverse proxy configuration for vaultwarden + copy: + dest: "{{ caddy_sites_dir }}/vaultwarden.conf" + content: | + {{ vaultwarden_domain }} { + reverse_proxy localhost:{{ vaultwarden_port }} + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for Vaultwarden + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_vaultwarden_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ vaultwarden_domain }}/alive" + monitor_name: "Vaultwarden" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_vaultwarden_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_vaultwarden_monitor.py + - /tmp/ansible_config.yml + diff --git a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml index bccc2cd..b041e8e 100644 --- a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml +++ b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml @@ -1,7 +1,8 @@ - name: Disable Vaultwarden Signups - hosts: edge + hosts: vipy become: yes vars_files: + - ../../infra_vars.yml - ./vaultwarden_vars.yml tasks: diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml deleted file mode 100644 index d0dfdfa..0000000 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -# Vaultwarden backup: dumps locally on vipy, encrypted with age. -# Previously rsynced to lapy in the CLEAR; the artefact now never exists -# unencrypted, on disk or on the wire. -- name: Configure the Vaultwarden backup on the edge host - hosts: edge - become: yes - vars_files: - - ./vaultwarden_vars.yml - - tasks: - - name: Ensure Vaultwarden dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: vaultwarden - backup_source_description: "Vaultwarden" - backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data" - # Not systemd — a docker compose stack — so stop/start explicitly. - # sqlite in WAL mode, hence stopping at all. - backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" - backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:10:00" - # Reported to Gatus as backup-dump_vaultwarden. The token is this HOST's token, - # shared with its other checks - see infra/400_host_monitoring.yml. - backup_source_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/backup-dump_vaultwarden/external" - backup_source_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml new file mode 100644 index 0000000..064d633 --- /dev/null +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml @@ -0,0 +1,105 @@ +- name: Configure local backup for Vaultwarden from remote + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./vaultwarden_vars.yml + vars: + remote_data_path: "{{ vaultwarden_data_dir }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "remote_host={{ remote_host }}" + - "remote_user={{ remote_user }}" + - "remote_data_path='{{ remote_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if remote_key_file %} + SSH_CMD="ssh -i {{ remote_key_file }} -p {{ remote_port }}" + {% else %} + SSH_CMD="ssh -p {{ remote_port }}" + {% endif %} + + rsync -az -e "$SSH_CMD" --delete {{ remote_user }}@{{ remote_host }}:{{ remote_data_path }}/ "$BACKUP_DIR/" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for dir in "{{ local_backup_dir }}"/20*; do + if [ -d "$dir" ]; then + dir_date=$(basename "$dir") + if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then + rm -rf "$dir" + fi + fi + done + + - name: Ensure cronjob for backup exists + cron: + name: "Vaultwarden backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 5 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check backup directory exists and contains files + stat: + path: "{{ local_backup_dir }}/{{ today_date.stdout }}" + register: backup_dir_stat + + - name: Verify backup directory exists + assert: + that: + - backup_dir_stat.stat.exists + - backup_dir_stat.stat.isdir + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" + success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" + + - name: Check if backup directory contains files + find: + paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" + recurse: yes + register: backup_files + + - name: Verify backup directory is not empty + assert: + that: + - backup_files.files | length > 0 + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" + success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" diff --git a/ansible/services/vaultwarden/vaultwarden_vars.yml b/ansible/services/vaultwarden/vaultwarden_vars.yml index 9edc2cd..75e527d 100644 --- a/ansible/services/vaultwarden/vaultwarden_vars.yml +++ b/ansible/services/vaultwarden/vaultwarden_vars.yml @@ -3,12 +3,15 @@ vaultwarden_dir: /opt/vaultwarden vaultwarden_data_dir: "{{ vaultwarden_dir }}/data" vaultwarden_port: 8222 -# (subdomain in group_vars/all/main.yml, caddy_sites_dir in roles/caddy_site/defaults/) +# (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "{{ groups['edge'] | first }}" +remote_host_name: "vipy" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# Local backup +local_backup_dir: "{{ lookup('env', 'HOME') }}/vaultwarden-backups" +backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/vaultwarden_backup.sh" diff --git a/ansible/services_config.yml b/ansible/services_config.yml new file mode 100644 index 0000000..1bc9db0 --- /dev/null +++ b/ansible/services_config.yml @@ -0,0 +1,39 @@ +# Centralized Services Configuration +# Subdomains and Caddy settings for all services + +# Edit these subdomains to match your preferences +subdomains: + # Monitoring Services (on watchtower) + ntfy: ntfy + uptime_kuma: uptime + + # VPN Infrastructure (on spacey) + headscale: headscale + + # Core Services (on vipy) + vaultwarden: vault + forgejo: forgejo + lnbits: wallet + + # Secondary Services (on vipy) + ntfy_emergency_app: avisame + personal_blog: pablohere + + # Memos (on memos-box) + memos: memos + + # Mempool Block Explorer (on mempool_box, proxied via vipy) + mempool: mempool + + # DATUM Gateway dashboard (on knots_box, proxied via vipy) + datum_gateway: datum + +# Caddy configuration +caddy_sites_dir: /etc/caddy/sites-enabled + +# Service-specific settings shared across playbooks +service_settings: + ntfy: + topic: alerts + headscale: + namespace: counter-net diff --git a/ansible/site.yml b/ansible/site.yml deleted file mode 100644 index 35f5867..0000000 --- a/ansible/site.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -# Everything, in the order it has to happen. -# -# This file is a TABLE OF CONTENTS, not a second source of truth. It says what -# runs and in what order. It does NOT say which hosts get what — that stays on -# the `hosts:` line inside each playbook, exactly where it is today. Nothing -# moves; this file only makes the set readable in one place. -# -# What runs on a host? ansible-playbook site.yml --limit --list-hosts -# Who gets thing Y? the `hosts:` line in Y's own playbook -# What is a host? ansible-inventory --graph -# -# Run a slice with --limit, or run one playbook directly as before. Nothing here -# changes how any individual playbook behaves. - -# ── Baseline: every managed machine ───────────────────────────────────────── -- import_playbook: infra/01_user_and_access_setup_playbook.yml -- import_playbook: infra/02_firewall_and_fail2ban_playbook.yml -- import_playbook: infra/900_install_rsync.yml -- import_playbook: infra/920_join_headscale_mesh.yml -# Idempotent and kept permanently: guarantees a rebuilt or restored host cannot -# quietly bring the Uptime-Kuma-era monitoring back. -- import_playbook: infra/409_remove_legacy_monitoring.yml - -# ── Monitoring ────────────────────────────────────────────────────────────── -# Gatus first: the three plays below register endpoints with it, and registering -# against a host that is not serving yet would simply fail. -- import_playbook: services/gatus/deploy_gatus_playbook.yml -# The Signal transport for Gatus alerts. Shares a docker network with Gatus and -# publishes no ports - the API has no authentication. Needs a one-time manual -# device link; see roles/signal_api/README.md. -- import_playbook: services/signal-api/deploy_signal_api_playbook.yml -- import_playbook: infra/400_host_monitoring.yml -- import_playbook: infra/401_service_monitoring.yml -- import_playbook: infra/402_public_monitoring.yml -# Registers where the per-service probes report. The probes themselves are -# deployed by each service's own playbook further down; the endpoints must exist -# before the first push arrives. -- import_playbook: infra/403_service_probe_registration.yml -# 910_docker says `hosts: managed`, but only 5 of 11 managed hosts have or need -# Docker. Left out until it has a [docker] group — see the note in PLAN_7. - -# ── The hypervisor ────────────────────────────────────────────────────────── -- import_playbook: infra/nodito/31_proxmox_community_repos_playbook.yml -- import_playbook: infra/nodito/32_zfs_pool_setup_playbook.yml -- import_playbook: infra/nodito/34_nut_ups_setup_playbook.yml - -# ── Reverse proxy, before anything that registers a vhost ─────────────────── -- import_playbook: services/caddy_playbook.yml - -# ── Services ──────────────────────────────────────────────────────────────── -- import_playbook: services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml -- import_playbook: services/fulcrum/deploy_fulcrum_playbook.yml -- import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml -- import_playbook: services/mempool/deploy_mempool_playbook.yml -- import_playbook: services/memos/deploy_memos_playbook.yml -- import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml -- import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml -- import_playbook: services/headscale/deploy_headscale_playbook.yml -- import_playbook: services/vaultwarden/deploy_vaultwarden_playbook.yml -- import_playbook: services/forgejo/deploy_forgejo_playbook.yml -- import_playbook: services/lnbits/deploy_lnbits_playbook.yml -- import_playbook: services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml -- import_playbook: services/personal-blog/deploy_personal_blog_playbook.yml - -# ── Backups: each source dumps itself, the box pulls ──────────────────────── -- import_playbook: services/headscale/setup_backup_headscale.yml -- import_playbook: services/vaultwarden/setup_backup_vaultwarden.yml -- import_playbook: services/forgejo/setup_backup_forgejo.yml -- import_playbook: services/lnbits/setup_backup_lnbits.yml -- import_playbook: services/memos/setup_backup_memos.yml -- import_playbook: playbooks/backups.yml - -# Deliberately not here. Every playbook in the repo is either imported above or -# listed below, so this file accounts for all of them: -# -# infra/910_docker_playbook.yml says `hosts: managed`, but Docker is on 5 -# of 11 managed hosts and those 5 are exactly -# the ones that need it. Running it would -# install Docker on the Bitcoin node and the -# hypervisor. Needs a [docker] group first. -# -# infra/nodito/30_proxmox_bootstrap one-shot: bare-metal bootstrap, run once -# infra/nodito/33_..._cloud_template one-shot: builds the VM template -# -# -# services/vaultwarden/disable_ deliberate manual actions, not convergence -# vaultwarden_sign_ups_playbook.yml -# services/personal-blog/setup_ -# deploy_alias_lapy.yml diff --git a/archive/uptime_kuma/MONITORS.md b/archive/uptime_kuma/MONITORS.md deleted file mode 100644 index 53b1d89..0000000 --- a/archive/uptime_kuma/MONITORS.md +++ /dev/null @@ -1,152 +0,0 @@ -# Uptime Kuma — monitor inventory (archived) - -Captured from the live instance at `https://uptime.contrapeso.xyz` on 2026-09-11, -immediately before decommissioning. This is the **authoritative** record: most of -these monitors existed only in the Uptime Kuma UI and were never described by any -playbook in this repo. - -**75 monitors total** — 16 group, 8 http, 3 port, 48 push. All were active. - -Push tokens are deliberately **not** recorded here: they are live credentials, and -anything holding one could report a false 'up'. They die with the server. - ---- - -## arbret - production *(7 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| arbret.com - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | -| arbret.com - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | -| arbret.com - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | -| arbret.com - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | -| arbret.com - health | push | — | 120s | Healthy when GET /api/health returns status ok | -| arbret.com - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | -| arbret.com - postgresql | push | — | 120s | Healthy when postgresql.service is active | - -## arbret - staging *(7 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| arbretstaging.contrapeso.xyz - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | -| arbretstaging.contrapeso.xyz - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | -| arbretstaging.contrapeso.xyz - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | -| arbretstaging.contrapeso.xyz - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | -| arbretstaging.contrapeso.xyz - health | push | — | 120s | Healthy when GET /api/health returns status ok | -| arbretstaging.contrapeso.xyz - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | -| arbretstaging.contrapeso.xyz - postgresql | push | — | 120s | Healthy when postgresql.service is active | - -## arbret-staging-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-arbret-staging-box-root | push | — | 960s | upside-down, Disk Usage: arbret-staging-box (/) - Alerts when usage excee | -| system-healthcheck-arbret-staging-box | push | — | 90s | System Healthcheck: arbret-staging-box - Regular healthcheck | - -## forgejo-runner-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-forgejo-runner-box-root | push | — | 960s | upside-down, Disk Usage: forgejo-runner-box (/) - Alerts when usage excee | -| system-healthcheck-forgejo-runner-box | push | — | 90s | System Healthcheck: forgejo-runner-box - Regular healthcheck | - -## fulcrum-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-fulcrum-box-root | push | — | 960s | upside-down, Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80% | -| system-healthcheck-fulcrum-box | push | — | 90s | System Healthcheck: fulcrum-box - Regular healthcheck ping e | - -## knots-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-knots-box-root | push | — | 960s | upside-down, Disk Usage: knots-box (/) - Alerts when usage exceeds 80% | -| system-healthcheck-knots-box | push | — | 90s | System Healthcheck: knots-box - Regular healthcheck ping eve | - -## memos-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-memos-box-root | push | — | 960s | upside-down, Disk Usage: memos-box (/) - Alerts when usage exceeds 80% | -| system-healthcheck-memos-box | push | — | 90s | System Healthcheck: memos-box - Regular healthcheck ping eve | - -## mempool-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-mempool-box-root | push | — | 960s | upside-down, Disk Usage: mempool-box (/) - Alerts when usage exceeds 80% | -| system-healthcheck-mempool-box | push | — | 90s | System Healthcheck: mempool-box - Regular healthcheck ping e | - -## nodito - infra *(4 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| UPS ONLINE | push | https:// | 90s | — | -| cpu-temp-nodito | push | — | 120s | upside-down, CPU Temperature: nodito - Alerts when temperature exceeds 80 | -| system-healthcheck-nodito | push | — | 300s | System Healthcheck: nodito - Regular healthcheck ping every | -| zfs-health-nodito | push | — | 90000s | ZFS Pool Health: nodito - Daily health check for pool proxmo | - -## nonkeiwaisi-box - infra *(0 monitors)* - -_(empty)_ - -## prd-arbret - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-prd-arbret-root | push | — | 960s | upside-down, Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80% | -| system-healthcheck-prd-arbret | push | — | 90s | System Healthcheck: prd-arbret - Regular healthcheck ping ev | - -## prd-spacey - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-prd-spacey-root | push | — | 960s | upside-down, Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80% | -| system-healthcheck-prd-spacey | push | — | 90s | System Healthcheck: prd-spacey - Regular healthcheck ping ev | - -## prd-vipy - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-prd-vipy-root | push | — | 960s | upside-down, Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80% | -| system-healthcheck-prd-vipy | push | — | 90s | System Healthcheck: prd-vipy - Regular healthcheck ping ever | - -## prd-watchtower - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-prd-watchtower-root | push | — | 960s | upside-down, Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 8 | -| system-healthcheck-prd-watchtower | push | — | 90s | System Healthcheck: prd-watchtower - Regular healthcheck pin | - -## services *(19 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| Forgejo | http | https://forgejo.contrapeso.xyz/api/healthz | 90s | — | -| Headscale | http | https://headscale.contrapeso.xyz/health | 60s | — | -| LNBits | http | https://wallet.contrapeso.xyz/api/v1/health | 60s | — | -| Memos | http | https://memos.contrapeso.xyz/healthz | 60s | — | -| Mempool Public Access | http | https://mempool.contrapeso.xyz | 60s | — | -| Personal Blog | http | https://pablohere.contrapeso.xyz | 60s | — | -| Vaultwarden | http | https://vault.contrapeso.xyz/alive | 60s | — | -| ntfy-emergency-app | http | https://avisame.contrapeso.xyz | 60s | — | -| Bitcoin Knots P2P Public | port | 167.172.107.33:8333 | 60s | — | -| DATUM Stratum (public) | port | 167.172.107.33:23334 | 60s | — | -| Fulcrum SSL Public | port | 167.172.107.33:50002 | 60s | — | -| Bitcoin Knots | push | — | 90s | — | -| DATUM Gateway | push | — | 90s | — | -| Fulcrum | push | — | 90s | — | -| Mempool Backend | push | — | 180s | — | -| Mempool Frontend | push | — | 90s | — | -| Mempool MariaDB | push | — | 90s | — | -| Phoenixd | push | — | 90s | — | -| forgejo-runner-healthcheck | push | — | 90s | Forgejo Runner healthcheck - ping every 60s | - -## small-backups-box - infra *(2 monitors)* - -| Monitor | Type | Target | Interval | Notes | -|---|---|---|---|---| -| disk-usage-small-backups-box-root | push | — | 960s | upside-down, Disk Usage: small-backups-box (/) - Alerts when usage exceed | -| system-healthcheck-small-backups-box | push | — | 90s | System Healthcheck: small-backups-box - Regular healthcheck | - diff --git a/archive/uptime_kuma/README.md b/archive/uptime_kuma/README.md deleted file mode 100644 index 5405ae0..0000000 --- a/archive/uptime_kuma/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Uptime Kuma — archived - -Uptime Kuma was the monitoring stack for this infrastructure until **2026-09-11**, when -it was decommissioned. Everything that referenced it has been removed from the live -playbooks; this folder is the record of what it was, kept so the setup can be understood -later without digging through git history. - -## Contents - -| File | What it is | -|---|---| -| `MONITORS.md` | Every monitor that existed, grouped as it was in the UI. The authoritative record. | -| `monitors.json` | The same data, machine-readable, as returned by the API. | -| `deploy_uptime_kuma_playbook.yml` | How the server itself was deployed (Docker Compose on `monitoring`, behind Caddy). | -| `uptime_kuma_vars.yml` | Variables the deploy playbook needed — it is unreadable without these. | -| `setup_backup_uptime_kuma_to_lapy.yml` | How its data was backed up, and to where. Useful when disposing of the old volumes. | - -## Why the inventory was captured from the live server, not the repo - -The playbooks only ever created **17** monitors. The live instance had **75**. The -difference was created by hand in the UI and existed nowhere else — so a repo-derived -list would have silently lost two thirds of the picture. `MONITORS.md` is a snapshot of -the real thing, taken immediately before removal. - -Push tokens are deliberately excluded. They are live credentials — anything holding one -can report a false "up" — and they become meaningless once the server is gone. - -## What removal did NOT do - -Removing the playbook code does not touch the machines. **48 push monitors** were driven -by scripts and systemd timers installed *on the hosts*, which keep firing on their -schedule and curling an endpoint that no longer answers. They are harmless but they are -still there, writing logs and failing quietly. - -Left behind, per host: - -- `/opt/disk-monitoring` + `disk-usage-monitor.{service,timer}` — on all 12 `managed` hosts -- `/opt/system-healthcheck` + `system-healthcheck.{service,timer}` — on all 12 `managed` hosts -- `/opt/nodito-monitoring` + `nodito-cpu-temp-monitor.{service,timer}` — `nodito` -- `/opt/zfs-monitoring` — `nodito` -- `/opt/ups-monitoring` — `nodito` -- `bitcoin-knots-healthcheck.{service,timer}` — `bitcoin` -- `datum-gateway-healthcheck.{service,timer}` — `bitcoin` -- `fulcrum-healthcheck.{service,timer}` — `electrum` -- `mempool-{backend,frontend,mariadb}-healthcheck.service` — `mempool` -- phoenixd and forgejo-runner healthcheck units — `edge`, `ci_runner` - -Note `nut-monitor.service` on `nodito` is **NUT's own daemon**, not a monitoring -leftover — do not remove it with the rest. - -Cleaning these up is a separate decommissioning pass and was not part of the removal. diff --git a/archive/uptime_kuma/monitors.json b/archive/uptime_kuma/monitors.json deleted file mode 100644 index d1b0b0a..0000000 --- a/archive/uptime_kuma/monitors.json +++ /dev/null @@ -1,1202 +0,0 @@ -[ - { - "id": 1, - "name": "services", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 3, - "name": "Vaultwarden", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://vault.contrapeso.xyz/alive", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 4, - "name": "prd-spacey - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 5, - "name": "prd-watchtower - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 6, - "name": "prd-vipy - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 7, - "name": "disk-usage-prd-vipy-root", - "type": "push", - "active": true, - "parent": 6, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 8, - "name": "disk-usage-prd-watchtower-root", - "type": "push", - "active": true, - "parent": 5, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 9, - "name": "disk-usage-prd-spacey-root", - "type": "push", - "active": true, - "parent": 4, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 10, - "name": "system-healthcheck-prd-vipy", - "type": "push", - "active": true, - "parent": 6, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: prd-vipy - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 11, - "name": "system-healthcheck-prd-spacey", - "type": "push", - "active": true, - "parent": 4, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: prd-spacey - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 12, - "name": "system-healthcheck-prd-watchtower", - "type": "push", - "active": true, - "parent": 5, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: prd-watchtower - Regular healthcheck ping every 60s", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 13, - "name": "Forgejo", - "type": "http", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": "https://forgejo.contrapeso.xyz/api/healthz", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 10, - "resendInterval": 0 - }, - { - "id": 14, - "name": "Personal Blog", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://pablohere.contrapeso.xyz", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 15, - "name": "LNBits", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://wallet.contrapeso.xyz/api/v1/health", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 16, - "name": "ntfy-emergency-app", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://avisame.contrapeso.xyz", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 17, - "name": "Headscale", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://headscale.contrapeso.xyz/health", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 18, - "name": "knots-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 19, - "name": "disk-usage-knots-box-root", - "type": "push", - "active": true, - "parent": 18, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: knots-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 20, - "name": "system-healthcheck-knots-box", - "type": "push", - "active": true, - "parent": 18, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: knots-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 21, - "name": "nodito - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 22, - "name": "system-healthcheck-nodito", - "type": "push", - "active": true, - "parent": 21, - "interval": 300, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: nodito - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 23, - "name": "cpu-temp-nodito", - "type": "push", - "active": true, - "parent": 21, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "CPU Temperature: nodito - Alerts when temperature exceeds 80\u00b0C", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 24, - "name": "Bitcoin Knots", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 25, - "name": "fulcrum-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 26, - "name": "disk-usage-fulcrum-box-root", - "type": "push", - "active": true, - "parent": 25, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 27, - "name": "system-healthcheck-fulcrum-box", - "type": "push", - "active": true, - "parent": 25, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: fulcrum-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 28, - "name": "Fulcrum", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 35, - "name": "mempool-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 36, - "name": "disk-usage-mempool-box-root", - "type": "push", - "active": true, - "parent": 35, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: mempool-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 37, - "name": "system-healthcheck-mempool-box", - "type": "push", - "active": true, - "parent": 35, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: mempool-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 38, - "name": "Mempool MariaDB", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 39, - "name": "Mempool Backend", - "type": "push", - "active": true, - "parent": 1, - "interval": 180, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 40, - "name": "Mempool Frontend", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 41, - "name": "Mempool Public Access", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://mempool.contrapeso.xyz", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 42, - "name": "memos-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 43, - "name": "disk-usage-memos-box-root", - "type": "push", - "active": true, - "parent": 42, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: memos-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 44, - "name": "system-healthcheck-memos-box", - "type": "push", - "active": true, - "parent": 42, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: memos-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 45, - "name": "Memos", - "type": "http", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": "https://memos.contrapeso.xyz/healthz", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 46, - "name": "Bitcoin Knots P2P Public", - "type": "port", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": null, - "hostname": "167.172.107.33", - "port": 8333, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 47, - "name": "Fulcrum SSL Public", - "type": "port", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": null, - "hostname": "167.172.107.33", - "port": 50002, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 49, - "name": "zfs-health-nodito", - "type": "push", - "active": true, - "parent": 21, - "interval": 90000, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "ZFS Pool Health: nodito - Daily health check for pool proxmox-tank-1", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 50, - "name": "UPS ONLINE", - "type": "push", - "active": true, - "parent": 21, - "interval": 90, - "retries": null, - "url": "https://", - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 51, - "name": "forgejo-runner-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 52, - "name": "disk-usage-forgejo-runner-box-root", - "type": "push", - "active": true, - "parent": 51, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: forgejo-runner-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 53, - "name": "system-healthcheck-forgejo-runner-box", - "type": "push", - "active": true, - "parent": 51, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: forgejo-runner-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 55, - "name": "forgejo-runner-healthcheck", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Forgejo Runner healthcheck - ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 59, - "name": "arbret-staging-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 60, - "name": "disk-usage-arbret-staging-box-root", - "type": "push", - "active": true, - "parent": 59, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: arbret-staging-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 61, - "name": "system-healthcheck-arbret-staging-box", - "type": "push", - "active": true, - "parent": 59, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: arbret-staging-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 66, - "name": "arbret - staging", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 67, - "name": "arbretstaging.contrapeso.xyz - health", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when GET /api/health returns status ok", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 68, - "name": "arbretstaging.contrapeso.xyz - https", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when HTTPS front-door returns 200", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 69, - "name": "arbretstaging.contrapeso.xyz - postgresql", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when postgresql.service is active", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 70, - "name": "arbretstaging.contrapeso.xyz - arbret-server", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when arbret-server.service is active", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 71, - "name": "arbretstaging.contrapeso.xyz - arbret-worker", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when arbret-worker.service is active", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 72, - "name": "arbretstaging.contrapeso.xyz - arbret-analytics", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when timer is scheduled and last run succeeded", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 73, - "name": "arbretstaging.contrapeso.xyz - arbret-backup", - "type": "push", - "active": true, - "parent": 66, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when timer is scheduled and last run succeeded", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 74, - "name": "prd-arbret - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 85, - "name": "disk-usage-prd-arbret-root", - "type": "push", - "active": true, - "parent": 74, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 86, - "name": "system-healthcheck-prd-arbret", - "type": "push", - "active": true, - "parent": 74, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: prd-arbret - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 87, - "name": "arbret - production", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 88, - "name": "arbret.com - health", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when GET /api/health returns status ok", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 89, - "name": "arbret.com - https", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when HTTPS front-door returns 200", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 90, - "name": "arbret.com - postgresql", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when postgresql.service is active", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 91, - "name": "arbret.com - arbret-server", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when arbret-server.service is active", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 92, - "name": "arbret.com - arbret-worker", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when arbret-worker.service is active", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 93, - "name": "arbret.com - arbret-analytics", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when timer is scheduled and last run succeeded", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 94, - "name": "arbret.com - arbret-backup", - "type": "push", - "active": true, - "parent": 87, - "interval": 120, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "Healthy when timer is scheduled and last run succeeded", - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 95, - "name": "small-backups-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 96, - "name": "disk-usage-small-backups-box-root", - "type": "push", - "active": true, - "parent": 95, - "interval": 960, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": true, - "description": "Disk Usage: small-backups-box (/) - Alerts when usage exceeds 80%", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 97, - "name": "system-healthcheck-small-backups-box", - "type": "push", - "active": true, - "parent": 95, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": "System Healthcheck: small-backups-box - Regular healthcheck ping every 60s", - "maxretries": 1, - "resendInterval": 0 - }, - { - "id": 109, - "name": "DATUM Gateway", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 111, - "name": "DATUM Stratum (public)", - "type": "port", - "active": true, - "parent": 1, - "interval": 60, - "retries": null, - "url": null, - "hostname": "167.172.107.33", - "port": 23334, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 112, - "name": "Phoenixd", - "type": "push", - "active": true, - "parent": 1, - "interval": 90, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 3, - "resendInterval": 0 - }, - { - "id": 113, - "name": "nonkeiwaisi-box - infra", - "type": "group", - "active": true, - "parent": null, - "interval": 60, - "retries": null, - "url": null, - "hostname": null, - "port": null, - "upsideDown": false, - "description": null, - "maxretries": 1, - "resendInterval": 0 - } -] diff --git a/requirements.txt b/requirements.txt index 653523e..972dc4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,4 @@ packaging==25.0 pycparser==2.22 PyYAML==6.0.2 resolvelib==1.0.1 -# Only needed by the deprecated Uptime Kuma monitoring blocks, which are kept for -# reference but never run (uptime_kuma_enabled: false). Drop this once they are -# rewired to a replacement. See archive/uptime_kuma/. uptime-kuma-api>=1.2.1 diff --git a/tofu/nodito/.terraform.lock.hcl b/tofu/nodito/.terraform.lock.hcl deleted file mode 100644 index fadff38..0000000 --- a/tofu/nodito/.terraform.lock.hcl +++ /dev/null @@ -1,24 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/telmate/proxmox" { - version = "3.0.2-rc05" - constraints = "3.0.2-rc05" - hashes = [ - "h1:QfHovHn8h9uJXdJ+urOuiD7R46OXmdZQiRcCBaV6AD4=", - "zh:042d748367f33aaf440698644be4f2a2875f9db31915c1ef84616f176fc6174f", - "zh:1488781da1920d60d933c8ce926c34b5e989ffae58e3fbe437973d2b1d2faafc", - "zh:283dd6f74627f1d1d75d616b31f8ced3f97fd5277a07c9535e85cfa765d7a321", - "zh:378f1c2da21aeea083ac2e632db274a02c7a01e2486a40d3c813d05a21142db3", - "zh:38d63d0961f8c32273392caaace30f50cff8ab06e5dda17f67a8827ebffeba98", - "zh:52159782df101ec98f20faff81e8f2d9d92cb4ec903314fcddcc57ec16cdaacb", - "zh:6ca47b90c66b1d2706cb3cbb05da8b3f90a202c4865010202b2962e2b64d217e", - "zh:6e7b85cb2380e4dc0be694dd0e4a24927f7f66df41960eca3cfe907443d4f0b9", - "zh:758775f733673ab5c196db6a33648458037746f94d4bef7ce148cb01474efe2d", - "zh:7c31a3ca6d52db39da2bdd60be37af71d59d808fc206de50fe661535ea436da3", - "zh:af16984350a2f4d77c21f66a479007801e2527543310567c99cd82eb421e249e", - "zh:c1f965d3f96cf3f87af2c12ab9d4bde42f8ef660f8dc34ba3cfc9b20435a7269", - "zh:c2b9022a31103919a5ffbac6ee8d7feb6c4f5f580c1766f769569c2e8e4ce7f1", - "zh:e90162c42f1237323291e3d0de0c62701b3f89350fae18246da06702f41a6123", - ] -} diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md index bff6585..3a0b18f 100644 --- a/tofu/nodito/README.md +++ b/tofu/nodito/README.md @@ -27,17 +27,16 @@ This directory lets you declare VMs on the `nodito` Proxmox node and apply with - The Ansible template exists: `debian-13-cloud-init` (VMID 9001 by default). ### Provider Auth -Credentials are the only thing not in git. Copy `terraform.tfvars.example` to -`terraform.tfvars` (gitignored) and set: +Create a `terraform.tfvars` (copy from `terraform.tfvars.example`) and set: - `proxmox_api_url` (e.g. `https://nodito:8006/api2/json`) - `proxmox_api_token_id` (e.g. `root@pam!tofu`) - `proxmox_api_token_secret` +- `ssh_authorized_keys` (your public key content) -Alternatively, export them as `TF_VAR_proxmox_api_token_secret` etc. +Alternatively, you can export env vars and reference them in a tfvars file. ### Declare VMs -VMs are declared in `vms.auto.tfvars`, which is committed. `*.auto.tfvars` is -loaded automatically, so it needs no `-var-file`. Example entry: +Edit `terraform.tfvars` and fill the `vms` map. Example entry: ``` vms = { web1 = { @@ -55,24 +54,15 @@ All VM disks are created on `zfs_storage_name` (defaults to `proxmox-tank-1`). N ### Usage ``` tofu init -tofu plan -tofu apply +tofu plan -var-file=terraform.tfvars +tofu apply -var-file=terraform.tfvars ``` -`terraform.tfvars` and `vms.auto.tfvars` are both auto-loaded. > VMs are created once and then protected: the module sets `lifecycle.prevent_destroy = true` and ignores subsequent config changes. After the initial apply, manage day‑2 changes directly in Proxmox (or remove the lifecycle block if you need OpenTofu to own ongoing updates). ### Notes - Clones are full clones by default (`full_clone = true`). - Cloud-init injects `cloud_init_user` and `ssh_authorized_keys`. -- `.terraform.lock.hcl` is committed: it pins the provider hashes, which matters - because `versions.tf` tracks a release candidate (`3.0.2-rc05`). -- State is local (`terraform.tfstate`, gitignored) and has no remote backend, so - it exists only on the machine that last ran `tofu apply`. -- The map is not a complete inventory of nodito: state also manages - `bastion-box` (1100) and `nonkeiwaisi-box` (3300), which are not declared in - `vms.auto.tfvars`. `tofu plan` is clean today, but relaxing the `lifecycle` - block without first declaring them would put them up for destruction. - Disks use `scsi0` on ZFS with `discard` enabled. diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example index 37b6d7a..cc88b3f 100644 --- a/tofu/nodito/terraform.tfvars.example +++ b/tofu/nodito/terraform.tfvars.example @@ -1,7 +1,35 @@ -# Provider credentials. Copy to terraform.tfvars (gitignored) and fill in. -# Everything else — the node, storage, template, SSH key and the vms map — -# is committed in vms.auto.tfvars and needs no local copy. +proxmox_api_url = "https://nodito:8006/api2/json" +proxmox_api_token_id = "root@pam!tofu" +proxmox_api_token_secret = "REPLACE_ME" + +proxmox_node = "nodito" +zfs_storage_name = "proxmox-tank-1" +template_name = "debian-13-cloud-init" +cloud_init_user = "counterweight" + +# paste your ~/.ssh/id_ed25519.pub or similar +ssh_authorized_keys = <