From c3f5e47559bd19010a6925d0e8336d5213f1a7c0 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 17:42:49 +0200 Subject: [PATCH 01/44] add ansible.cfg --- ansible/ansible.cfg | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ansible/ansible.cfg diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..56620fa --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,14 @@ +[defaults] +inventory = inventory.ini +roles_path = roles +collections_path = collections +interpreter_python = auto_silent +stdout_callback = yaml +retry_files_enabled = False +host_key_checking = True +forks = 10 +# vault_password_file = .vault_pass # uncomment in Stage 2 + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=300s From 6029317f3bfe3475bb4496bfd399a2833d4531a6 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 17:53:12 +0200 Subject: [PATCH 02/44] ansible: vault encrypt secrets --- ansible/infra/nodito/nodito_secrets.yml | 17 +++++++++ ansible/infra_secrets.yml | 50 +++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 ansible/infra/nodito/nodito_secrets.yml create mode 100644 ansible/infra_secrets.yml diff --git a/ansible/infra/nodito/nodito_secrets.yml b/ansible/infra/nodito/nodito_secrets.yml new file mode 100644 index 0000000..7983cdb --- /dev/null +++ b/ansible/infra/nodito/nodito_secrets.yml @@ -0,0 +1,17 @@ +$ANSIBLE_VAULT;1.1;AES256 +33303038356165616435663034356134363963323865356133353239373263623430373263613430 +6463323436386666633666333331636132653963663735330a333063303038313839653330303231 +37313332653033636265353730616431356337366566656533363838303939636266626565333365 +3764316366653361310a386330326335343936356430333864336431376230323734303030613032 +31343737323636363366623530353433313862663038376439343331633038363138316639613531 +63663866613037336630316464383464613165326632376337383338336432326564396261383561 +61313337336466653431613338656136653862646662323435396462623366366438626566373331 +34663433333965626161356636633930313961313531656233303638346265316232326530613062 +31343732636466393334623734346639636637363831393337363562343461616639316235336362 +62306464383263626235613533336130326232613563336630373661623637303961353863303633 +66306664653035363236323063616363313739633633373231333833393562633666356530383065 +36373830393965663861663231633838303863363763613663363734383961623466653735636138 +34376233363831363732643631646364363066626230353330316562376337633462323966376239 +30666263326263366563323739653538616462303566633037393861613664646331633264613338 +35636131383937393430666430646530663866626438643430316430376133653135333838636436 +37613465666265373336 diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml new file mode 100644 index 0000000..c48d456 --- /dev/null +++ b/ansible/infra_secrets.yml @@ -0,0 +1,50 @@ +$ANSIBLE_VAULT;1.1;AES256 +32643038333165373363353439313639326538343435393633343765386662643231333936386630 +6564313061366166353662373564393261396666663432610a633161643066663232336163306663 +61623535643231366638616630666562333936383232373130623631306334653634386132396431 +3036383735316339650a313565313062636461396563336461633838343037666237623038633133 +35333531313036666531633230646464636631666133313636353362646336623135363736316562 +36656564396230333231643266303833633134303862323633636136363136373836346363663364 +32306234393863613534663031646632643734366531343339386133363837396138616435313365 +65353037326266386538633665386638656539376133636435346136643163643238656434346132 +34363036623935343162313031343036393536613136306435323730346561393261333966663465 +35333864323638356565316632353662366262363962376134366432333361343639633133626533 +65303964643330393264643136346432313164396664333535363135646464326539626433623732 +34666530666234376637623930633433656163353136646134326365353563343833666161666435 +30313866396136623036613933376164346533383964303664633830376636363565323832343136 +62653137366632313133626234386235356630383433623934303737653438623136366638396237 +66303435356539343061313337646261343738373866646465373339616161323531346361383363 +64643363616130323935656264326631336231323236616438366632356333333534303861663238 +61376464623030363938653230316265366533623364653930343537326532393433363230356136 +62643135636130626436386239366561393537333736383361336162636434656339653630353732 +37343265366234656637336135633461333735363932636130323564623661303730303363383833 +34373963373765643437303534366233626662656531373261633565353137643761356135376430 +30373639336565356364313333333237323338346631666235613961656336643438653834313137 +35303666366433646334643264633837343134373637643535633864313062613336333538303931 +34396266326166666165333639363961346639343130656235323332346631643539363861666436 +63343733656362313165623130383838376138613534363535336662366564306132353362666463 +61613534613431623263323831646635323362343634616566393834353964643861633639346132 +66363233373964323139313966616365343535306365373532313063383236353964343265383761 +64663833363035363736643236373634663466666138346266643930643033343839313534376666 +37363461623965636164663339386237643635353237353263633131363434643066393538363639 +39346530366361656536383233663766616234383535643234616233666262353364633036663062 +64386663383363633365656631326332353261303562346666636565313564306635333562363930 +33306237333638323131613161336165636237623332643137363038303330313663346639326361 +39383565366431363361313361303436343761626462306537633337393636333135343833623464 +32323663616166653063303861396664613434643536633066626466303634663031623938313436 +32303763616562376137303061633864393863373334373338626566653633383132306635396566 +64616463663265643732366465666662663334666362383036326330356535333465326534363266 +33326265326365353231613839383336656562333837376565306230646466333238323936316463 +64396266306562303438363635343865376364333239376664346534636664326532353265313563 +66636331333430616462326334623637646539643638323036303335656632363565373739373234 +35666532346364376335346238376562336164386363623862356539316466336531643936336131 +31356332306639626532646338393638373764653961616638643066313832626561336435383839 +64363232313633643133383066636539663933323064366536666462646466323166306562623538 +64393835383431663561343231303961383337656331663261393439336136623066666434336362 +36313239336537376134396361643532633437633163373931353139303534356665383538393838 +31306336326538353232336533643961383337313161663266353730643931656165353264303938 +30636363666136653963306130666433633562313464653862656262313035306635306230653666 +34336631623235323734616163623039613636393863383966336435313230313231356563636337 +61373336336563653030366130356331616438336638323263663164366435653266643161653331 +31323733313936313833386665633235633339376330373161326361336266633838326336356363 +37383165393866656565316337386639346531323439626465613834303636663933 From 9db0b0ae847dd9f3d5057513612e0544a18104ec Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 17:59:41 +0200 Subject: [PATCH 03/44] now using password --- .gitignore | 11 ++++++----- ansible/ansible.cfg | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 6c0a084..750a233 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,9 @@ inventory.ini venv/* .env -# Secrets and sensitive files -*_secrets.yml -*_secrets.yaml -secrets/ -.secrets/ +# Secrets are ansible-vault encrypted and ARE committed. +# Anything matching *_secrets.plain.yml is a working decryption — never commit those. +*_secrets.plain.yml + +# Vault password — never commit +ansible/.vault_pass \ No newline at end of file diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 56620fa..aa10181 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -7,7 +7,7 @@ stdout_callback = yaml retry_files_enabled = False host_key_checking = True forks = 10 -# vault_password_file = .vault_pass # uncomment in Stage 2 +vault_password_file = .vault_pass [ssh_connection] pipelining = True From 5e06021938f3be8e576d963c45db031a7af4caad Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 20:08:19 +0200 Subject: [PATCH 04/44] secrets note in readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f343cbc..80dbaf6 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ My repo documenting my personal infra, along with artifacts, scripts, etc. Go through the different numbered markdowns in the repo root to do the different parts. +## How to edit secrets + +`ansible-vault edit ansible/your_file_with_secrets.yml` + +Assumes that you've set `ansible/.vault_pass` with `chmod 600`. + ## Overview ### Services From 123107b7fbaf9798e39483404582ac027e1ef5d8 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:32:23 +0200 Subject: [PATCH 05/44] track inventory --- .gitignore | 2 -- ansible/inventory.ini | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 ansible/inventory.ini diff --git a/.gitignore b/.gitignore index 750a233..9ca7b8e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,6 @@ crash.log *.tfvars *.tfvars.json -test-inventory.ini -inventory.ini venv/* .env diff --git a/ansible/inventory.ini b/ansible/inventory.ini new file mode 100644 index 0000000..3d6c209 --- /dev/null +++ b/ansible/inventory.ini @@ -0,0 +1,25 @@ +[vps] +vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +[nodito_host] +nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + + +[nodito_vms] +knots_box_local ansible_host=192.168.1.135 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +fulcrum_box_local ansible_host=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +mempool_box_local ansible_host=192.168.1.140 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +memos_box_local ansible_host=192.168.1.130 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +forgejo_runner_local ansible_host=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +arbret_staging_local ansible_host=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +small_backups_local ansible_host=192.168.1.148 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +nonkeiwaisi_local ansible_host=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +# Local connection to laptop: this assumes you're running ansible commands from your personal laptop +[lapy] +localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C + +[arbret] +prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + From acb8f9d82ff2c87f1770d25800d2efe954fbbe1c Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:32:56 +0200 Subject: [PATCH 06/44] remove old example password --- ansible/inventory.ini.example | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 ansible/inventory.ini.example diff --git a/ansible/inventory.ini.example b/ansible/inventory.ini.example deleted file mode 100644 index bde96dd..0000000 --- a/ansible/inventory.ini.example +++ /dev/null @@ -1,16 +0,0 @@ -[vps] -vipy ansible_host=your.services.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key -watchtower ansible_host=your.monitoring.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key -spacey ansible_host=your.headscale.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key - -[nodito_host] -nodito ansible_host=your.proxmox.ip.here ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key ansible_ssh_pass=your_root_password - -[nodito_vms] -# Example node, replace with your VM names and addresses -# memos_box ansible_host=192.168.1.150 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key - -# Local connection to laptop: this assumes you're running ansible commands from your personal laptop -# Make sure to adjust the username -[lapy] -localhost ansible_connection=local ansible_user=your laptop user gpg_recipient=your_email@example.com gpg_key_id=your_gpg_key_id_here \ No newline at end of file From 28a0bb280670fed018c88ea3f598bee808544dac Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:51:50 +0200 Subject: [PATCH 07/44] new groups, stop using all --- .../01_user_and_access_setup_playbook.yml | 2 +- .../02_firewall_and_fail2ban_playbook.yml | 2 +- ansible/infra/410_disk_usage_alerts.yml | 2 +- ansible/infra/420_system_healthcheck.yml | 2 +- ansible/infra/900_install_rsync.yml | 2 +- ansible/infra/910_docker_playbook.yml | 2 +- ansible/infra/920_join_headscale_mesh.yml | 2 +- ansible/inventory.ini | 36 +++++++++++++++++++ 8 files changed, 43 insertions(+), 7 deletions(-) diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index 13e5149..c6eed18 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,5 +1,5 @@ - name: Secure Debian - hosts: all + hosts: managed vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index e83cbcb..9f37c70 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,5 +1,5 @@ - name: Secure Debian - hosts: all + hosts: managed vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml index de02f53..7905cf3 100644 --- a/ansible/infra/410_disk_usage_alerts.yml +++ b/ansible/infra/410_disk_usage_alerts.yml @@ -1,5 +1,5 @@ - name: Deploy Disk Usage Monitoring - hosts: all + hosts: managed become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml index 2580ff0..5a45ce1 100644 --- a/ansible/infra/420_system_healthcheck.yml +++ b/ansible/infra/420_system_healthcheck.yml @@ -1,5 +1,5 @@ - name: Deploy System Healthcheck Monitoring - hosts: all + hosts: managed become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/900_install_rsync.yml b/ansible/infra/900_install_rsync.yml index c0b7318..6c6b10e 100644 --- a/ansible/infra/900_install_rsync.yml +++ b/ansible/infra/900_install_rsync.yml @@ -1,5 +1,5 @@ - name: Install rsync - hosts: all + hosts: managed vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/910_docker_playbook.yml b/ansible/infra/910_docker_playbook.yml index f137b6a..62b7147 100644 --- a/ansible/infra/910_docker_playbook.yml +++ b/ansible/infra/910_docker_playbook.yml @@ -1,5 +1,5 @@ - name: Install Docker and Docker Compose on Debian 12 - hosts: all + hosts: managed become: yes tasks: diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 8d06d44..5b5e4ce 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -1,5 +1,5 @@ - name: Join machine to headscale mesh network - hosts: all + hosts: managed become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/inventory.ini b/ansible/inventory.ini index 3d6c209..a061637 100644 --- a/ansible/inventory.ini +++ b/ansible/inventory.ini @@ -23,3 +23,39 @@ localhost ansible_connection=local ansible_user=counterweight gpg_recipient=coun [arbret] prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +[edge] +vipy + +[monitoring] +watchtower + +[vpn_control] +spacey + +[hypervisor] +nodito + +[bitcoin] +knots_box_local + +[electrum] +fulcrum_box_local + +[mempool] +mempool_box_local + +[memos] +memos_box_local + +[ci_runner] +forgejo_runner_local + +[control] +localhost + +# Every machine Ansible may configure as a server. +# Deliberately EXCLUDES [control] (your laptop) and [arbret]. +[managed:children] +vps +nodito_host +nodito_vms \ No newline at end of file From e63fa1ff110396168324b3633dd0563626916ff1 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:12 +0200 Subject: [PATCH 08/44] personal-blog: target edge group instead of vipy Co-Authored-By: Claude Opus 5 (1M context) --- .../services/personal-blog/deploy_personal_blog_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index f4ee8ec..63bbae5 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy personal blog static site with Caddy file server - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From a26c0eca4635ff54af4ccb6065d67b950c4f0ac7 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:12 +0200 Subject: [PATCH 09/44] ntfy-emergency-app: target edge group instead of vipy Co-Authored-By: Claude Opus 5 (1M context) --- .../ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index b8c0064..a04d6af 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy ntfy-emergency-app with Docker Compose and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From 007fdb43af98f40ac90b91402e78f46f5d6f4c8b Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:12 +0200 Subject: [PATCH 10/44] memos: target memos and edge groups instead of hostnames Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/memos/deploy_memos_playbook.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index da56bd6..a823135 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Memos on memos-box - hosts: memos_box_local + hosts: memos become: yes vars_files: - ../../infra_vars.yml @@ -146,7 +146,7 @@ - name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From 31efa8365b1e57d12b89ac00dad240e68747c308 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:12 +0200 Subject: [PATCH 11/44] uptime_kuma: target monitoring group instead of watchtower Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml b/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml index 4af3858..8754dce 100644 --- a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml +++ b/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Uptime Kuma with Docker Compose and configure Caddy reverse proxy - hosts: watchtower + hosts: monitoring become: yes vars_files: - ../../infra_vars.yml From 791f6f3b69a36ad5fc6f8ed30ea43ebd50cd9cdb Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:13 +0200 Subject: [PATCH 12/44] ntfy: target monitoring group instead of watchtower Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/ntfy/deploy_ntfy_playbook.yml | 2 +- ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml index 0729baa..d030123 100644 --- a/ansible/services/ntfy/deploy_ntfy_playbook.yml +++ b/ansible/services/ntfy/deploy_ntfy_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy ntfy and configure Caddy reverse proxy - hosts: watchtower + hosts: monitoring become: yes vars_files: - ../../infra_vars.yml diff --git a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml index 5ba03f1..8041905 100644 --- a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml +++ b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml @@ -1,5 +1,5 @@ - name: Setup ntfy as Uptime Kuma Notification Channel - hosts: watchtower + hosts: monitoring become: no vars_files: - ../../infra_vars.yml From cc1aecd098dfd7e4c91ec110bdff925fb9ca0406 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:13 +0200 Subject: [PATCH 13/44] vaultwarden: target edge group instead of vipy Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml | 2 +- .../vaultwarden/disable_vaultwarden_sign_ups_playbook.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 0340538..65f1f64 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Vaultwarden with Docker Compose and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml diff --git a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml index b041e8e..eebb214 100644 --- a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml +++ b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml @@ -1,5 +1,5 @@ - name: Disable Vaultwarden Signups - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From 6680918fb7fc0c9bdc743605e4a5519cc90601af Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:13 +0200 Subject: [PATCH 14/44] forgejo: target edge group instead of vipy Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/forgejo/deploy_forgejo_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index e17d08f..ff865de 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -1,5 +1,5 @@ - name: Install Forgejo on Debian 12 with Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From f3ff1169db526a4e7cf6dd66d3ebe80e3846424f Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:13 +0200 Subject: [PATCH 15/44] forgejo-runner: target ci_runner group instead of forgejo_runner_local Co-Authored-By: Claude Opus 5 (1M context) --- .../services/forgejo-runner/deploy_forgejo_runner_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index a194178..d94f73c 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -1,5 +1,5 @@ - name: Install Forgejo Runner on Debian 13 - hosts: forgejo_runner_local + hosts: ci_runner become: yes vars_files: - ../../infra_vars.yml From 900a0b4826be730c673409d05b8d660f32880f71 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:14 +0200 Subject: [PATCH 16/44] headscale: target vpn_control group instead of spacey Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/headscale/deploy_headscale_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 1bcf5bf..7ed81ed 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy headscale and configure Caddy reverse proxy - hosts: spacey + hosts: vpn_control become: no vars_files: - ../../infra_vars.yml From 9ae0cc36d89de0ea83d09807f01fa3c712ff1c55 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:14 +0200 Subject: [PATCH 17/44] lnbits: target edge group instead of vipy Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/lnbits/deploy_lnbits_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index e5d546b..6df63f5 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy LNBits with Poetry and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From 8d402b64fcffb1255aedad62ba2b81c275a75aea Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:14 +0200 Subject: [PATCH 18/44] phoenixd: target edge group instead of vipy Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/phoenixd/deploy_phoenixd_playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index 65ad7c9..9e465eb 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -20,7 +20,7 @@ # means losing the funds. See setup_backup_phoenixd_to_lapy.yml. - name: Deploy phoenixd on vipy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From fd7803755bee1a52de3c8c1261fb99d4e209073e Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:14 +0200 Subject: [PATCH 19/44] mempool: target mempool and edge groups instead of hostnames Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/mempool/deploy_mempool_playbook.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 658180a..c0fe51d 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Mempool Block Explorer with Docker - hosts: mempool_box_local + hosts: mempool become: yes vars_files: - ../../infra_vars.yml @@ -584,7 +584,7 @@ - /tmp/mempool_push_urls.yml - name: Configure Caddy reverse proxy for Mempool on vipy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From a6c621e95c608286f86d3d0168c4d353bdd08fd2 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:15 +0200 Subject: [PATCH 20/44] fulcrum: target electrum and edge groups instead of hostnames Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/fulcrum/deploy_fulcrum_playbook.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 1255cd6..6b17280 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Fulcrum Electrum Server - hosts: fulcrum_box_local + hosts: electrum become: yes vars_files: - ../../infra_vars.yml @@ -524,7 +524,7 @@ - name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From 52d4a377f7062eb284ea7cccb64db4ce1f94f21e Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:15 +0200 Subject: [PATCH 21/44] bitcoin-knots: target bitcoin and edge groups instead of hostnames Co-Authored-By: Claude Opus 5 (1M context) --- .../services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index d818945..f84d627 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -1,5 +1,5 @@ - name: Build and Deploy Bitcoin Knots from Source - hosts: knots_box_local + hosts: bitcoin become: yes vars_files: - ../../infra_vars.yml @@ -724,7 +724,7 @@ - name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From 4349006d51ea2769bc21b0717a2b240e2b49f197 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:15 +0200 Subject: [PATCH 22/44] datum-gateway: target bitcoin and edge groups instead of hostnames Co-Authored-By: Claude Opus 5 (1M context) --- .../datum-gateway/deploy_datum_gateway_playbook.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index f33ded7..5fa360c 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -25,7 +25,7 @@ # bitcoin_rpc_password - Shared with the bitcoin-knots deployment - name: Deploy DATUM Gateway on knots_box_local - hosts: knots_box_local + hosts: bitcoin become: yes vars_files: - ../../infra_vars.yml @@ -470,7 +470,7 @@ # Caddy Reverse Proxy for DATUM Dashboard (on vipy) # =========================================== - name: Configure Caddy reverse proxy for DATUM Gateway dashboard on vipy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -663,7 +663,7 @@ # over the Tailscale network, matching the Bitcoin P2P proxy pattern. # =========================================== - name: Setup public Stratum port forwarding on vipy via systemd-socket-proxyd - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml From ceb5c4ad2743d3c45fe446916407cf450c219b99 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:24 +0200 Subject: [PATCH 23/44] infra/nodito: target hypervisor group instead of nodito_host/nodito Co-Authored-By: Claude Opus 5 (1M context) --- ansible/infra/430_cpu_temp_alerts.yml | 2 +- ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml | 2 +- ansible/infra/nodito/31_proxmox_community_repos_playbook.yml | 2 +- ansible/infra/nodito/32_zfs_pool_setup_playbook.yml | 4 ++-- ansible/infra/nodito/33_proxmox_debian_cloud_template.yml | 2 +- ansible/infra/nodito/34_nut_ups_setup_playbook.yml | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml index 3b87102..b06f0b6 100644 --- a/ansible/infra/430_cpu_temp_alerts.yml +++ b/ansible/infra/430_cpu_temp_alerts.yml @@ -1,5 +1,5 @@ - name: Deploy CPU Temperature Monitoring - hosts: nodito_host + hosts: hypervisor become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml index 02c6679..86f693f 100644 --- a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml +++ b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml @@ -1,5 +1,5 @@ - name: Bootstrap Nodito SSH Key Access - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../infra_vars.yml diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml index b0be2ef..378a674 100644 --- a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -1,5 +1,5 @@ - name: Switch Proxmox VE from Enterprise to Community Repositories - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../infra_vars.yml diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index cb72328..2939ce8 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -1,5 +1,5 @@ - name: Setup ZFS RAID 1 Pool for Proxmox Storage - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../infra_vars.yml @@ -172,7 +172,7 @@ when: "'ONLINE' not in final_zfs_status.stdout" - name: Setup ZFS Pool Health Monitoring and Monthly Scrubs - hosts: nodito + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index e8f8332..3b687ee 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -1,5 +1,5 @@ - name: Create Proxmox template from Debian cloud image (no VM clone) - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index 02468d5..6b366ba 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -1,5 +1,5 @@ - name: Setup NUT (Network UPS Tools) for CyberPower UPS - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml @@ -251,7 +251,7 @@ - name: Setup UPS Heartbeat Monitoring with Uptime Kuma - hosts: nodito + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml From ed99e17aaeddfd1821b879eba02f4e38ad27aa7e Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 21:56:24 +0200 Subject: [PATCH 24/44] backups: target control group instead of lapy Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml | 2 +- ansible/services/headscale/setup_backup_headscale_to_lapy.yml | 2 +- ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml | 2 +- ansible/services/memos/setup_backup_memos_to_lapy.yml | 2 +- ansible/services/personal-blog/setup_deploy_alias_lapy.yml | 2 +- ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml | 2 +- .../services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml | 2 +- .../services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml index b90f0fb..c3ff8d6 100644 --- a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml +++ b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml @@ -1,6 +1,6 @@ --- - name: Configure local backup for Forgejo from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml index 5f9a764..acc82ce 100644 --- a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml +++ b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Headscale from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml index 5d10dec..48296cb 100644 --- a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml +++ b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for LNBits from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml index 6d9c161..c32c332 100644 --- a/ansible/services/memos/setup_backup_memos_to_lapy.yml +++ b/ansible/services/memos/setup_backup_memos_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Memos from memos-box - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml index 2b90d68..99f9b34 100644 --- a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml +++ b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml @@ -1,5 +1,5 @@ - name: Configure deployment alias for personal blog in lapy .bashrc - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml index cc008e6..7474af1 100644 --- a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml +++ b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml @@ -13,7 +13,7 @@ # Because both files are static, phoenixd does not need to be stopped. - name: Configure local backup for phoenixd from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml b/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml index 9ae9713..54c0b79 100644 --- a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml +++ b/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Uptime Kuma from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml index 064d633..096f9d3 100644 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Vaultwarden from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml From 3c2aacad46c18bfafae6ac32bb2ac563de4a50af Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:00:43 +0200 Subject: [PATCH 25/44] vars: derive remote_host_name from role groups instead of hostnames Resolved values verified unchanged: same host, IP, user, key and port for all 8. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/services/forgejo/forgejo_vars.yml | 2 +- ansible/services/headscale/headscale_vars.yml | 2 +- ansible/services/lnbits/lnbits_vars.yml | 2 +- ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml | 2 +- ansible/services/personal-blog/personal_blog_vars.yml | 2 +- ansible/services/phoenixd/phoenixd_vars.yml | 2 +- ansible/services/uptime_kuma/uptime_kuma_vars.yml | 2 +- ansible/services/vaultwarden/vaultwarden_vars.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/services/forgejo/forgejo_vars.yml b/ansible/services/forgejo/forgejo_vars.yml index 0bbb5a5..2a24133 100644 --- a/ansible/services/forgejo/forgejo_vars.yml +++ b/ansible/services/forgejo/forgejo_vars.yml @@ -12,7 +12,7 @@ forgejo_user: "git" # (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index 653c175..0b0ee50 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -13,7 +13,7 @@ headscale_data_dir: /var/lib/headscale # Namespace now configured in services_config.yml under service_settings.headscale.namespace # Remote access -remote_host_name: "spacey" +remote_host_name: "{{ groups['vpn_control'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/lnbits/lnbits_vars.yml b/ansible/services/lnbits/lnbits_vars.yml index bdb97df..eabd6fc 100644 --- a/ansible/services/lnbits/lnbits_vars.yml +++ b/ansible/services/lnbits/lnbits_vars.yml @@ -6,7 +6,7 @@ lnbits_port: 8765 # (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml index 415bc4d..a3bb480 100644 --- a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml +++ b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml @@ -9,7 +9,7 @@ ntfy_emergency_app_topic: "emergencia" ntfy_emergency_app_ui_message: "Leave Pablo a message, he will respond as soon as possible" # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/personal-blog/personal_blog_vars.yml b/ansible/services/personal-blog/personal_blog_vars.yml index 59e0921..a1f34e3 100644 --- a/ansible/services/personal-blog/personal_blog_vars.yml +++ b/ansible/services/personal-blog/personal_blog_vars.yml @@ -4,7 +4,7 @@ personal_blog_web_root: "/var/www/pablohere.contrapeso.xyz" # Remote access for deployment -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/phoenixd/phoenixd_vars.yml b/ansible/services/phoenixd/phoenixd_vars.yml index 57218dc..f67c302 100644 --- a/ansible/services/phoenixd/phoenixd_vars.yml +++ b/ansible/services/phoenixd/phoenixd_vars.yml @@ -41,7 +41,7 @@ phoenixd_healthcheck_service_name: phoenixd-healthcheck phoenixd_monitor_name: "Phoenixd" # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/uptime_kuma/uptime_kuma_vars.yml b/ansible/services/uptime_kuma/uptime_kuma_vars.yml index 3263f49..33ffb6d 100644 --- a/ansible/services/uptime_kuma/uptime_kuma_vars.yml +++ b/ansible/services/uptime_kuma/uptime_kuma_vars.yml @@ -4,7 +4,7 @@ uptime_kuma_data_dir: "{{ uptime_kuma_dir }}/data" uptime_kuma_port: 3001 # Remote access -remote_host_name: "watchtower" +remote_host_name: "{{ groups['monitoring'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/vaultwarden/vaultwarden_vars.yml b/ansible/services/vaultwarden/vaultwarden_vars.yml index 75e527d..e60f1d4 100644 --- a/ansible/services/vaultwarden/vaultwarden_vars.yml +++ b/ansible/services/vaultwarden/vaultwarden_vars.yml @@ -6,7 +6,7 @@ vaultwarden_port: 8222 # (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" From b0f14ea3655f96cb817f9435731b14a26db5ea1a Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:17:13 +0200 Subject: [PATCH 26/44] vars --- ansible/group_vars/all/main.yml | 4 +++ ansible/group_vars/all/vault.yml | 50 +++++++++++++++++++++++++++++++ ansible/host_vars/nodito/main.yml | 28 +++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 ansible/group_vars/all/main.yml create mode 100644 ansible/group_vars/all/vault.yml create mode 100644 ansible/host_vars/nodito/main.yml diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml new file mode 100644 index 0000000..952df93 --- /dev/null +++ b/ansible/group_vars/all/main.yml @@ -0,0 +1,4 @@ +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: contrapeso.xyz diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml new file mode 100644 index 0000000..c48d456 --- /dev/null +++ b/ansible/group_vars/all/vault.yml @@ -0,0 +1,50 @@ +$ANSIBLE_VAULT;1.1;AES256 +32643038333165373363353439313639326538343435393633343765386662643231333936386630 +6564313061366166353662373564393261396666663432610a633161643066663232336163306663 +61623535643231366638616630666562333936383232373130623631306334653634386132396431 +3036383735316339650a313565313062636461396563336461633838343037666237623038633133 +35333531313036666531633230646464636631666133313636353362646336623135363736316562 +36656564396230333231643266303833633134303862323633636136363136373836346363663364 +32306234393863613534663031646632643734366531343339386133363837396138616435313365 +65353037326266386538633665386638656539376133636435346136643163643238656434346132 +34363036623935343162313031343036393536613136306435323730346561393261333966663465 +35333864323638356565316632353662366262363962376134366432333361343639633133626533 +65303964643330393264643136346432313164396664333535363135646464326539626433623732 +34666530666234376637623930633433656163353136646134326365353563343833666161666435 +30313866396136623036613933376164346533383964303664633830376636363565323832343136 +62653137366632313133626234386235356630383433623934303737653438623136366638396237 +66303435356539343061313337646261343738373866646465373339616161323531346361383363 +64643363616130323935656264326631336231323236616438366632356333333534303861663238 +61376464623030363938653230316265366533623364653930343537326532393433363230356136 +62643135636130626436386239366561393537333736383361336162636434656339653630353732 +37343265366234656637336135633461333735363932636130323564623661303730303363383833 +34373963373765643437303534366233626662656531373261633565353137643761356135376430 +30373639336565356364313333333237323338346631666235613961656336643438653834313137 +35303666366433646334643264633837343134373637643535633864313062613336333538303931 +34396266326166666165333639363961346639343130656235323332346631643539363861666436 +63343733656362313165623130383838376138613534363535336662366564306132353362666463 +61613534613431623263323831646635323362343634616566393834353964643861633639346132 +66363233373964323139313966616365343535306365373532313063383236353964343265383761 +64663833363035363736643236373634663466666138346266643930643033343839313534376666 +37363461623965636164663339386237643635353237353263633131363434643066393538363639 +39346530366361656536383233663766616234383535643234616233666262353364633036663062 +64386663383363633365656631326332353261303562346666636565313564306635333562363930 +33306237333638323131613161336165636237623332643137363038303330313663346639326361 +39383565366431363361313361303436343761626462306537633337393636333135343833623464 +32323663616166653063303861396664613434643536633066626466303634663031623938313436 +32303763616562376137303061633864393863373334373338626566653633383132306635396566 +64616463663265643732366465666662663334666362383036326330356535333465326534363266 +33326265326365353231613839383336656562333837376565306230646466333238323936316463 +64396266306562303438363635343865376364333239376664346534636664326532353265313563 +66636331333430616462326334623637646539643638323036303335656632363565373739373234 +35666532346364376335346238376562336164386363623862356539316466336531643936336131 +31356332306639626532646338393638373764653961616638643066313832626561336435383839 +64363232313633643133383066636539663933323064366536666462646466323166306562623538 +64393835383431663561343231303961383337656331663261393439336136623066666434336362 +36313239336537376134396361643532633437633163373931353139303534356665383538393838 +31306336326538353232336533643961383337313161663266353730643931656165353264303938 +30636363666136653963306130666433633562313464653862656262313035306635306230653666 +34336631623235323734616163623039613636393863383966336435313230313231356563636337 +61373336336563653030366130356331616438336638323263663164366435653266643161653331 +31323733313936313833386665633235633339376330373161326361336266633838326336356363 +37383165393866656565316337386639346531323439626465613834303636663933 diff --git a/ansible/host_vars/nodito/main.yml b/ansible/host_vars/nodito/main.yml new file mode 100644 index 0000000..c0002f3 --- /dev/null +++ b/ansible/host_vars/nodito/main.yml @@ -0,0 +1,28 @@ +# Nodito CPU Temperature Monitoring Configuration + +# Temperature Monitoring Configuration +temp_threshold_celsius: 80 +temp_check_interval_minutes: 1 + +# Script Configuration +monitoring_script_dir: /opt/nodito-monitoring +monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" +log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" + +# System Configuration +systemd_service_name: nodito-cpu-temp-monitor + +# ZFS Pool Configuration +zfs_pool_name: "proxmox-tank-1" +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror +zfs_pool_mountpoint: "/var/lib/vz" + +# UPS Configuration (CyberPower CP900EPFCLCD via USB) +ups_name: cyberpower +ups_desc: "CyberPower CP900EPFCLCD" +ups_driver: usbhid-ups +ups_port: auto +ups_user: counterweight +ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power +ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power From 79942525b12808673754792d4fcf6e1cde08b3ca Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:43:15 +0200 Subject: [PATCH 27/44] archive: record Uptime Kuma monitors and setup before decommissioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captured from the live instance rather than the repo: the playbooks created 17 monitors, the server had 75. The rest existed only in the UI. Push tokens are excluded deliberately — they are live credentials. Co-Authored-By: Claude Opus 5 (1M context) --- archive/uptime_kuma/MONITORS.md | 152 +++ archive/uptime_kuma/README.md | 51 + .../deploy_uptime_kuma_playbook.yml | 0 archive/uptime_kuma/monitors.json | 1202 +++++++++++++++++ .../setup_backup_uptime_kuma_to_lapy.yml | 0 .../uptime_kuma/uptime_kuma_vars.yml | 0 6 files changed, 1405 insertions(+) create mode 100644 archive/uptime_kuma/MONITORS.md create mode 100644 archive/uptime_kuma/README.md rename {ansible/services => archive}/uptime_kuma/deploy_uptime_kuma_playbook.yml (100%) create mode 100644 archive/uptime_kuma/monitors.json rename {ansible/services => archive}/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml (100%) rename {ansible/services => archive}/uptime_kuma/uptime_kuma_vars.yml (100%) diff --git a/archive/uptime_kuma/MONITORS.md b/archive/uptime_kuma/MONITORS.md new file mode 100644 index 0000000..53b1d89 --- /dev/null +++ b/archive/uptime_kuma/MONITORS.md @@ -0,0 +1,152 @@ +# Uptime Kuma — monitor inventory (archived) + +Captured from the live instance at `https://uptime.contrapeso.xyz` on 2026-09-11, +immediately before decommissioning. This is the **authoritative** record: most of +these monitors existed only in the Uptime Kuma UI and were never described by any +playbook in this repo. + +**75 monitors total** — 16 group, 8 http, 3 port, 48 push. All were active. + +Push tokens are deliberately **not** recorded here: they are live credentials, and +anything holding one could report a false 'up'. They die with the server. + +--- + +## arbret - production *(7 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| arbret.com - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbret.com - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbret.com - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | +| arbret.com - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | +| arbret.com - health | push | — | 120s | Healthy when GET /api/health returns status ok | +| arbret.com - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | +| arbret.com - postgresql | push | — | 120s | Healthy when postgresql.service is active | + +## arbret - staging *(7 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| arbretstaging.contrapeso.xyz - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbretstaging.contrapeso.xyz - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbretstaging.contrapeso.xyz - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | +| arbretstaging.contrapeso.xyz - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | +| arbretstaging.contrapeso.xyz - health | push | — | 120s | Healthy when GET /api/health returns status ok | +| arbretstaging.contrapeso.xyz - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | +| arbretstaging.contrapeso.xyz - postgresql | push | — | 120s | Healthy when postgresql.service is active | + +## arbret-staging-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-arbret-staging-box-root | push | — | 960s | upside-down, Disk Usage: arbret-staging-box (/) - Alerts when usage excee | +| system-healthcheck-arbret-staging-box | push | — | 90s | System Healthcheck: arbret-staging-box - Regular healthcheck | + +## forgejo-runner-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-forgejo-runner-box-root | push | — | 960s | upside-down, Disk Usage: forgejo-runner-box (/) - Alerts when usage excee | +| system-healthcheck-forgejo-runner-box | push | — | 90s | System Healthcheck: forgejo-runner-box - Regular healthcheck | + +## fulcrum-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-fulcrum-box-root | push | — | 960s | upside-down, Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-fulcrum-box | push | — | 90s | System Healthcheck: fulcrum-box - Regular healthcheck ping e | + +## knots-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-knots-box-root | push | — | 960s | upside-down, Disk Usage: knots-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-knots-box | push | — | 90s | System Healthcheck: knots-box - Regular healthcheck ping eve | + +## memos-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-memos-box-root | push | — | 960s | upside-down, Disk Usage: memos-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-memos-box | push | — | 90s | System Healthcheck: memos-box - Regular healthcheck ping eve | + +## mempool-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-mempool-box-root | push | — | 960s | upside-down, Disk Usage: mempool-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-mempool-box | push | — | 90s | System Healthcheck: mempool-box - Regular healthcheck ping e | + +## nodito - infra *(4 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| UPS ONLINE | push | https:// | 90s | — | +| cpu-temp-nodito | push | — | 120s | upside-down, CPU Temperature: nodito - Alerts when temperature exceeds 80 | +| system-healthcheck-nodito | push | — | 300s | System Healthcheck: nodito - Regular healthcheck ping every | +| zfs-health-nodito | push | — | 90000s | ZFS Pool Health: nodito - Daily health check for pool proxmo | + +## nonkeiwaisi-box - infra *(0 monitors)* + +_(empty)_ + +## prd-arbret - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-arbret-root | push | — | 960s | upside-down, Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-arbret | push | — | 90s | System Healthcheck: prd-arbret - Regular healthcheck ping ev | + +## prd-spacey - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-spacey-root | push | — | 960s | upside-down, Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-spacey | push | — | 90s | System Healthcheck: prd-spacey - Regular healthcheck ping ev | + +## prd-vipy - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-vipy-root | push | — | 960s | upside-down, Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-vipy | push | — | 90s | System Healthcheck: prd-vipy - Regular healthcheck ping ever | + +## prd-watchtower - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-watchtower-root | push | — | 960s | upside-down, Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 8 | +| system-healthcheck-prd-watchtower | push | — | 90s | System Healthcheck: prd-watchtower - Regular healthcheck pin | + +## services *(19 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| Forgejo | http | https://forgejo.contrapeso.xyz/api/healthz | 90s | — | +| Headscale | http | https://headscale.contrapeso.xyz/health | 60s | — | +| LNBits | http | https://wallet.contrapeso.xyz/api/v1/health | 60s | — | +| Memos | http | https://memos.contrapeso.xyz/healthz | 60s | — | +| Mempool Public Access | http | https://mempool.contrapeso.xyz | 60s | — | +| Personal Blog | http | https://pablohere.contrapeso.xyz | 60s | — | +| Vaultwarden | http | https://vault.contrapeso.xyz/alive | 60s | — | +| ntfy-emergency-app | http | https://avisame.contrapeso.xyz | 60s | — | +| Bitcoin Knots P2P Public | port | 167.172.107.33:8333 | 60s | — | +| DATUM Stratum (public) | port | 167.172.107.33:23334 | 60s | — | +| Fulcrum SSL Public | port | 167.172.107.33:50002 | 60s | — | +| Bitcoin Knots | push | — | 90s | — | +| DATUM Gateway | push | — | 90s | — | +| Fulcrum | push | — | 90s | — | +| Mempool Backend | push | — | 180s | — | +| Mempool Frontend | push | — | 90s | — | +| Mempool MariaDB | push | — | 90s | — | +| Phoenixd | push | — | 90s | — | +| forgejo-runner-healthcheck | push | — | 90s | Forgejo Runner healthcheck - ping every 60s | + +## small-backups-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-small-backups-box-root | push | — | 960s | upside-down, Disk Usage: small-backups-box (/) - Alerts when usage exceed | +| system-healthcheck-small-backups-box | push | — | 90s | System Healthcheck: small-backups-box - Regular healthcheck | + diff --git a/archive/uptime_kuma/README.md b/archive/uptime_kuma/README.md new file mode 100644 index 0000000..5405ae0 --- /dev/null +++ b/archive/uptime_kuma/README.md @@ -0,0 +1,51 @@ +# Uptime Kuma — archived + +Uptime Kuma was the monitoring stack for this infrastructure until **2026-09-11**, when +it was decommissioned. Everything that referenced it has been removed from the live +playbooks; this folder is the record of what it was, kept so the setup can be understood +later without digging through git history. + +## Contents + +| File | What it is | +|---|---| +| `MONITORS.md` | Every monitor that existed, grouped as it was in the UI. The authoritative record. | +| `monitors.json` | The same data, machine-readable, as returned by the API. | +| `deploy_uptime_kuma_playbook.yml` | How the server itself was deployed (Docker Compose on `monitoring`, behind Caddy). | +| `uptime_kuma_vars.yml` | Variables the deploy playbook needed — it is unreadable without these. | +| `setup_backup_uptime_kuma_to_lapy.yml` | How its data was backed up, and to where. Useful when disposing of the old volumes. | + +## Why the inventory was captured from the live server, not the repo + +The playbooks only ever created **17** monitors. The live instance had **75**. The +difference was created by hand in the UI and existed nowhere else — so a repo-derived +list would have silently lost two thirds of the picture. `MONITORS.md` is a snapshot of +the real thing, taken immediately before removal. + +Push tokens are deliberately excluded. They are live credentials — anything holding one +can report a false "up" — and they become meaningless once the server is gone. + +## What removal did NOT do + +Removing the playbook code does not touch the machines. **48 push monitors** were driven +by scripts and systemd timers installed *on the hosts*, which keep firing on their +schedule and curling an endpoint that no longer answers. They are harmless but they are +still there, writing logs and failing quietly. + +Left behind, per host: + +- `/opt/disk-monitoring` + `disk-usage-monitor.{service,timer}` — on all 12 `managed` hosts +- `/opt/system-healthcheck` + `system-healthcheck.{service,timer}` — on all 12 `managed` hosts +- `/opt/nodito-monitoring` + `nodito-cpu-temp-monitor.{service,timer}` — `nodito` +- `/opt/zfs-monitoring` — `nodito` +- `/opt/ups-monitoring` — `nodito` +- `bitcoin-knots-healthcheck.{service,timer}` — `bitcoin` +- `datum-gateway-healthcheck.{service,timer}` — `bitcoin` +- `fulcrum-healthcheck.{service,timer}` — `electrum` +- `mempool-{backend,frontend,mariadb}-healthcheck.service` — `mempool` +- phoenixd and forgejo-runner healthcheck units — `edge`, `ci_runner` + +Note `nut-monitor.service` on `nodito` is **NUT's own daemon**, not a monitoring +leftover — do not remove it with the rest. + +Cleaning these up is a separate decommissioning pass and was not part of the removal. diff --git a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml b/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml similarity index 100% rename from ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml rename to archive/uptime_kuma/deploy_uptime_kuma_playbook.yml diff --git a/archive/uptime_kuma/monitors.json b/archive/uptime_kuma/monitors.json new file mode 100644 index 0000000..d1b0b0a --- /dev/null +++ b/archive/uptime_kuma/monitors.json @@ -0,0 +1,1202 @@ +[ + { + "id": 1, + "name": "services", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 3, + "name": "Vaultwarden", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://vault.contrapeso.xyz/alive", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 4, + "name": "prd-spacey - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 5, + "name": "prd-watchtower - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 6, + "name": "prd-vipy - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 7, + "name": "disk-usage-prd-vipy-root", + "type": "push", + "active": true, + "parent": 6, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 8, + "name": "disk-usage-prd-watchtower-root", + "type": "push", + "active": true, + "parent": 5, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 9, + "name": "disk-usage-prd-spacey-root", + "type": "push", + "active": true, + "parent": 4, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 10, + "name": "system-healthcheck-prd-vipy", + "type": "push", + "active": true, + "parent": 6, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-vipy - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 11, + "name": "system-healthcheck-prd-spacey", + "type": "push", + "active": true, + "parent": 4, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-spacey - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 12, + "name": "system-healthcheck-prd-watchtower", + "type": "push", + "active": true, + "parent": 5, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-watchtower - Regular healthcheck ping every 60s", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 13, + "name": "Forgejo", + "type": "http", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": "https://forgejo.contrapeso.xyz/api/healthz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 10, + "resendInterval": 0 + }, + { + "id": 14, + "name": "Personal Blog", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://pablohere.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 15, + "name": "LNBits", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://wallet.contrapeso.xyz/api/v1/health", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 16, + "name": "ntfy-emergency-app", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://avisame.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 17, + "name": "Headscale", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://headscale.contrapeso.xyz/health", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 18, + "name": "knots-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 19, + "name": "disk-usage-knots-box-root", + "type": "push", + "active": true, + "parent": 18, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: knots-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 20, + "name": "system-healthcheck-knots-box", + "type": "push", + "active": true, + "parent": 18, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: knots-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 21, + "name": "nodito - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 22, + "name": "system-healthcheck-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 300, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: nodito - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 23, + "name": "cpu-temp-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "CPU Temperature: nodito - Alerts when temperature exceeds 80\u00b0C", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 24, + "name": "Bitcoin Knots", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 25, + "name": "fulcrum-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 26, + "name": "disk-usage-fulcrum-box-root", + "type": "push", + "active": true, + "parent": 25, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 27, + "name": "system-healthcheck-fulcrum-box", + "type": "push", + "active": true, + "parent": 25, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: fulcrum-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 28, + "name": "Fulcrum", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 35, + "name": "mempool-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 36, + "name": "disk-usage-mempool-box-root", + "type": "push", + "active": true, + "parent": 35, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: mempool-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 37, + "name": "system-healthcheck-mempool-box", + "type": "push", + "active": true, + "parent": 35, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: mempool-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 38, + "name": "Mempool MariaDB", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 39, + "name": "Mempool Backend", + "type": "push", + "active": true, + "parent": 1, + "interval": 180, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 40, + "name": "Mempool Frontend", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 41, + "name": "Mempool Public Access", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://mempool.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 42, + "name": "memos-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 43, + "name": "disk-usage-memos-box-root", + "type": "push", + "active": true, + "parent": 42, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: memos-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 44, + "name": "system-healthcheck-memos-box", + "type": "push", + "active": true, + "parent": 42, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: memos-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 45, + "name": "Memos", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://memos.contrapeso.xyz/healthz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 46, + "name": "Bitcoin Knots P2P Public", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 8333, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 47, + "name": "Fulcrum SSL Public", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 50002, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 49, + "name": "zfs-health-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 90000, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "ZFS Pool Health: nodito - Daily health check for pool proxmox-tank-1", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 50, + "name": "UPS ONLINE", + "type": "push", + "active": true, + "parent": 21, + "interval": 90, + "retries": null, + "url": "https://", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 51, + "name": "forgejo-runner-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 52, + "name": "disk-usage-forgejo-runner-box-root", + "type": "push", + "active": true, + "parent": 51, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: forgejo-runner-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 53, + "name": "system-healthcheck-forgejo-runner-box", + "type": "push", + "active": true, + "parent": 51, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: forgejo-runner-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 55, + "name": "forgejo-runner-healthcheck", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Forgejo Runner healthcheck - ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 59, + "name": "arbret-staging-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 60, + "name": "disk-usage-arbret-staging-box-root", + "type": "push", + "active": true, + "parent": 59, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: arbret-staging-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 61, + "name": "system-healthcheck-arbret-staging-box", + "type": "push", + "active": true, + "parent": 59, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: arbret-staging-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 66, + "name": "arbret - staging", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 67, + "name": "arbretstaging.contrapeso.xyz - health", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when GET /api/health returns status ok", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 68, + "name": "arbretstaging.contrapeso.xyz - https", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when HTTPS front-door returns 200", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 69, + "name": "arbretstaging.contrapeso.xyz - postgresql", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when postgresql.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 70, + "name": "arbretstaging.contrapeso.xyz - arbret-server", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-server.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 71, + "name": "arbretstaging.contrapeso.xyz - arbret-worker", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-worker.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 72, + "name": "arbretstaging.contrapeso.xyz - arbret-analytics", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 73, + "name": "arbretstaging.contrapeso.xyz - arbret-backup", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 74, + "name": "prd-arbret - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 85, + "name": "disk-usage-prd-arbret-root", + "type": "push", + "active": true, + "parent": 74, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 86, + "name": "system-healthcheck-prd-arbret", + "type": "push", + "active": true, + "parent": 74, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-arbret - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 87, + "name": "arbret - production", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 88, + "name": "arbret.com - health", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when GET /api/health returns status ok", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 89, + "name": "arbret.com - https", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when HTTPS front-door returns 200", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 90, + "name": "arbret.com - postgresql", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when postgresql.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 91, + "name": "arbret.com - arbret-server", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-server.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 92, + "name": "arbret.com - arbret-worker", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-worker.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 93, + "name": "arbret.com - arbret-analytics", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 94, + "name": "arbret.com - arbret-backup", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 95, + "name": "small-backups-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 96, + "name": "disk-usage-small-backups-box-root", + "type": "push", + "active": true, + "parent": 95, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: small-backups-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 97, + "name": "system-healthcheck-small-backups-box", + "type": "push", + "active": true, + "parent": 95, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: small-backups-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 109, + "name": "DATUM Gateway", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 111, + "name": "DATUM Stratum (public)", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 23334, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 112, + "name": "Phoenixd", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 113, + "name": "nonkeiwaisi-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + } +] diff --git a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml b/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml similarity index 100% rename from ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml rename to archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml diff --git a/ansible/services/uptime_kuma/uptime_kuma_vars.yml b/archive/uptime_kuma/uptime_kuma_vars.yml similarity index 100% rename from ansible/services/uptime_kuma/uptime_kuma_vars.yml rename to archive/uptime_kuma/uptime_kuma_vars.yml From 80c9e6f3e36dd6f10fef1055062e936d704d2ba8 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:43:15 +0200 Subject: [PATCH 28/44] uptime-kuma: remove credentials from the vaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops uptime_kuma_username/password from infra_secrets.yml and its group_vars copy and example, plus a dead push token in nodito_secrets.yml that nothing referenced. They remain in git history — rotation is what actually retires them. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/group_vars/all/vault.yml | 93 ++++++++++++------------- ansible/infra/nodito/nodito_secrets.yml | 27 +++---- ansible/infra_secrets.yml | 93 ++++++++++++------------- ansible/infra_secrets.yml.example | 2 - 4 files changed, 99 insertions(+), 116 deletions(-) diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index c48d456..798e2d8 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,50 +1,45 @@ $ANSIBLE_VAULT;1.1;AES256 -32643038333165373363353439313639326538343435393633343765386662643231333936386630 -6564313061366166353662373564393261396666663432610a633161643066663232336163306663 -61623535643231366638616630666562333936383232373130623631306334653634386132396431 -3036383735316339650a313565313062636461396563336461633838343037666237623038633133 -35333531313036666531633230646464636631666133313636353362646336623135363736316562 -36656564396230333231643266303833633134303862323633636136363136373836346363663364 -32306234393863613534663031646632643734366531343339386133363837396138616435313365 -65353037326266386538633665386638656539376133636435346136643163643238656434346132 -34363036623935343162313031343036393536613136306435323730346561393261333966663465 -35333864323638356565316632353662366262363962376134366432333361343639633133626533 -65303964643330393264643136346432313164396664333535363135646464326539626433623732 -34666530666234376637623930633433656163353136646134326365353563343833666161666435 -30313866396136623036613933376164346533383964303664633830376636363565323832343136 -62653137366632313133626234386235356630383433623934303737653438623136366638396237 -66303435356539343061313337646261343738373866646465373339616161323531346361383363 -64643363616130323935656264326631336231323236616438366632356333333534303861663238 -61376464623030363938653230316265366533623364653930343537326532393433363230356136 -62643135636130626436386239366561393537333736383361336162636434656339653630353732 -37343265366234656637336135633461333735363932636130323564623661303730303363383833 -34373963373765643437303534366233626662656531373261633565353137643761356135376430 -30373639336565356364313333333237323338346631666235613961656336643438653834313137 -35303666366433646334643264633837343134373637643535633864313062613336333538303931 -34396266326166666165333639363961346639343130656235323332346631643539363861666436 -63343733656362313165623130383838376138613534363535336662366564306132353362666463 -61613534613431623263323831646635323362343634616566393834353964643861633639346132 -66363233373964323139313966616365343535306365373532313063383236353964343265383761 -64663833363035363736643236373634663466666138346266643930643033343839313534376666 -37363461623965636164663339386237643635353237353263633131363434643066393538363639 -39346530366361656536383233663766616234383535643234616233666262353364633036663062 -64386663383363633365656631326332353261303562346666636565313564306635333562363930 -33306237333638323131613161336165636237623332643137363038303330313663346639326361 -39383565366431363361313361303436343761626462306537633337393636333135343833623464 -32323663616166653063303861396664613434643536633066626466303634663031623938313436 -32303763616562376137303061633864393863373334373338626566653633383132306635396566 -64616463663265643732366465666662663334666362383036326330356535333465326534363266 -33326265326365353231613839383336656562333837376565306230646466333238323936316463 -64396266306562303438363635343865376364333239376664346534636664326532353265313563 -66636331333430616462326334623637646539643638323036303335656632363565373739373234 -35666532346364376335346238376562336164386363623862356539316466336531643936336131 -31356332306639626532646338393638373764653961616638643066313832626561336435383839 -64363232313633643133383066636539663933323064366536666462646466323166306562623538 -64393835383431663561343231303961383337656331663261393439336136623066666434336362 -36313239336537376134396361643532633437633163373931353139303534356665383538393838 -31306336326538353232336533643961383337313161663266353730643931656165353264303938 -30636363666136653963306130666433633562313464653862656262313035306635306230653666 -34336631623235323734616163623039613636393863383966336435313230313231356563636337 -61373336336563653030366130356331616438336638323263663164366435653266643161653331 -31323733313936313833386665633235633339376330373161326361336266633838326336356363 -37383165393866656565316337386639346531323439626465613834303636663933 +65343264303166346334396163363362326334626531336363363766326135393462373564313539 +6633346163343664393232363965383334303563396236360a316265623239333431636133306663 +65623237393564323936303936633036343137646637633963396166313462626165616463616665 +3235373562626238640a636163623535376565373835653831666361636563616230353639626131 +38393530363461346334356437613934306330666563326661343465653362313038346535386337 +62623964633764343964653061373137366539326433396231396339313231633632353465323139 +33333866383834376165303964353639386234646535626333363932333764646137663036303530 +32326635343631623535323830333934376466323338353234323464666435383436633139343037 +62666333383231356163383530353339336230373431616665333638633234656231393166303639 +32386134386264613839313564316164336461373738646630636630363731643066633132336636 +64663532663661653539653431353763303436323234383864353064333631643435633733366435 +32656332626633313134623361343965643962663134613363626265353865643738363738316232 +62356136623166303038653830393138646637636335366362323239316333616331383765663861 +33643331323766646463663862396232306233343134326436326235336165616662383636616539 +39373962336264636363303734343666646534343733666632626338393936393032626130643233 +30633963656166343834396431323233363036353165623934336532333662323932336630636235 +63326631366234383732316464383364316337393034303433646365343763633135336336376161 +33326436303863326666616234396438623864333538313363623261353962336430316132343830 +36353761326133666166393930343233643861366138363930623362336566386230316236353365 +65663632383436333536636161393133316334376262653330303238636431353966646666663665 +30386137623066343135616534323336313162643066636330323934646162303362373838313532 +61386163356537613334616537656134373835326265393765303436396233383239393365336136 +64623130383538306331323939386634316663663263306362363830383930346232313965613035 +33353235663230663635316136666335343233646539613832643431393134303865616232636435 +66366437353863336235646438353431616465396365373562636461623938303663326461656134 +66353735343237363164326236663363376333316163353462626234393861663535323463623233 +66623934336662323830616362653035353563343564636431383837643961303637366131633132 +33376133376465623030643935643737636339333236316566636663323934626265633863343963 +38626563323166343264643465626264393838643330393739386330656461656561306539653232 +63393838626233386531353063366131366237633162393537363963633361386238643232336230 +36363062386262303364303736373061316265373730366437326166336163386335623665323234 +35383865646466363534383833653764303833323230613863313735393264336132303934633465 +65616138636134613361316539343739353735666538653438333264623034306633663634306438 +33643165353534623039626536636433373963383866303535333931633633393138663464336562 +36643432366233323235613139626236346135633237613265343933646434373939396265626366 +34313835346463303132326537323166363032306133623662343736393435383330356433306138 +65653931613832313832323936353638303937613863646430633266306330656135386533306462 +30663366396566393134653663356531316639303635343236666333363637636433336533356663 +35346261363134326165316463626439306130653165636134616434663736616666643963336464 +61643834613164336234386237393333636234306162306563363430646262363963366666653666 +64323036323731626236343361613566643565666138386631633462613836613766626562353934 +32363936626466346464356131653232326330386239666638626333346238623964383666393961 +38393264663636353161663636393733396332333838333962393439346533353332653362356230 +62613039376539636363 diff --git a/ansible/infra/nodito/nodito_secrets.yml b/ansible/infra/nodito/nodito_secrets.yml index 7983cdb..c34b1e8 100644 --- a/ansible/infra/nodito/nodito_secrets.yml +++ b/ansible/infra/nodito/nodito_secrets.yml @@ -1,17 +1,12 @@ $ANSIBLE_VAULT;1.1;AES256 -33303038356165616435663034356134363963323865356133353239373263623430373263613430 -6463323436386666633666333331636132653963663735330a333063303038313839653330303231 -37313332653033636265353730616431356337366566656533363838303939636266626565333365 -3764316366653361310a386330326335343936356430333864336431376230323734303030613032 -31343737323636363366623530353433313862663038376439343331633038363138316639613531 -63663866613037336630316464383464613165326632376337383338336432326564396261383561 -61313337336466653431613338656136653862646662323435396462623366366438626566373331 -34663433333965626161356636633930313961313531656233303638346265316232326530613062 -31343732636466393334623734346639636637363831393337363562343461616639316235336362 -62306464383263626235613533336130326232613563336630373661623637303961353863303633 -66306664653035363236323063616363313739633633373231333833393562633666356530383065 -36373830393965663861663231633838303863363763613663363734383961623466653735636138 -34376233363831363732643631646364363066626230353330316562376337633462323966376239 -30666263326263366563323739653538616462303566633037393861613664646331633264613338 -35636131383937393430666430646530663866626438643430316430376133653135333838636436 -37613465666265373336 +39386134336137333139343861353965353665626161303634323563386563356364373265616163 +3237623438636161323366313363356335613566303338630a663732316663333136653737663935 +34373466316538663962636134303530323837316563336136653636643430363566376363323666 +3531613636393331310a306633613030343138656237663061653438336562323135633732356563 +33396136656564353465636361613161326465303166373131343562363338303966666362643532 +33383561366539393930633766363533313363653733393263376634316362643863376635366638 +36396333633231616662323465323932656565396138313264613832346261616231333265636439 +61666334613662373839613833613663333436373365376534643662656335316536303739616437 +65333635613330353536353162646234323266316338643435653864666364313734386665303830 +34383961633734646134373866623330663038366130306265656466653562643764346162313333 +353664363335303433393330306332666336 diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml index c48d456..798e2d8 100644 --- a/ansible/infra_secrets.yml +++ b/ansible/infra_secrets.yml @@ -1,50 +1,45 @@ $ANSIBLE_VAULT;1.1;AES256 -32643038333165373363353439313639326538343435393633343765386662643231333936386630 -6564313061366166353662373564393261396666663432610a633161643066663232336163306663 -61623535643231366638616630666562333936383232373130623631306334653634386132396431 -3036383735316339650a313565313062636461396563336461633838343037666237623038633133 -35333531313036666531633230646464636631666133313636353362646336623135363736316562 -36656564396230333231643266303833633134303862323633636136363136373836346363663364 -32306234393863613534663031646632643734366531343339386133363837396138616435313365 -65353037326266386538633665386638656539376133636435346136643163643238656434346132 -34363036623935343162313031343036393536613136306435323730346561393261333966663465 -35333864323638356565316632353662366262363962376134366432333361343639633133626533 -65303964643330393264643136346432313164396664333535363135646464326539626433623732 -34666530666234376637623930633433656163353136646134326365353563343833666161666435 -30313866396136623036613933376164346533383964303664633830376636363565323832343136 -62653137366632313133626234386235356630383433623934303737653438623136366638396237 -66303435356539343061313337646261343738373866646465373339616161323531346361383363 -64643363616130323935656264326631336231323236616438366632356333333534303861663238 -61376464623030363938653230316265366533623364653930343537326532393433363230356136 -62643135636130626436386239366561393537333736383361336162636434656339653630353732 -37343265366234656637336135633461333735363932636130323564623661303730303363383833 -34373963373765643437303534366233626662656531373261633565353137643761356135376430 -30373639336565356364313333333237323338346631666235613961656336643438653834313137 -35303666366433646334643264633837343134373637643535633864313062613336333538303931 -34396266326166666165333639363961346639343130656235323332346631643539363861666436 -63343733656362313165623130383838376138613534363535336662366564306132353362666463 -61613534613431623263323831646635323362343634616566393834353964643861633639346132 -66363233373964323139313966616365343535306365373532313063383236353964343265383761 -64663833363035363736643236373634663466666138346266643930643033343839313534376666 -37363461623965636164663339386237643635353237353263633131363434643066393538363639 -39346530366361656536383233663766616234383535643234616233666262353364633036663062 -64386663383363633365656631326332353261303562346666636565313564306635333562363930 -33306237333638323131613161336165636237623332643137363038303330313663346639326361 -39383565366431363361313361303436343761626462306537633337393636333135343833623464 -32323663616166653063303861396664613434643536633066626466303634663031623938313436 -32303763616562376137303061633864393863373334373338626566653633383132306635396566 -64616463663265643732366465666662663334666362383036326330356535333465326534363266 -33326265326365353231613839383336656562333837376565306230646466333238323936316463 -64396266306562303438363635343865376364333239376664346534636664326532353265313563 -66636331333430616462326334623637646539643638323036303335656632363565373739373234 -35666532346364376335346238376562336164386363623862356539316466336531643936336131 -31356332306639626532646338393638373764653961616638643066313832626561336435383839 -64363232313633643133383066636539663933323064366536666462646466323166306562623538 -64393835383431663561343231303961383337656331663261393439336136623066666434336362 -36313239336537376134396361643532633437633163373931353139303534356665383538393838 -31306336326538353232336533643961383337313161663266353730643931656165353264303938 -30636363666136653963306130666433633562313464653862656262313035306635306230653666 -34336631623235323734616163623039613636393863383966336435313230313231356563636337 -61373336336563653030366130356331616438336638323263663164366435653266643161653331 -31323733313936313833386665633235633339376330373161326361336266633838326336356363 -37383165393866656565316337386639346531323439626465613834303636663933 +65343264303166346334396163363362326334626531336363363766326135393462373564313539 +6633346163343664393232363965383334303563396236360a316265623239333431636133306663 +65623237393564323936303936633036343137646637633963396166313462626165616463616665 +3235373562626238640a636163623535376565373835653831666361636563616230353639626131 +38393530363461346334356437613934306330666563326661343465653362313038346535386337 +62623964633764343964653061373137366539326433396231396339313231633632353465323139 +33333866383834376165303964353639386234646535626333363932333764646137663036303530 +32326635343631623535323830333934376466323338353234323464666435383436633139343037 +62666333383231356163383530353339336230373431616665333638633234656231393166303639 +32386134386264613839313564316164336461373738646630636630363731643066633132336636 +64663532663661653539653431353763303436323234383864353064333631643435633733366435 +32656332626633313134623361343965643962663134613363626265353865643738363738316232 +62356136623166303038653830393138646637636335366362323239316333616331383765663861 +33643331323766646463663862396232306233343134326436326235336165616662383636616539 +39373962336264636363303734343666646534343733666632626338393936393032626130643233 +30633963656166343834396431323233363036353165623934336532333662323932336630636235 +63326631366234383732316464383364316337393034303433646365343763633135336336376161 +33326436303863326666616234396438623864333538313363623261353962336430316132343830 +36353761326133666166393930343233643861366138363930623362336566386230316236353365 +65663632383436333536636161393133316334376262653330303238636431353966646666663665 +30386137623066343135616534323336313162643066636330323934646162303362373838313532 +61386163356537613334616537656134373835326265393765303436396233383239393365336136 +64623130383538306331323939386634316663663263306362363830383930346232313965613035 +33353235663230663635316136666335343233646539613832643431393134303865616232636435 +66366437353863336235646438353431616465396365373562636461623938303663326461656134 +66353735343237363164326236663363376333316163353462626234393861663535323463623233 +66623934336662323830616362653035353563343564636431383837643961303637366131633132 +33376133376465623030643935643737636339333236316566636663323934626265633863343963 +38626563323166343264643465626264393838643330393739386330656461656561306539653232 +63393838626233386531353063366131366237633162393537363963633361386238643232336230 +36363062386262303364303736373061316265373730366437326166336163386335623665323234 +35383865646466363534383833653764303833323230613863313735393264336132303934633465 +65616138636134613361316539343739353735666538653438333264623034306633663634306438 +33643165353534623039626536636433373963383866303535333931633633393138663464336562 +36643432366233323235613139626236346135633237613265343933646434373939396265626366 +34313835346463303132326537323166363032306133623662343736393435383330356433306138 +65653931613832313832323936353638303937613863646430633266306330656135386533306462 +30663366396566393134653663356531316639303635343236666333363637636433336533356663 +35346261363134326165316463626439306130653165636134616434663736616666643963336464 +61643834613164336234386237393333636234306162306563363430646262363963366666653666 +64323036323731626236343361613566643565666138386631633462613836613766626562353934 +32363936626466346464356131653232326330386239666638626333346238623964383666393961 +38393264663636353161663636393733396332333838333962393439346533353332653362356230 +62613039376539636363 diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index d539282..c95234d 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -1,8 +1,6 @@ # Uptime Kuma login credentials # Used by the disk monitoring playbook to create monitors automatically -uptime_kuma_username: "admin" -uptime_kuma_password: "your_password_here" # ntfy credentials # Used for notification channel setup in Uptime Kuma From 2dabc6ad62bff830005c83fe71da531b59c593d2 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:43:55 +0200 Subject: [PATCH 29/44] uptime-kuma: add uptime_kuma_enabled flag and make monitoring blocks inert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the credentials would otherwise break these playbooks mid-deploy: they template uptime_kuma_password with no assert to stop them first. 100 tasks are now guarded by uptime_kuma_enabled (false), so deployments run normally and the monitoring sections skip. A further 28 tasks were already self-guarding on monitor_setup/push_url being defined; verified that a skipped task's registered variable makes those skip cleanly rather than error. The blocks are kept on purpose — the health-check logic is the durable part and should be rewired to whatever replaces Uptime Kuma. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/group_vars/all/main.yml | 5 ++++ ansible/infra_vars.yml | 5 ++++ .../deploy_bitcoin_knots_playbook.yml | 22 +++++++++++++++ .../deploy_datum_gateway_playbook.yml | 28 +++++++++++++++++++ .../deploy_forgejo_runner_playbook.yml | 17 +++++++++++ .../forgejo/deploy_forgejo_playbook.yml | 14 ++++++++++ .../fulcrum/deploy_fulcrum_playbook.yml | 22 +++++++++++++++ .../headscale/deploy_headscale_playbook.yml | 14 ++++++++++ .../lnbits/deploy_lnbits_playbook.yml | 14 ++++++++++ .../services/memos/deploy_memos_playbook.yml | 15 ++++++++++ .../mempool/deploy_mempool_playbook.yml | 25 +++++++++++++++++ .../deploy_ntfy_emergency_app_playbook.yml | 14 ++++++++++ .../deploy_personal_blog_playbook.yml | 14 ++++++++++ .../phoenixd/deploy_phoenixd_playbook.yml | 17 +++++++++++ .../deploy_vaultwarden_playbook.yml | 14 ++++++++++ 15 files changed, 240 insertions(+) diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 952df93..36d35f8 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -2,3 +2,8 @@ new_user: counterweight ssh_port: 22 allow_ssh_from: "any" root_domain: contrapeso.xyz + +# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the +# playbooks are kept deliberately — the check logic is meant to be rewired to +# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. +uptime_kuma_enabled: false diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml index 952df93..36d35f8 100644 --- a/ansible/infra_vars.yml +++ b/ansible/infra_vars.yml @@ -2,3 +2,8 @@ new_user: counterweight ssh_port: 22 allow_ssh_from: "any" root_domain: contrapeso.xyz + +# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the +# playbooks are kept deliberately — the check logic is meant to be rewired to +# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. +uptime_kuma_enabled: false diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index f84d627..fcc02b4 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -439,7 +439,18 @@ debug: msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Bitcoin Knots health check and push script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh content: | @@ -534,6 +545,7 @@ mode: '0644' - name: Create systemd service for Bitcoin Knots health check + when: uptime_kuma_enabled | default(false) copy: dest: /etc/systemd/system/bitcoin-knots-healthcheck.service content: | @@ -566,6 +578,7 @@ state: started - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -655,6 +668,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -667,6 +681,7 @@ mode: '0644' - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_bitcoin_knots_monitor.py delegate_to: localhost become: no @@ -707,6 +722,7 @@ when: uptime_kuma_push_url | default('') != '' - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -718,6 +734,7 @@ handlers: - name: Restart bitcoind + when: uptime_kuma_enabled | default(false) systemd: name: bitcoind state: restarted @@ -794,6 +811,7 @@ ignore_errors: yes - name: Display public endpoint + when: uptime_kuma_enabled | default(false) debug: msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" @@ -801,6 +819,7 @@ # Uptime Kuma TCP Monitor for Public P2P # =========================================== - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -873,6 +892,7 @@ mode: '0755' - name: Create temporary config for TCP monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -887,6 +907,7 @@ mode: '0644' - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py delegate_to: localhost become: no @@ -900,6 +921,7 @@ when: tcp_monitor_setup.stdout is defined - name: Clean up TCP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 5fa360c..3fa36d4 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -206,7 +206,18 @@ # =========================================== # Health Check Script + Systemd Timer # =========================================== + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create DATUM Gateway health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/datum-gateway-healthcheck-push.sh content: | @@ -287,6 +298,7 @@ daemon_reload: yes - name: Enable and start datum-gateway health check timer + when: uptime_kuma_enabled | default(false) systemd: name: datum-gateway-healthcheck.timer enabled: yes @@ -296,6 +308,7 @@ # Uptime Kuma Push Monitor Setup # =========================================== - name: Create Uptime Kuma push monitor setup script for DATUM Gateway + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -396,6 +409,7 @@ mode: "0755" - name: Create temporary config for push monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -408,6 +422,7 @@ mode: "0644" - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_datum_gateway_monitor.py delegate_to: localhost become: no @@ -421,6 +436,7 @@ when: monitor_setup.stdout is defined - name: Read push URL from file + when: uptime_kuma_enabled | default(false) slurp: src: /tmp/datum_gateway_push_url.txt delegate_to: localhost @@ -442,6 +458,7 @@ notify: Restart datum-gateway health check timer - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -460,6 +477,7 @@ daemon_reload: yes - name: Restart datum-gateway health check timer + when: uptime_kuma_enabled | default(false) systemd: name: datum-gateway-healthcheck.timer state: restarted @@ -539,6 +557,7 @@ msg: "{{ caddy_reload.stdout_lines + caddy_reload.stderr_lines }}" - name: Display DATUM Gateway dashboard URL + when: uptime_kuma_enabled | default(false) debug: msg: "DATUM Gateway dashboard: https://{{ datum_gateway_domain }}" @@ -546,6 +565,7 @@ # Uptime Kuma HTTP Monitor for Public Dashboard # =========================================== - name: Create Uptime Kuma HTTP monitor setup script for DATUM dashboard + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -621,6 +641,7 @@ mode: "0755" - name: Create temporary config for HTTP monitor + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -634,6 +655,7 @@ mode: "0644" - name: Run Uptime Kuma HTTP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_datum_http_monitor.py delegate_to: localhost become: no @@ -647,6 +669,7 @@ when: http_monitor_setup.stdout is defined - name: Clean up HTTP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -733,6 +756,7 @@ ignore_errors: yes - name: Display public Stratum endpoint + when: uptime_kuma_enabled | default(false) debug: msg: "DATUM Stratum public endpoint: {{ ansible_host }}:{{ datum_gateway_stratum_port }}" @@ -740,6 +764,7 @@ # Uptime Kuma TCP Monitor for Public Stratum # =========================================== - name: Create Uptime Kuma TCP monitor setup script for Stratum + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -816,6 +841,7 @@ mode: "0755" - name: Create temporary config for Stratum TCP monitor + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -830,6 +856,7 @@ mode: "0644" - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_datum_stratum_tcp_monitor.py delegate_to: localhost become: no @@ -843,6 +870,7 @@ when: tcp_monitor_setup.stdout is defined - name: Clean up Stratum TCP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index d94f73c..bdc8428 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -147,7 +147,18 @@ register: runner_active changed_when: false + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Assert runner is running + when: uptime_kuma_enabled | default(false) assert: that: - runner_active.stdout == "active" @@ -155,6 +166,7 @@ # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── - name: Create Uptime Kuma push monitor setup script + when: uptime_kuma_enabled | default(false) copy: dest: /tmp/setup_forgejo_runner_monitor.py content: | @@ -238,6 +250,7 @@ become: no - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: > {{ ansible_playbook_python }} /tmp/setup_forgejo_runner_monitor.py @@ -256,10 +269,12 @@ changed_when: false - name: Parse monitor setup result + when: uptime_kuma_enabled | default(false) set_fact: monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - name: Set push URL + when: uptime_kuma_enabled | default(false) set_fact: uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" @@ -272,6 +287,7 @@ mode: '0755' - name: Create forgejo-runner healthcheck script + when: uptime_kuma_enabled | default(false) copy: dest: "{{ healthcheck_script_path }}" content: | @@ -385,6 +401,7 @@ Timeout: {{ healthcheck_timeout_seconds }}s - name: Clean up temporary monitor setup script + when: uptime_kuma_enabled | default(false) file: path: /tmp/setup_forgejo_runner_monitor.py state: absent diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index ff865de..acb1473 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -118,7 +118,18 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Forgejo + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -193,6 +204,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -206,6 +218,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_forgejo_monitor.py delegate_to: localhost become: no @@ -214,6 +227,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 6b17280..a1548a7 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -259,7 +259,18 @@ debug: msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Fulcrum health check and push script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/fulcrum-healthcheck-push.sh content: | @@ -332,6 +343,7 @@ mode: '0644' - name: Create systemd service for Fulcrum health check + when: uptime_kuma_enabled | default(false) copy: dest: /etc/systemd/system/fulcrum-healthcheck.service content: | @@ -364,6 +376,7 @@ state: started - name: Create Uptime Kuma push monitor setup script for Fulcrum + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -453,6 +466,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -465,6 +479,7 @@ mode: '0644' - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_fulcrum_monitor.py delegate_to: localhost become: no @@ -505,6 +520,7 @@ when: uptime_kuma_push_url | default('') != '' - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -518,6 +534,7 @@ handlers: - name: Restart fulcrum + when: uptime_kuma_enabled | default(false) systemd: name: fulcrum state: restarted @@ -593,6 +610,7 @@ ignore_errors: yes - name: Display public endpoint + when: uptime_kuma_enabled | default(false) debug: msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" @@ -600,6 +618,7 @@ # Uptime Kuma TCP Monitor for Public SSL Port # =========================================== - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -672,6 +691,7 @@ mode: '0755' - name: Create temporary config for TCP monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -686,6 +706,7 @@ mode: '0644' - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py delegate_to: localhost become: no @@ -699,6 +720,7 @@ when: tcp_monitor_setup.stdout is defined - name: Clean up TCP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 7ed81ed..61ba22e 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -264,7 +264,18 @@ become: yes command: systemctl reload caddy + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Headscale + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -339,6 +350,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -352,6 +364,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_headscale_monitor.py delegate_to: localhost become: no @@ -360,6 +373,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index 6df63f5..d596052 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -181,7 +181,18 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for LNBits + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -256,6 +267,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -269,6 +281,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_lnbits_monitor.py delegate_to: localhost become: no @@ -277,6 +290,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index a823135..5ab254e 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -139,7 +139,18 @@ msg: "Memos is running on port {{ memos_port }}. Access via Tailscale at http://{{ memos_tailscale_hostname }}:{{ memos_port }}" handlers: + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Restart memos + when: uptime_kuma_enabled | default(false) systemd: name: memos state: restarted @@ -196,6 +207,7 @@ command: systemctl reload caddy - name: Create Uptime Kuma monitor setup script for Memos + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -275,6 +287,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -288,6 +301,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_memos_monitor.py delegate_to: localhost become: no @@ -296,6 +310,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index c0fe51d..1dfea1b 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -226,7 +226,18 @@ delay: 5 ignore_errors: yes + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Display deployment status + when: uptime_kuma_enabled | default(false) debug: msg: - "Mempool deployment complete!" @@ -239,6 +250,7 @@ # Health Check Scripts for Uptime Kuma Push Monitors # =========================================== - name: Create Mempool MariaDB health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh content: | @@ -273,6 +285,7 @@ mode: '0755' - name: Create Mempool backend health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/mempool-backend-healthcheck-push.sh content: | @@ -307,6 +320,7 @@ mode: '0755' - name: Create Mempool frontend health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh content: | @@ -396,6 +410,7 @@ daemon_reload: yes - name: Enable and start health check timers + when: uptime_kuma_enabled | default(false) systemd: name: "mempool-{{ item }}-healthcheck.timer" enabled: yes @@ -409,6 +424,7 @@ # Uptime Kuma Push Monitor Setup # =========================================== - name: Create Uptime Kuma push monitor setup script for Mempool + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -492,6 +508,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -507,6 +524,7 @@ mode: '0644' - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_mempool_monitors.py delegate_to: localhost become: no @@ -520,6 +538,7 @@ when: monitor_setup.stdout is defined - name: Read push URLs from file + when: uptime_kuma_enabled | default(false) slurp: src: /tmp/mempool_push_urls.yml delegate_to: localhost @@ -573,6 +592,7 @@ when: push_urls is defined - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -637,6 +657,7 @@ state: reloaded - name: Display Mempool URL + when: uptime_kuma_enabled | default(false) debug: msg: "Mempool is now available at https://{{ mempool_domain }}" @@ -644,6 +665,7 @@ # Uptime Kuma HTTP Monitor for Public Endpoint # =========================================== - name: Create Uptime Kuma HTTP monitor setup script for Mempool + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -714,6 +736,7 @@ mode: '0755' - name: Create temporary config for HTTP monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -727,6 +750,7 @@ mode: '0644' - name: Run Uptime Kuma HTTP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_mempool_http_monitor.py delegate_to: localhost become: no @@ -740,6 +764,7 @@ when: http_monitor_setup.stdout is defined - name: Clean up HTTP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index a04d6af..a8d97a6 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -79,7 +79,18 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -159,6 +170,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -172,6 +184,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_ntfy_emergency_app_monitor.py delegate_to: localhost become: no @@ -180,6 +193,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index 63bbae5..af7a1f3 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -82,7 +82,18 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Personal Blog + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -157,6 +168,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -170,6 +182,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_personal_blog_monitor.py delegate_to: localhost become: no @@ -178,6 +191,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index 9e465eb..f20ecdc 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -253,7 +253,18 @@ # =========================================== # Health Check Script + Systemd Timer # =========================================== + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create phoenixd health check script + when: uptime_kuma_enabled | default(false) copy: dest: "{{ phoenixd_healthcheck_script_path }}" content: | @@ -340,6 +351,7 @@ daemon_reload: yes - name: Enable and start phoenixd health check timer + when: uptime_kuma_enabled | default(false) systemd: name: "{{ phoenixd_healthcheck_service_name }}.timer" enabled: yes @@ -349,6 +361,7 @@ # Uptime Kuma Push Monitor Setup # =========================================== - name: Create Uptime Kuma push monitor setup script for phoenixd + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -443,6 +456,7 @@ mode: "0755" - name: Create temporary config for push monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -455,6 +469,7 @@ mode: "0644" - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_phoenixd_monitor.py delegate_to: localhost become: no @@ -468,6 +483,7 @@ when: monitor_setup.stdout is defined - name: Read push URL from file + when: uptime_kuma_enabled | default(false) slurp: src: /tmp/phoenixd_push_url.txt delegate_to: localhost @@ -489,6 +505,7 @@ notify: Restart phoenixd health check timer - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 65f1f64..9868d13 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -111,7 +111,18 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Vaultwarden + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -191,6 +202,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -204,6 +216,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_vaultwarden_monitor.py delegate_to: localhost become: no @@ -212,6 +225,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: From b5b028c3567012b94adf9c7124c89034a0507891 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:43:55 +0200 Subject: [PATCH 30/44] uptime-kuma: deprecation banners on the monitoring-only plays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These five plus the ntfy notification playbook assert on the credentials, so they now fail immediately instead of running — deliberately, before anything is installed. The banner says so and points at archive/uptime_kuma/. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/infra/410_disk_usage_alerts.yml | 13 +++++++++++++ ansible/infra/420_system_healthcheck.yml | 13 +++++++++++++ ansible/infra/430_cpu_temp_alerts.yml | 13 +++++++++++++ ansible/infra/nodito/32_zfs_pool_setup_playbook.yml | 13 +++++++++++++ ansible/infra/nodito/34_nut_ups_setup_playbook.yml | 13 +++++++++++++ .../ntfy/setup_ntfy_uptime_kuma_notification.yml | 13 +++++++++++++ 6 files changed, 78 insertions(+) diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml index 7905cf3..dcd9bdc 100644 --- a/ansible/infra/410_disk_usage_alerts.yml +++ b/ansible/infra/410_disk_usage_alerts.yml @@ -1,3 +1,16 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Deploy Disk Usage Monitoring hosts: managed become: yes diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml index 5a45ce1..05532a7 100644 --- a/ansible/infra/420_system_healthcheck.yml +++ b/ansible/infra/420_system_healthcheck.yml @@ -1,3 +1,16 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Deploy System Healthcheck Monitoring hosts: managed become: yes diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml index b06f0b6..5c9e855 100644 --- a/ansible/infra/430_cpu_temp_alerts.yml +++ b/ansible/infra/430_cpu_temp_alerts.yml @@ -1,3 +1,16 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Deploy CPU Temperature Monitoring hosts: hypervisor become: yes diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 2939ce8..8cdde6a 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -171,6 +171,19 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup ZFS Pool Health Monitoring and Monthly Scrubs hosts: hypervisor become: true diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index 6b366ba..f39fb56 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -250,6 +250,19 @@ - nut-monitor +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup UPS Heartbeat Monitoring with Uptime Kuma hosts: hypervisor become: true diff --git a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml index 8041905..2d3d22a 100644 --- a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml +++ b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml @@ -1,3 +1,16 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup ntfy as Uptime Kuma Notification Channel hosts: monitoring become: no From 338f2ae6362cf100b04714a9ff5fc20faed1eff8 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:43:56 +0200 Subject: [PATCH 31/44] uptime-kuma: annotate config and drop the unused collection lucasheld.uptime_kuma was pinned but never used - every monitor was created by hand-rolled Python. The uptime subdomain stays because the deprecated blocks still template it. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/requirements.yml | 13 ++++++------- ansible/services_config.yml | 3 +++ requirements.txt | 3 +++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ansible/requirements.yml b/ansible/requirements.yml index dd7eef4..16ef1fd 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,11 +1,10 @@ ---- # Ansible Galaxy Collections Requirements # Install with: ansible-galaxy collection install -r requirements.yml -collections: - # Uptime Kuma Ansible Collection - # Used by: infra/41_disk_usage_alerts.yml - # Provides modules to manage Uptime Kuma monitors programmatically - - name: lucasheld.uptime_kuma - version: ">=1.0.0" +# No collections are currently required. +# +# lucasheld.uptime_kuma was pinned here but never used — every monitor was created +# by hand-rolled Python instead. Removed 2026-09-11 along with Uptime Kuma itself. +# See archive/uptime_kuma/. +collections: [] diff --git a/ansible/services_config.yml b/ansible/services_config.yml index 1bc9db0..a20c853 100644 --- a/ansible/services_config.yml +++ b/ansible/services_config.yml @@ -5,6 +5,9 @@ subdomains: # Monitoring Services (on watchtower) ntfy: ntfy + # DEPRECATED 2026-09-11 — Uptime Kuma is decommissioned and this subdomain no + # longer resolves to anything. Kept only because the deprecated monitoring + # blocks still template it into uptime_kuma_api_url. See archive/uptime_kuma/. uptime_kuma: uptime # VPN Infrastructure (on spacey) diff --git a/requirements.txt b/requirements.txt index 972dc4f..653523e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,7 @@ packaging==25.0 pycparser==2.22 PyYAML==6.0.2 resolvelib==1.0.1 +# Only needed by the deprecated Uptime Kuma monitoring blocks, which are kept for +# reference but never run (uptime_kuma_enabled: false). Drop this once they are +# rewired to a replacement. See archive/uptime_kuma/. uptime-kuma-api>=1.2.1 From 8a3fddbe4967f34e43af1265bfeecb6cd22f908c Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 22:43:56 +0200 Subject: [PATCH 32/44] docs: mark Uptime Kuma as decommissioned README, both setup guides and the forgejo-runner notes now point at archive/uptime_kuma/ instead of describing a live service. Co-Authored-By: Claude Opus 5 (1M context) --- 01_infra_setup.md | 6 ++++++ 02_vps_core_services_setup.md | 6 ++++++ README.md | 2 +- ansible/services/forgejo-runner/SETUP.md | 4 +++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/01_infra_setup.md b/01_infra_setup.md index 6ef0978..a8ba9c0 100644 --- a/01_infra_setup.md +++ b/01_infra_setup.md @@ -162,6 +162,12 @@ Note that, by applying these playbooks, both the root user and the `counterweigh ```bash cp ansible/infra_secrets.yml.example ansible/infra_secrets.yml ``` + > **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server + > deployment was removed from this repo; what it monitored and how it was set up is + > preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in + > the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic + > survives for whatever replaces it. The credentials below no longer exist in the vault. + * Edit `ansible/infra_secrets.yml` and add your Uptime Kuma credentials: ```yaml uptime_kuma_username: "admin" diff --git a/02_vps_core_services_setup.md b/02_vps_core_services_setup.md index 19cd122..5d75de9 100644 --- a/02_vps_core_services_setup.md +++ b/02_vps_core_services_setup.md @@ -49,6 +49,12 @@ Checklist: ## Uptime Kuma +> **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server +> deployment was removed from this repo; what it monitored and how it was set up is +> preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in +> the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic +> survives for whatever replaces it. The credentials below no longer exist in the vault. + Uptime Kuma gets used to monitor the availability of services, keep track of their uptime and notify issues. ### Deploy diff --git a/README.md b/README.md index 80dbaf6..89ef092 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Assumes that you've set `ansible/.vault_pass` with `chmod 600`. + Plan install + File based config + Crossbackup to Desky via rsync -* Uptime Kuma +* ~~Uptime Kuma~~ — decommissioned 2026-09-11, see `archive/uptime_kuma/` + Deployed on Vipy + Crossbackup to Desky via rsync * Vaultwarden diff --git a/ansible/services/forgejo-runner/SETUP.md b/ansible/services/forgejo-runner/SETUP.md index a66d295..5cb2a3f 100644 --- a/ansible/services/forgejo-runner/SETUP.md +++ b/ansible/services/forgejo-runner/SETUP.md @@ -25,4 +25,6 @@ ansible-playbook ansible/services/forgejo-runner/deploy_forgejo_runner_playbook. 1. On the VM: `systemctl status forgejo-runner` should show active 2. In Forgejo: **Site Administration** > **Actions** > **Runners** should show the runner as online -3. In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings +3. ~~In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings~~ + *(Uptime Kuma was decommissioned 2026-09-11 — this check no longer applies. The + healthcheck timer still runs on the host but pushes nowhere. See `archive/uptime_kuma/`.)* From b07ed72a92852a8da35d86f52db98f7160b361fb Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:03:25 +0200 Subject: [PATCH 33/44] caddy: add a [caddy] role group and target it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit services/caddy_playbook.yml was the one play still targeting a location group (vps) rather than a role group. The two coincide today — vps is exactly vipy, watchtower and spacey, the three hosts with /etc/caddy/sites-enabled — but adding a fourth VPS that does not run Caddy would have silently pulled it into the play. [caddy:children] is edge + monitoring + vpn_control. Verified the play selects the same three machines before and after. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/inventory.ini | 8 +++++++- ansible/services/caddy_playbook.yml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ansible/inventory.ini b/ansible/inventory.ini index a061637..4643dbb 100644 --- a/ansible/inventory.ini +++ b/ansible/inventory.ini @@ -58,4 +58,10 @@ localhost [managed:children] vps nodito_host -nodito_vms \ No newline at end of file +nodito_vms + +# Hosts that run Caddy and therefore have /etc/caddy/sites-enabled. +[caddy:children] +edge +monitoring +vpn_control diff --git a/ansible/services/caddy_playbook.yml b/ansible/services/caddy_playbook.yml index de98c8f..29e74b2 100644 --- a/ansible/services/caddy_playbook.yml +++ b/ansible/services/caddy_playbook.yml @@ -1,5 +1,5 @@ - name: Install and configure Caddy on Debian 12 - hosts: vps + hosts: caddy become: yes tasks: From cc9340b7cc0801c4dd080a82a4044dcfeacae439 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:10:43 +0200 Subject: [PATCH 34/44] caddy: add the caddy_site role Replaces the four-task Caddy vhost block currently copy-pasted into 10 playbooks. Nothing calls it yet; this commit only adds the role. Verified by rendering all 10 sites through the template and diffing against what the current playbooks produce: 9 of 10 byte-identical. The tenth is datum-gateway, where the resolvers comment is standardised, rewriting one comment line Caddy ignores. Then dry-run against the live hosts (--check, nothing written): - vipy: forgejo, vaultwarden, lnbits, personal-blog, ntfy-emergency-app all report ok/unchanged against the real files - watchtower: ntfy renders identical via caddy_site_body, blank line and {host}{uri} placeholders intact - spacey: headscale renders identical when given the config that is actually running - memos, mempool, datum-gateway report changed - the comment, as expected All 14 site files on all 3 hosts confirmed unchanged afterwards. Two things the build turned up: - Ansible does not template dict *keys*, so caddy_site_basic_auth is a list of {user, hash}. As a dict, a Jinja username passes through literally. The assert refuses a mapping. - `caddy validate` does accept a single site fragment - rc=0 on a good one, rc=1 with a line number on a broken one. This was the plan's one untested claim. A failed validate leaves the live file untouched. The reload is now a handler, so it fires once at end of play rather than immediately; anything needing the new config live mid-play must flush_handlers first. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/caddy_site/README.md | 94 +++++++++++++++++++ ansible/roles/caddy_site/defaults/main.yml | 23 +++++ ansible/roles/caddy_site/handlers/main.yml | 8 ++ ansible/roles/caddy_site/tasks/main.yml | 46 +++++++++ .../roles/caddy_site/templates/site.conf.j2 | 34 +++++++ 5 files changed, 205 insertions(+) create mode 100644 ansible/roles/caddy_site/README.md create mode 100644 ansible/roles/caddy_site/defaults/main.yml create mode 100644 ansible/roles/caddy_site/handlers/main.yml create mode 100644 ansible/roles/caddy_site/tasks/main.yml create mode 100644 ansible/roles/caddy_site/templates/site.conf.j2 diff --git a/ansible/roles/caddy_site/README.md b/ansible/roles/caddy_site/README.md new file mode 100644 index 0000000..6891070 --- /dev/null +++ b/ansible/roles/caddy_site/README.md @@ -0,0 +1,94 @@ +# `caddy_site` + +Writes one Caddy site file into `{{ caddy_sites_dir }}`, makes sure the main +Caddyfile imports that directory, validates the result, and reloads Caddy once. + +Replaces the four-task block that was copy-pasted into 10 playbooks. + +Runs on any host in the `[caddy]` group — `edge` (vipy), `monitoring` +(watchtower) and `vpn_control` (spacey). + +## Usage + +```yaml +- ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: forgejo # -> forgejo.conf + caddy_site_domain: "{{ forgejo_domain }}" + caddy_site_upstream: "localhost:{{ forgejo_port }}" +``` + +Use `include_role`, not a `roles:` block, so the call stays in task order next +to the tasks it depends on. Variables passed this way are scoped to the include +and do not leak into later calls — so **every call must pass everything it +needs**; nothing carries over. + +## Shapes + +Pick exactly one of `caddy_site_upstream`, `caddy_site_root`, `caddy_site_body`. + +| Want | Set | +|---|---| +| `reverse_proxy host:port` | `caddy_site_upstream` | +| static `root *` + `file_server` | `caddy_site_root` | +| anything else | `caddy_site_body` (raw, indented 4 for you) | + +`caddy_site_upstream` accepts two modifiers, which add a block to the +`reverse_proxy`: + +- `caddy_site_headers_up: {"X-Forwarded-Host": "..."}` +- `caddy_site_resolvers: "100.100.100.100"` — Tailscale MagicDNS + +and `caddy_site_basic_auth` wraps the site in a `basic_auth` block. + +## `caddy_site_basic_auth` is a LIST, not a dict + +```yaml +caddy_site_basic_auth: + - user: "{{ datum_dashboard_username }}" + hash: "{{ datum_dashboard_password_hash }}" +``` + +**Ansible does not template dictionary keys.** With `{ "{{ user }}": "hash" }` +the value is rendered and the key is not, so the literal string +`{{ datum_dashboard_username }}` lands in the config file. Found while building +this role; the `assert` refuses a mapping so it cannot happen again. + +## Secrets and `--diff` + +Rendered site files can carry credentials — `datum-gateway.conf` holds a bcrypt +hash — and `--diff` prints rendered content. The template task therefore sets +`diff: "{{ caddy_site_reveal | bool }}"`, default `false`, so `--diff` runs are +safe everywhere. Pass `-e caddy_site_reveal=true` to see what moved on a site +you know is not secret. + +## Validation + +`validate: "caddy validate --adapter caddyfile --config %s"` runs against the +rendered temp file before it is moved into place. Verified on vipy that a single +site fragment validates cleanly (rc=0, `Valid configuration`) and that a +malformed one is rejected (rc=1, with the syntax error and line number). A +failed validate leaves the live file untouched, so a broken config can no longer +reach a running Caddy. + +What it cannot catch is a conflict with the global `/etc/caddy/Caddyfile`. + +## The reload is a handler + +`Reload caddy` fires **once, at the end of the play**, however many sites +notified it. The code this replaced ran `command: systemctl reload caddy` +immediately, mid-play. If a later task in the same play needs the new config to +be live, flush first: + +```yaml +- ansible.builtin.meta: flush_handlers +``` + +## Known intentional difference + +The `resolvers` block is commented `# Use Tailscale MagicDNS to resolve the +upstream hostname` in every case. `datum-gateway` previously said `# Resolve via +Tailscale MagicDNS`. Migrating it therefore rewrites one comment line, which +Caddy ignores. Every other site renders byte-identical to what its playbook +produced. diff --git a/ansible/roles/caddy_site/defaults/main.yml b/ansible/roles/caddy_site/defaults/main.yml new file mode 100644 index 0000000..cafa3dc --- /dev/null +++ b/ansible/roles/caddy_site/defaults/main.yml @@ -0,0 +1,23 @@ +--- +# Required +caddy_site_name: "" # file basename -> .conf +caddy_site_domain: "" # site address line; may hold several, comma separated + +# Pick exactly one shape +caddy_site_upstream: "" # "localhost:3000" -> reverse_proxy +caddy_site_root: "" # filesystem path -> root * + file_server +caddy_site_body: "" # raw escape hatch for one-off sites; wins over both + +# reverse_proxy modifiers +caddy_site_resolvers: "" # "100.100.100.100" for Tailscale MagicDNS +caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"} +# A LIST, not a dict: Ansible does not template dict *keys*, so a Jinja +# expression for the username silently passes through as literal text. +caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}] + +# Placement. caddy_sites_dir comes from services_config.yml; this is the fallback. +caddy_sites_dir: /etc/caddy/sites-enabled + +# Rendered site files can carry credentials (basic_auth hashes), so --diff is +# suppressed by default. Pass -e caddy_site_reveal=true to see what moved. +caddy_site_reveal: false diff --git a/ansible/roles/caddy_site/handlers/main.yml b/ansible/roles/caddy_site/handlers/main.yml new file mode 100644 index 0000000..fb57280 --- /dev/null +++ b/ansible/roles/caddy_site/handlers/main.yml @@ -0,0 +1,8 @@ +--- +# Fires once at the end of the play, however many sites notified it. +# Anything later in the same play that needs the new config live must be +# preceded by `- ansible.builtin.meta: flush_handlers`. +- name: Reload caddy + ansible.builtin.systemd: + name: caddy + state: reloaded diff --git a/ansible/roles/caddy_site/tasks/main.yml b/ansible/roles/caddy_site/tasks/main.yml new file mode 100644 index 0000000..5ad3976 --- /dev/null +++ b/ansible/roles/caddy_site/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: Assert caddy_site parameters are sane + ansible.builtin.assert: + that: + - caddy_site_name | length > 0 + - caddy_site_domain | length > 0 + - (caddy_site_upstream | length > 0) or (caddy_site_root | length > 0) or (caddy_site_body | length > 0) + - caddy_site_basic_auth is not mapping + fail_msg: >- + caddy_site: '{{ caddy_site_name | default("") }}' needs a name, a domain and + one of caddy_site_upstream / caddy_site_root / caddy_site_body. + caddy_site_basic_auth must be a LIST of {user, hash} — Ansible does not template dict keys. + quiet: true + +- name: Ensure Caddy sites-enabled directory exists + ansible.builtin.file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Ensure Caddyfile imports sites-enabled + ansible.builtin.lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + create: yes + mode: '0644' + backup: yes + +# `validate` runs `caddy validate` against the rendered temp file before it is +# moved into place: verified on vipy that a single site fragment validates +# cleanly (rc=0, "Valid configuration") and that a malformed one is rejected +# (rc=1). A failed validate leaves the live file untouched. +- name: "Write Caddy site '{{ caddy_site_name }}'" + ansible.builtin.template: + src: site.conf.j2 + dest: "{{ caddy_sites_dir }}/{{ caddy_site_name }}.conf" + owner: root + group: root + mode: '0644' + validate: "caddy validate --adapter caddyfile --config %s" + diff: "{{ caddy_site_reveal | bool }}" + notify: Reload caddy diff --git a/ansible/roles/caddy_site/templates/site.conf.j2 b/ansible/roles/caddy_site/templates/site.conf.j2 new file mode 100644 index 0000000..5d34c01 --- /dev/null +++ b/ansible/roles/caddy_site/templates/site.conf.j2 @@ -0,0 +1,34 @@ +{{ caddy_site_domain }} { +{% if caddy_site_body %} +{{ caddy_site_body | trim | indent(4, first=True) }} +{% else %} +{% if caddy_site_basic_auth %} + basic_auth { +{% for cred in caddy_site_basic_auth %} + {{ cred.user }} {{ cred.hash }} +{% endfor %} + } +{% endif %} +{% if caddy_site_root %} + root * {{ caddy_site_root }} + file_server +{% endif %} +{% if caddy_site_upstream %} +{% if caddy_site_headers_up or caddy_site_resolvers %} + reverse_proxy {{ caddy_site_upstream }} { +{% for key, value in caddy_site_headers_up.items() %} + header_up {{ key }} {{ value }} +{% endfor %} +{% if caddy_site_resolvers %} + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers {{ caddy_site_resolvers }} + } +{% endif %} + } +{% else %} + reverse_proxy {{ caddy_site_upstream }} +{% endif %} +{% endif %} +{% endif %} +} From 39a1b43577fb6e46b7741ed58134731132dca2de Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:19:01 +0200 Subject: [PATCH 35/44] personal-blog: use the caddy_site role First service on the role. 31 lines of copy-pasted Caddy plumbing become 7. Verified: --check before and after the edit reports the same three unrelated tasks as changed, so the edit introduces nothing. Real run leaves all 14 site files on all 3 hosts byte-identical, and the blog still answers HTTP 200. A second consecutive run reports the site task ok with the handler not firing. Side effect worth noting: the playbook no longer has a perpetually-changed task. `command: systemctl reload caddy` always reported changed; the role's handler only fires when the file actually moves. Co-Authored-By: Claude Opus 5 (1M context) --- .../deploy_personal_blog_playbook.yml | 38 ++++--------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index af7a1f3..96d030f 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -8,7 +8,6 @@ - ./personal_blog_vars.yml vars: personal_blog_subdomain: "{{ subdomains.personal_blog }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" personal_blog_domain: "{{ personal_blog_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -51,36 +50,13 @@ group: www-data mode: '0664' - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy file server configuration for personal blog - copy: - dest: "{{ caddy_sites_dir }}/personal-blog.conf" - content: | - {{ personal_blog_domain }} { - root * {{ personal_blog_web_root }} - file_server - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish the blog through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: personal-blog + caddy_site_domain: "{{ personal_blog_domain }}" + caddy_site_root: "{{ personal_blog_web_root }}" # ═════════════════════════════════════════════════════════════════════════ # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. From 4bee18297892196ffed1cabd0d2f7cbb28d36a4a Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:24:57 +0200 Subject: [PATCH 36/44] ntfy-emergency-app, vaultwarden, forgejo: use the caddy_site role The plain reverse_proxy shape. All three removed a byte-identical 23-line block (verified by md5 of the diff with the service name normalised) and gained the same 7-line include_role call. The caddy_sites_dir self-reference goes with it. Verified in check mode, nothing applied to the hosts yet: - ntfy-emergency-app: site task ok, changed=0 - vaultwarden: site task ok; the one changed task is a pre-existing always-restarts fail2ban step, identical before the edit - forgejo: check mode cannot run this playbook at all - get_url does not download in check mode so the next task fails on "Source /tmp/forgejo not found". Confirmed identical before the edit. Covered instead by the Stage 2 dry-run, which ran the role against vipy with forgejo's real parameters and reported ok/unchanged. All 14 site files on all 3 hosts still byte-identical. Real runs for these three are still outstanding. Co-Authored-By: Claude Opus 5 (1M context) --- .../forgejo/deploy_forgejo_playbook.yml | 37 ++++--------------- .../deploy_ntfy_emergency_app_playbook.yml | 37 ++++--------------- .../deploy_vaultwarden_playbook.yml | 37 ++++--------------- 3 files changed, 21 insertions(+), 90 deletions(-) diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index acb1473..db78e95 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -8,7 +8,6 @@ - ./forgejo_vars.yml vars: forgejo_subdomain: "{{ subdomains.forgejo }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" forgejo_domain: "{{ forgejo_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -88,35 +87,13 @@ enabled: yes state: started - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for forgejo - copy: - dest: "{{ caddy_sites_dir }}/forgejo.conf" - content: | - {{ forgejo_domain }} { - reverse_proxy localhost:{{ forgejo_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish Forgejo through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: forgejo + caddy_site_domain: "{{ forgejo_domain }}" + caddy_site_upstream: "localhost:{{ forgejo_port }}" # ═════════════════════════════════════════════════════════════════════════ # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index a8d97a6..7379d5f 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -8,7 +8,6 @@ - ./ntfy_emergency_app_vars.yml vars: ntfy_emergency_app_subdomain: "{{ subdomains.ntfy_emergency_app }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" ntfy_emergency_app_domain: "{{ ntfy_emergency_app_subdomain }}.{{ root_domain }}" ntfy_service_domain: "{{ subdomains.ntfy }}.{{ root_domain }}" ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" @@ -49,35 +48,13 @@ args: chdir: "{{ ntfy_emergency_app_dir }}" - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for ntfy-emergency-app - copy: - dest: "{{ caddy_sites_dir }}/ntfy-emergency-app.conf" - content: | - {{ ntfy_emergency_app_domain }} { - reverse_proxy localhost:{{ ntfy_emergency_app_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish ntfy-emergency-app through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: ntfy-emergency-app + caddy_site_domain: "{{ ntfy_emergency_app_domain }}" + caddy_site_upstream: "localhost:{{ ntfy_emergency_app_port }}" # ═════════════════════════════════════════════════════════════════════════ # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 9868d13..74e87d8 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -8,7 +8,6 @@ - ./vaultwarden_vars.yml vars: vaultwarden_subdomain: "{{ subdomains.vaultwarden }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" vaultwarden_domain: "{{ vaultwarden_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -81,35 +80,13 @@ name: fail2ban state: restarted - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for vaultwarden - copy: - dest: "{{ caddy_sites_dir }}/vaultwarden.conf" - content: | - {{ vaultwarden_domain }} { - reverse_proxy localhost:{{ vaultwarden_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish Vaultwarden through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: vaultwarden + caddy_site_domain: "{{ vaultwarden_domain }}" + caddy_site_upstream: "localhost:{{ vaultwarden_port }}" # ═════════════════════════════════════════════════════════════════════════ # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. From 82fca48b0877008c5c2668ae0b5047d24a02eabb Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:29:06 +0200 Subject: [PATCH 37/44] lnbits, memos, mempool: use the caddy_site role lnbits is the header_up shape; memos and mempool are the Tailscale MagicDNS shape. 108 lines removed, 25 added. mempool was the one playbook already reloading Caddy correctly (systemd: state: reloaded rather than command: systemctl reload caddy), so its end marker differed - the role's handler does the same thing. Verified: - lnbits: full --check, site task ok, byte-identical to the live file - memos, mempool: --check --diff via --limit edge shows exactly one added line each, the standardised MagicDNS comment. Both playbooks fail earlier in check mode on their VM play ("Extract memos binary", the same download-does-not-happen-in-check-mode artifact as forgejo), but the edits are confined to the hosts: edge play - memos at line 169+, play 2 starts at 159; mempool at 617+, play 2 starts at 606. The added comment means the next real run of memos/mempool rewrites one comment line. Those two host files were already stale against their playbooks before this change. All 14 site files on all 3 hosts still byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- .../lnbits/deploy_lnbits_playbook.yml | 43 ++++------------- .../services/memos/deploy_memos_playbook.yml | 43 ++++------------- .../mempool/deploy_mempool_playbook.yml | 47 ++++--------------- 3 files changed, 25 insertions(+), 108 deletions(-) diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index d596052..65bdfbf 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -8,7 +8,6 @@ - ./lnbits_vars.yml vars: lnbits_subdomain: "{{ subdomains.lnbits }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" lnbits_domain: "{{ lnbits_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -147,39 +146,15 @@ enabled: yes state: started - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for lnbits - copy: - dest: "{{ caddy_sites_dir }}/lnbits.conf" - content: | - {{ lnbits_domain }} { - reverse_proxy localhost:{{ lnbits_port }} { - header_up X-Forwarded-Host {{ lnbits_domain }} - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish LNBits through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: lnbits + caddy_site_domain: "{{ lnbits_domain }}" + caddy_site_upstream: "localhost:{{ lnbits_port }}" + caddy_site_headers_up: + X-Forwarded-Host: "{{ lnbits_domain }}" # ═════════════════════════════════════════════════════════════════════════ # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index 5ab254e..83c187e 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -166,45 +166,18 @@ - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for memos (via Tailscale) - copy: - dest: "{{ caddy_sites_dir }}/memos.conf" - content: | - {{ memos_domain }} { - reverse_proxy {{ memos_tailscale_hostname }}:{{ memos_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish Memos through Caddy (via Tailscale) + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: memos + caddy_site_domain: "{{ memos_domain }}" + caddy_site_upstream: "{{ memos_tailscale_hostname }}:{{ memos_port }}" + caddy_site_resolvers: "100.100.100.100" - name: Create Uptime Kuma monitor setup script for Memos when: uptime_kuma_enabled | default(false) diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 1dfea1b..ba3e9e9 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -614,47 +614,16 @@ vars: mempool_subdomain: "{{ subdomains.mempool }}" mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for Mempool - copy: - dest: "{{ caddy_sites_dir }}/mempool.conf" - content: | - {{ mempool_domain }} { - reverse_proxy mempool-box:{{ mempool_frontend_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - systemd: - name: caddy - state: reloaded + - name: Publish Mempool through Caddy (via Tailscale) + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: mempool + caddy_site_domain: "{{ mempool_domain }}" + caddy_site_upstream: "mempool-box:{{ mempool_frontend_port }}" + caddy_site_resolvers: "100.100.100.100" - name: Display Mempool URL when: uptime_kuma_enabled | default(false) From 16cbd189b8a9b715ad14a92563103533efaa1f4e Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:37:38 +0200 Subject: [PATCH 38/44] ntfy, datum-gateway, headscale: use the caddy_site role Completes Stage 3. No hand-rolled Caddy plumbing remains anywhere: `grep sites-enabled` outside roles/ returns nothing, and so does `grep "systemctl reload caddy"`. ntfy uses caddy_site_body for its plain-HTTP listener and @httpget redirect. Verified ok/unchanged against watchtower; the one other changed task is a pre-existing "Update APT cache". datum-gateway keeps a whole-Caddyfile validate after the role call. The role validates its own fragment, but only a whole-file validate catches a conflict between two sites, and this playbook was the only one that ever had it. Its two debug tasks that echoed command output are gone with the commands. headscale is the one that mattered. Its playbook wrote `reverse_proxy localhost:8080`, but spacey is actually running a /admin* route in front of Headplane behind Caddy basic auth. Running that playbook would have deleted the admin route and its auth - a hazard that predates this work. It now renders the config that is really there, verified ok/unchanged via --start-at-task (the play cannot reach Caddy in check mode: "Install headscale package" fails because the .deb is not really downloaded, before and after this edit alike). Supporting changes for headscale: - headscale_ui_password_hash added to infra_secrets.yml and the identical group_vars/all/vault.yml, read from the live config on spacey. The vault already had headscale_ui_username (= counterweight, confirmed) and headscale_ui_password; I did not verify the password is the plaintext of this hash. - headplane_port added to headscale_vars.yml. - The role's handler now sets become: true. Handlers do not inherit become from the task that notified them, and this play runs become: no. - The include uses `apply: become: yes`; `become:` on an include_role is rejected outright. All 14 site files on all 3 hosts still byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/group_vars/all/vault.yml | 103 ++++++++++-------- ansible/infra_secrets.yml | 103 ++++++++++-------- ansible/roles/caddy_site/handlers/main.yml | 4 + .../deploy_datum_gateway_playbook.yml | 69 +++--------- .../headscale/deploy_headscale_playbook.yml | 60 +++++----- ansible/services/headscale/headscale_vars.yml | 3 + .../services/ntfy/deploy_ntfy_playbook.yml | 50 +++------ 7 files changed, 184 insertions(+), 208 deletions(-) diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index 798e2d8..d1f18ec 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,45 +1,60 @@ $ANSIBLE_VAULT;1.1;AES256 -65343264303166346334396163363362326334626531336363363766326135393462373564313539 -6633346163343664393232363965383334303563396236360a316265623239333431636133306663 -65623237393564323936303936633036343137646637633963396166313462626165616463616665 -3235373562626238640a636163623535376565373835653831666361636563616230353639626131 -38393530363461346334356437613934306330666563326661343465653362313038346535386337 -62623964633764343964653061373137366539326433396231396339313231633632353465323139 -33333866383834376165303964353639386234646535626333363932333764646137663036303530 -32326635343631623535323830333934376466323338353234323464666435383436633139343037 -62666333383231356163383530353339336230373431616665333638633234656231393166303639 -32386134386264613839313564316164336461373738646630636630363731643066633132336636 -64663532663661653539653431353763303436323234383864353064333631643435633733366435 -32656332626633313134623361343965643962663134613363626265353865643738363738316232 -62356136623166303038653830393138646637636335366362323239316333616331383765663861 -33643331323766646463663862396232306233343134326436326235336165616662383636616539 -39373962336264636363303734343666646534343733666632626338393936393032626130643233 -30633963656166343834396431323233363036353165623934336532333662323932336630636235 -63326631366234383732316464383364316337393034303433646365343763633135336336376161 -33326436303863326666616234396438623864333538313363623261353962336430316132343830 -36353761326133666166393930343233643861366138363930623362336566386230316236353365 -65663632383436333536636161393133316334376262653330303238636431353966646666663665 -30386137623066343135616534323336313162643066636330323934646162303362373838313532 -61386163356537613334616537656134373835326265393765303436396233383239393365336136 -64623130383538306331323939386634316663663263306362363830383930346232313965613035 -33353235663230663635316136666335343233646539613832643431393134303865616232636435 -66366437353863336235646438353431616465396365373562636461623938303663326461656134 -66353735343237363164326236663363376333316163353462626234393861663535323463623233 -66623934336662323830616362653035353563343564636431383837643961303637366131633132 -33376133376465623030643935643737636339333236316566636663323934626265633863343963 -38626563323166343264643465626264393838643330393739386330656461656561306539653232 -63393838626233386531353063366131366237633162393537363963633361386238643232336230 -36363062386262303364303736373061316265373730366437326166336163386335623665323234 -35383865646466363534383833653764303833323230613863313735393264336132303934633465 -65616138636134613361316539343739353735666538653438333264623034306633663634306438 -33643165353534623039626536636433373963383866303535333931633633393138663464336562 -36643432366233323235613139626236346135633237613265343933646434373939396265626366 -34313835346463303132326537323166363032306133623662343736393435383330356433306138 -65653931613832313832323936353638303937613863646430633266306330656135386533306462 -30663366396566393134653663356531316639303635343236666333363637636433336533356663 -35346261363134326165316463626439306130653165636134616434663736616666643963336464 -61643834613164336234386237393333636234306162306563363430646262363963366666653666 -64323036323731626236343361613566643565666138386631633462613836613766626562353934 -32363936626466346464356131653232326330386239666638626333346238623964383666393961 -38393264663636353161663636393733396332333838333962393439346533353332653362356230 -62613039376539636363 +61356165613635386631393135656434646436303665313031346566323336313138353433316463 +3363323534613064643132663335623238366431393062340a346538396662306537663163623366 +38626166383933616331623231373137306562623637313263333237633661663436666266616433 +3862346438643638650a306634333535653633613534646630386131333236366538333765323333 +38656163303837303732663561373232393132343331663164656262393730326434373731333636 +36613538646431396536363936336562616431656665653965373864633366663836353434626434 +34383932373461333564303439623565383661646365386665393831383463663662356536356236 +36373964666236626465366161636135393734356536633466383262326537343833636630343738 +62353066316131613737373162643363653662656261363465386364323962656537373061373032 +33343763353464383438363438343965653532393831343930393562633630383932653862623637 +38386239353237356631646436356166373961333464396639383538383662326534343339313330 +65373339356364636634616532363832386631323062363530313861336238353261353334306235 +62616338396431316537346638656365356564346666366366343638356261623664393263323937 +30343261363562383332323462336435376664386134646562643836363834313237373631353731 +35653535663864643266313332356635363262363533663232656531373130336539633066376139 +32316665393831663035623962656364363831333563366135636164346335383738363336663566 +63356532643563393939383635386462663561386434323939303431653438653131363538383034 +33623933333464363032656636643033326162626163353633343062633966343332383138363963 +36383831663562616533316436366566323061386535343538393861383462333166343562316633 +30623665623035393537393965626363323132656433313339396233356666346634316332616336 +37363635363330323230373332326565343530653335383437373230366563366237633665626331 +66623336626230663361636439316337393865383035326136653264666438666566646132353036 +38616264313833316536623238633339373466613866626366383835656638623863323838653030 +63303938376164653966356435386333363731656666313234663535666165646233313137343563 +36386437393139656438333262383437656666343831313239323961373637653163643664356565 +62346431393133656530316262303763646165643336396661666431383436323562336137653031 +62626538373839613734396366653065306534636630346338316237616161613037616364356431 +38643965376133616161336633383664326230383435363334353137303162663738313331346238 +39356161616533616134356231323530306338333162343363353531303263636632613036386638 +63633136386232306234323936303563646466313935326631396565383432386130656638616266 +32383334363237336539396665336366643764633131643663643137376438323666326435626461 +64346633636431393137633537306431646564386565303933636434386462346630626537346438 +32623333666133303061646564366366326665363163396262633164323631636337346130303239 +38373936663337356134666132303165393365663763396362623434633737373538653566646134 +62316164396438303532616266313062326666633130656338653139376634306664633031333037 +31636166306565353334633435656233336233363664306264626237623366336161303134353433 +63383739666462623336386537346662633666626466393039653439346436653937633537396436 +35393339383066326630353066623132333034656539363561346462626265363263303535343961 +39316461616630326539613731303039613736393633373338646266323938326162373831346336 +30656130343463366534323030646238313465306266383034623065636665623366333063383736 +38373063393837306462303564643962373334343139626338623935336435643730646532633630 +66363730386636633639346463363365343239373265303738353732653633653437636130636664 +39386365353334653765343335303263363461313965383664326563333734626533376436626530 +63353163323637303730353564383733653365613635353764333266393532653663326533646132 +39313763373735323835626437306435373238653432393936643165663663656665316132653330 +35383038353532656434366336346235363563636264303734633138323963396562306232646236 +38366536306561653937336333373434336164663336613839353439356435333833396363636437 +38323934613735643363656233333037336465336564313966623063376566663030303230323262 +65643564666534326234306164343365383632333061316238623565353538313538396364313337 +35336230313339643736653238386231343661623337306236383665356632366236356335323530 +66373831616239376231636361333430343433303233393066323865663434643433303832373262 +36366534646161323130393931626362626139663139643263366639656531613436313533363130 +65643736363833333939613566663339623964333262643863333030623138633464386238613934 +31323639333234336264313663376465323737353766643839303665313737336534386665363034 +30326439666433373232306136306365343764643434306561343339353132346430646436343362 +32333765616262363930353435616563333736313533653339656231316230346166363335356638 +37333561366339613136306438306130343230663732333862663838396463623661303961336433 +65646332373139303462303633346432366530643130366133363937653739653036366136373434 +343864373734666431326239373866313734 diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml index 798e2d8..4803d86 100644 --- a/ansible/infra_secrets.yml +++ b/ansible/infra_secrets.yml @@ -1,45 +1,60 @@ $ANSIBLE_VAULT;1.1;AES256 -65343264303166346334396163363362326334626531336363363766326135393462373564313539 -6633346163343664393232363965383334303563396236360a316265623239333431636133306663 -65623237393564323936303936633036343137646637633963396166313462626165616463616665 -3235373562626238640a636163623535376565373835653831666361636563616230353639626131 -38393530363461346334356437613934306330666563326661343465653362313038346535386337 -62623964633764343964653061373137366539326433396231396339313231633632353465323139 -33333866383834376165303964353639386234646535626333363932333764646137663036303530 -32326635343631623535323830333934376466323338353234323464666435383436633139343037 -62666333383231356163383530353339336230373431616665333638633234656231393166303639 -32386134386264613839313564316164336461373738646630636630363731643066633132336636 -64663532663661653539653431353763303436323234383864353064333631643435633733366435 -32656332626633313134623361343965643962663134613363626265353865643738363738316232 -62356136623166303038653830393138646637636335366362323239316333616331383765663861 -33643331323766646463663862396232306233343134326436326235336165616662383636616539 -39373962336264636363303734343666646534343733666632626338393936393032626130643233 -30633963656166343834396431323233363036353165623934336532333662323932336630636235 -63326631366234383732316464383364316337393034303433646365343763633135336336376161 -33326436303863326666616234396438623864333538313363623261353962336430316132343830 -36353761326133666166393930343233643861366138363930623362336566386230316236353365 -65663632383436333536636161393133316334376262653330303238636431353966646666663665 -30386137623066343135616534323336313162643066636330323934646162303362373838313532 -61386163356537613334616537656134373835326265393765303436396233383239393365336136 -64623130383538306331323939386634316663663263306362363830383930346232313965613035 -33353235663230663635316136666335343233646539613832643431393134303865616232636435 -66366437353863336235646438353431616465396365373562636461623938303663326461656134 -66353735343237363164326236663363376333316163353462626234393861663535323463623233 -66623934336662323830616362653035353563343564636431383837643961303637366131633132 -33376133376465623030643935643737636339333236316566636663323934626265633863343963 -38626563323166343264643465626264393838643330393739386330656461656561306539653232 -63393838626233386531353063366131366237633162393537363963633361386238643232336230 -36363062386262303364303736373061316265373730366437326166336163386335623665323234 -35383865646466363534383833653764303833323230613863313735393264336132303934633465 -65616138636134613361316539343739353735666538653438333264623034306633663634306438 -33643165353534623039626536636433373963383866303535333931633633393138663464336562 -36643432366233323235613139626236346135633237613265343933646434373939396265626366 -34313835346463303132326537323166363032306133623662343736393435383330356433306138 -65653931613832313832323936353638303937613863646430633266306330656135386533306462 -30663366396566393134653663356531316639303635343236666333363637636433336533356663 -35346261363134326165316463626439306130653165636134616434663736616666643963336464 -61643834613164336234386237393333636234306162306563363430646262363963366666653666 -64323036323731626236343361613566643565666138386631633462613836613766626562353934 -32363936626466346464356131653232326330386239666638626333346238623964383666393961 -38393264663636353161663636393733396332333838333962393439346533353332653362356230 -62613039376539636363 +34616531626537376463326332306434383163393761363536633133363161373631376437653234 +3831656536313531343433376264643261396634633965370a303931336236323766313065636535 +31343338616364386132623232373266663665353563666535383232666262663062323864303932 +3439373734316639650a633732333064633335383833663261326464336538636666323063646331 +66653464636534663935616461636132613162303530663237376661343338396133323431623561 +30303563303465633231306339326561313533333536376433393130376634303932643339656336 +30616130343733626162366561393939393766623138616537393032376665396139333561623830 +33623130336235633034666561383165616439316334323664623661343438663733373833306335 +30396165313731636435636263353133326135346236323030653734353831626238666663656364 +62386164363361373464383730333733626333333736306535336635613634646535383237326332 +37636532663933663739303634333235343137333363316430643263326231613635643633636437 +30336462633233396430633939616661643261666136656363363461343230653738613166346434 +63376363666462346539343063343562623962303130616535306439653134633630353963373337 +64313239353162363862643432666438383330636638333464323731643163643236313535313030 +65656231383738356537326661663163376634613031396436646633376139313237343534653065 +64323532346133313265346334353530393633396339316330366536646565353836396662303866 +65373030393463306664623235626437613965303837313365373632643935656630303936373837 +64663539623764656636613263346638376665626262663430333231633735653563643864343835 +62303637383332653366383066326536316336306539623230353066343739316430356437313365 +33323031613130643832636133636263303565623330306135333762363036633737343933623437 +33633436363062643436376632336235666330656265316333323361316430656631383734343238 +65616366666536363866343361666563643336306532666332643230656264303565353032393932 +36613738326239376161383064646634623639356439663965323237303361343838373737636235 +39656465333063633136643463356564636537326339653033633063366265636138656466646264 +32643961343863656464353439346531633036613138353333353631626665396239326465303632 +39336664333038636435366333623336353963323535316436646136313739373537376537353735 +66333265633232346536313933333836643439656666636236626266323333383935383565616139 +38626133356364613562393664643563346539366364636332613162663337356232333735393431 +30343635393462356431646336636663333735343164616261343836363366616162613563613066 +66333034613234626431356536323832393466363135653231336238356334353638613838626430 +63636630313533363563376161663637353362383431663130646666646433353864373736613662 +62343666633862643839666532656132653939663437373164393536653739376265303130353636 +62616538326330333136626238306133363634326365366637363961333635356133613564373131 +64626139656633663065353862343135393233333231386166346532656534623361623061336434 +37616565626134333863376532393861376132323434656433613731613834633963386635353336 +39653263346637303836663763653264396139383363616461393333333362616332663863363834 +61613832326466646133626337663364333566326633313664613236346135653332373035313034 +62356530313065393737393634313166313039616635363365633031336434396463326462316236 +62353838636533303664663132353838636632353733316639643964666139383539333761663531 +37366530373165653063303032383461326535613336343635626538373635356266396666346133 +61383434323565306438356634646338666232363238393932353365363461376130323363363236 +34643862653937353461383265663933646264373365623465313666643662633334366437646434 +34343332613832366235663563353435323738356339383338333361383561336366376238393237 +38656265663164376666616366623062393530366261623361383365376235643035353838313235 +32396430376139313430316439333539343266393965353030636638656661316137313437663832 +64366661323865383262323036613230316233666566386133363633303931323464663238323861 +62333132346165353830653062353763306232366563323430383933653163373562326536393362 +66353931376262383931613931303034356537363137366235643430323465623162363031373034 +64653563666533363062333831336463376530306639616631303266386136666233653638653732 +31313537393838333730303235363661326333646531653935633935373362303732653565656438 +36626330623063373832333739643666396135653838643164376264393863646433353036643731 +34623636623366353466333237373265323936643134613230326565303663306431373634356334 +37613732393262626135306632623336613364323632316636623037656538313236666562633438 +37306361333130376538366232393637646263656239303431623963626265386634313735373162 +64393733303132383464386330626135336237303563326238633437346164333738646431333730 +62616533363638643261306565373835303832313539656564386132393863373039653938616261 +36343031356361336563316230343433633033353130373933313361326164633765316433616162 +62356338393634653330623933313433343630386264396337373532376335316362333164363963 +363663666365336564663232363462613162 diff --git a/ansible/roles/caddy_site/handlers/main.yml b/ansible/roles/caddy_site/handlers/main.yml index fb57280..4bfa7ff 100644 --- a/ansible/roles/caddy_site/handlers/main.yml +++ b/ansible/roles/caddy_site/handlers/main.yml @@ -2,7 +2,11 @@ # Fires once at the end of the play, however many sites notified it. # Anything later in the same play that needs the new config live must be # preceded by `- ansible.builtin.meta: flush_handlers`. +# become is explicit because handlers do not inherit it from the task that +# notified them. headscale's play runs become: no and elevates per task, so +# without this the reload would run unprivileged and fail. - name: Reload caddy + become: true ansible.builtin.systemd: name: caddy state: reloaded diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 3fa36d4..860b765 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -498,64 +498,29 @@ vars: datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: "0755" + - name: Publish the DATUM Gateway dashboard through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: datum-gateway + caddy_site_domain: "{{ datum_gateway_domain }}" + caddy_site_upstream: "knots-box:{{ datum_gateway_api_port }}" + caddy_site_resolvers: "100.100.100.100" + caddy_site_basic_auth: + - user: "{{ datum_dashboard_username }}" + hash: "{{ datum_dashboard_password_hash }}" - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: "import sites-enabled/*" - insertafter: EOF - state: present - backup: yes - create: yes - mode: "0644" - - - name: Create Caddy reverse proxy config for DATUM Gateway dashboard - copy: - dest: "{{ caddy_sites_dir }}/datum-gateway.conf" - content: | - {{ datum_gateway_domain }} { - basic_auth { - {{ datum_dashboard_username }} {{ datum_dashboard_password_hash }} - } - reverse_proxy knots-box:{{ datum_gateway_api_port }} { - # Resolve via Tailscale MagicDNS - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: "0644" - - - name: Validate Caddy config - command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile - register: caddy_validate + # The role validates the site fragment on its own. This re-validates the + # whole assembled Caddyfile, which is the only thing that catches a + # conflict between this site and another. Kept from the hand-rolled + # version; the other nine services never had it. + - name: Validate the assembled Caddyfile + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile changed_when: false - - name: Display Caddy validation output - debug: - msg: "{{ caddy_validate.stdout_lines + caddy_validate.stderr_lines }}" - - - name: Reload Caddy - command: systemctl reload caddy - register: caddy_reload - - - name: Display Caddy reload output - debug: - msg: "{{ caddy_reload.stdout_lines + caddy_reload.stderr_lines }}" - - name: Display DATUM Gateway dashboard URL when: uptime_kuma_enabled | default(false) debug: diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 61ba22e..527e4c2 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -8,7 +8,6 @@ - ./headscale_vars.yml vars: headscale_subdomain: "{{ subdomains.headscale }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" headscale_base_domain: "tailnet.{{ root_domain }}" headscale_namespace: "{{ service_settings.headscale.namespace }}" @@ -230,39 +229,34 @@ port: '3478' proto: udp - - name: Ensure Caddy sites-enabled directory exists - become: yes - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - become: yes - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for headscale - become: yes - copy: - dest: "{{ caddy_sites_dir }}/headscale.conf" - content: | - {{ headscale_domain }} { - reverse_proxy localhost:{{ headscale_port }} + - name: Publish headscale through Caddy + ansible.builtin.include_role: + name: caddy_site + # This play is become: no and elevates per task. `apply` is how an + # include_role passes become down to the role's tasks - `become:` on + # the include itself is rejected. The role's handler sets its own. + apply: + become: yes + vars: + caddy_site_name: headscale + caddy_site_domain: "{{ headscale_domain }}" + # Raw body, and it must stay raw: the /admin* route in front of + # Headplane is not expressible as a plain reverse_proxy. The previous + # version of this task wrote only `reverse_proxy localhost:8080`, which + # would have deleted the admin route and its auth on the next run. + caddy_site_body: | + @headplane { + path /admin* } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - become: yes - command: systemctl reload caddy + handle @headplane { + basicauth { + {{ headscale_ui_username }} {{ headscale_ui_password_hash }} + } + reverse_proxy http://localhost:{{ headplane_port }} + } + # Headscale API is protected by its own API key authentication + # All API operations require a valid Bearer token in the Authorization header + reverse_proxy * http://localhost:{{ headscale_port }} # ═════════════════════════════════════════════════════════════════════════ # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index 0b0ee50..05fc040 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -22,3 +22,6 @@ remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" # Local backup local_backup_dir: "{{ lookup('env', 'HOME') }}/headscale-backups" backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/headscale_backup.sh" + +# Headplane (headscale admin UI), proxied at /admin* behind Caddy basic auth +headplane_port: 3000 diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml index d030123..61fafe1 100644 --- a/ansible/services/ntfy/deploy_ntfy_playbook.yml +++ b/ansible/services/ntfy/deploy_ntfy_playbook.yml @@ -8,7 +8,6 @@ - ./ntfy_vars.yml vars: ntfy_subdomain: "{{ subdomains.ntfy }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" tasks: @@ -76,42 +75,23 @@ shell: | (echo "{{ ntfy_password }}"; echo "{{ ntfy_password }}") | ntfy user add --role=admin "{{ ntfy_username }}" - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' + - name: Publish ntfy through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: ntfy + caddy_site_domain: "{{ ntfy_domain }}, http://{{ ntfy_domain }}" + # Raw body: ntfy needs a plain-HTTP listener for its CLI/app clients, + # with only GETs to the docs and topic paths redirected to HTTPS. + caddy_site_body: | + reverse_proxy 127.0.0.1:{{ ntfy_port }} - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for ntfy - copy: - dest: "{{ caddy_sites_dir }}/ntfy.conf" - content: | - {{ ntfy_domain }}, http://{{ ntfy_domain }} { - reverse_proxy 127.0.0.1:{{ ntfy_port }} - - @httpget { - protocol http - method GET - path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) - } - redir @httpget https://{host}{uri} + @httpget { + protocol http + method GET + path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + redir @httpget https://{host}{uri} handlers: - name: Restart ntfy From c4094b692fddde6efc53b37cbadab632b1da1b17 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:50:15 +0200 Subject: [PATCH 39/44] bitcoin-knots, fulcrum, datum-gateway: add and use the socket_proxy role Three near-identical hosts: edge plays become one role plus three short calls. 183 lines removed, 34 added, plus a 111-line role. Verified before touching any playbook: all six live units on vipy reproduced byte-identically. Then --limit edge --check per playbook - bitcoin-knots and fulcrum changed=0; datum-gateway changed=2, both attributable to the already known caddy_site comment line and the Reload caddy handler it triggers. The 6 units and 14 Caddy files on the hosts are byte-identical afterwards. PLAN_4 claimed these three plays had "no behavioural drift at all". That was wrong - it came from a diff truncated by head -60. The live bitcoin-p2p-proxy units carry four settings this playbook never wrote: .socket Documentation=, FreeBind=true .service Documentation=, TimeoutStopSec=5, StandardOutput=journal, StandardError=journal FreeBind is the one that matters: it lets the socket bind to an address that is not up yet, so without it the socket can fail to start on boot. Running the bitcoin-knots playbook would have stripped it. Same class of hazard as headscale. The role expresses all four; bitcoin-p2p is the only caller that passes any. Also: UFW treats the rule comment as part of the rule. datum-stratum's live comment is "DATUM Gateway Stratum public access" but the role's derived default produced "DATUM Stratum public access", which rewrote the rule. Caught in the dry-run; datum now passes the comment explicitly. Two deliberate differences from the original, both documented in the README: ignore_errors: yes on the upstream check became failed_when: false, and the handler restarts the .socket, which drops connections open through it - it fires only when a unit file actually changes. The inert Uptime Kuma TCP monitor blocks stay in the playbooks rather than being pulled into a new role (12/12/18 guarded tasks). Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/socket_proxy/README.md | 62 +++++++++++++++ ansible/roles/socket_proxy/defaults/main.yml | 17 +++++ ansible/roles/socket_proxy/handlers/main.yml | 8 ++ ansible/roles/socket_proxy/tasks/main.yml | 54 +++++++++++++ .../socket_proxy/templates/proxy.service.j2 | 18 +++++ .../socket_proxy/templates/proxy.socket.j2 | 14 ++++ .../deploy_bitcoin_knots_playbook.yml | 76 ++++--------------- .../deploy_datum_gateway_playbook.yml | 72 +++--------------- .../fulcrum/deploy_fulcrum_playbook.yml | 69 ++--------------- 9 files changed, 207 insertions(+), 183 deletions(-) create mode 100644 ansible/roles/socket_proxy/README.md create mode 100644 ansible/roles/socket_proxy/defaults/main.yml create mode 100644 ansible/roles/socket_proxy/handlers/main.yml create mode 100644 ansible/roles/socket_proxy/tasks/main.yml create mode 100644 ansible/roles/socket_proxy/templates/proxy.service.j2 create mode 100644 ansible/roles/socket_proxy/templates/proxy.socket.j2 diff --git a/ansible/roles/socket_proxy/README.md b/ansible/roles/socket_proxy/README.md new file mode 100644 index 0000000..cebb95f --- /dev/null +++ b/ansible/roles/socket_proxy/README.md @@ -0,0 +1,62 @@ +# `socket_proxy` + +Exposes a service running on a private Tailscale host through a public TCP port +on an edge machine, using `systemd-socket-proxyd`. Writes a `.socket` and a +`.service` unit, enables the socket, opens the UFW port, and checks the upstream +is reachable. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: fulcrum-ssl # -> fulcrum-ssl-proxy.{socket,service} + socket_proxy_description: "Fulcrum SSL" # -> "Fulcrum SSL Proxy Socket" + socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" +``` + +`socket_proxy_upstream_port` defaults to `socket_proxy_listen_port`, which is +what all three current callers want. + +## Optional unit settings + +These exist because the **live** `bitcoin-p2p-proxy` units on vipy carried +settings the playbook never wrote. Somebody added them by hand, so running +`deploy_bitcoin_knots_playbook.yml` would have silently removed them: + +| Variable | Emits | Why it matters | +|---|---|---| +| `socket_proxy_free_bind` | `FreeBind=true` in `[Socket]` | Lets the socket bind to an address that is not up yet. Without it the socket can fail to start on boot. | +| `socket_proxy_documentation` | `Documentation=` in both units | Cosmetic. | +| `socket_proxy_timeout_stop_sec` | `TimeoutStopSec=` | Bounds how long a stop can hang. | +| `socket_proxy_log_to_journal` | `StandardOutput=journal` + `StandardError=journal` | Cosmetic on modern systemd, which defaults to the journal anyway. | + +Only `bitcoin-p2p` passes any of them. + +## `socket_proxy_ufw_comment` + +Defaults to `" public access"`, which reproduces the live rule +comment for bitcoin-p2p and fulcrum-ssl. **datum-stratum must pass it +explicitly** — its live comment is `DATUM Gateway Stratum public access` while +the derived default would be `DATUM Stratum public access`, and UFW treats the +comment as part of the rule, so the mismatch rewrites the rule on every run. + +## The upstream check never fails the play + +`wait_for` on the upstream carries `failed_when: false`. The proxy is correctly +configured whether or not the backend happens to be up, and this is the one task +that depends on another machine. The original plays used `ignore_errors: yes`, +which prints a red "ignoring" line; `failed_when: false` is the quieter +equivalent. + +## Restarts + +The handler restarts the `.socket`, not the `.service` — that is what picks up a +changed unit; the service is started by the socket on the next connection. + +**Restarting a socket drops connections that are currently open through it.** +For bitcoin-p2p that means peers reconnect; for datum-stratum it means a mining +client has to reconnect and may lose in-flight shares. The handler only fires +when a unit file actually changes. diff --git a/ansible/roles/socket_proxy/defaults/main.yml b/ansible/roles/socket_proxy/defaults/main.yml new file mode 100644 index 0000000..6a512ea --- /dev/null +++ b/ansible/roles/socket_proxy/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# Required +socket_proxy_name: "" # "bitcoin-p2p" -> bitcoin-p2p-proxy.{socket,service} +socket_proxy_description: "" # "Bitcoin P2P" -> "Bitcoin P2P Proxy Socket" +socket_proxy_listen_port: 0 # public port on the edge host +socket_proxy_upstream_host: "" # Tailscale hostname, e.g. "knots-box" + +# Optional +socket_proxy_upstream_port: "" # defaults to socket_proxy_listen_port +socket_proxy_documentation: "" # Documentation= in both units +socket_proxy_free_bind: false # FreeBind=true: bind before the address is up +socket_proxy_timeout_stop_sec: "" # TimeoutStopSec= +socket_proxy_log_to_journal: false # StandardOutput/StandardError=journal + +# Firewall +socket_proxy_ufw_proto: tcp +socket_proxy_ufw_comment: "" # defaults to " public access" diff --git a/ansible/roles/socket_proxy/handlers/main.yml b/ansible/roles/socket_proxy/handlers/main.yml new file mode 100644 index 0000000..7fdf421 --- /dev/null +++ b/ansible/roles/socket_proxy/handlers/main.yml @@ -0,0 +1,8 @@ +--- +# Restarting the .socket is what picks up a changed unit; the .service is +# started by the socket on the next connection. +- name: Restart socket proxy + ansible.builtin.systemd: + name: "{{ socket_proxy_name }}-proxy.socket" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/socket_proxy/tasks/main.yml b/ansible/roles/socket_proxy/tasks/main.yml new file mode 100644 index 0000000..4315555 --- /dev/null +++ b/ansible/roles/socket_proxy/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Assert socket_proxy parameters are sane + ansible.builtin.assert: + that: + - socket_proxy_name | length > 0 + - socket_proxy_description | length > 0 + - socket_proxy_listen_port | int > 0 + - socket_proxy_upstream_host | length > 0 + fail_msg: >- + socket_proxy: '{{ socket_proxy_name | default("") }}' needs a name, + a description, a listen port and an upstream host. + quiet: true + +- name: "Create the {{ socket_proxy_name }}-proxy socket unit" + ansible.builtin.template: + src: proxy.socket.j2 + dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.socket" + owner: root + group: root + mode: '0644' + notify: Restart socket proxy + +- name: "Create the {{ socket_proxy_name }}-proxy service unit" + ansible.builtin.template: + src: proxy.service.j2 + dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.service" + owner: root + group: root + mode: '0644' + notify: Restart socket proxy + +- name: "Enable and start the {{ socket_proxy_name }}-proxy socket" + ansible.builtin.systemd: + name: "{{ socket_proxy_name }}-proxy.socket" + enabled: yes + state: started + daemon_reload: yes + +- name: "Allow the {{ socket_proxy_name }} port through UFW" + community.general.ufw: + rule: allow + port: "{{ socket_proxy_listen_port | string }}" + proto: "{{ socket_proxy_ufw_proto }}" + comment: "{{ socket_proxy_ufw_comment | default(socket_proxy_description ~ ' public access', true) }}" + +# Reachability of the upstream over Tailscale. Deliberately non-fatal: the proxy +# is still correctly configured if the backend happens to be down, and this is +# the one check that depends on another machine being up. +- name: "Verify {{ socket_proxy_upstream_host }} is reachable over Tailscale" + ansible.builtin.wait_for: + host: "{{ socket_proxy_upstream_host }}" + port: "{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }}" + timeout: 10 + failed_when: false diff --git a/ansible/roles/socket_proxy/templates/proxy.service.j2 b/ansible/roles/socket_proxy/templates/proxy.service.j2 new file mode 100644 index 0000000..e59d251 --- /dev/null +++ b/ansible/roles/socket_proxy/templates/proxy.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description={{ socket_proxy_description }} Proxy to {{ socket_proxy_upstream_host }} +{% if socket_proxy_documentation %} +Documentation={{ socket_proxy_documentation }} +{% endif %} +Requires={{ socket_proxy_name }}-proxy.socket +After=network.target + +[Service] +Type=notify +ExecStart=/lib/systemd/systemd-socket-proxyd {{ socket_proxy_upstream_host }}:{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }} +{% if socket_proxy_timeout_stop_sec %} +TimeoutStopSec={{ socket_proxy_timeout_stop_sec }} +{% endif %} +{% if socket_proxy_log_to_journal %} +StandardOutput=journal +StandardError=journal +{% endif %} diff --git a/ansible/roles/socket_proxy/templates/proxy.socket.j2 b/ansible/roles/socket_proxy/templates/proxy.socket.j2 new file mode 100644 index 0000000..0dc1721 --- /dev/null +++ b/ansible/roles/socket_proxy/templates/proxy.socket.j2 @@ -0,0 +1,14 @@ +[Unit] +Description={{ socket_proxy_description }} Proxy Socket +{% if socket_proxy_documentation %} +Documentation={{ socket_proxy_documentation }} +{% endif %} + +[Socket] +ListenStream={{ socket_proxy_listen_port }} +{% if socket_proxy_free_bind %} +FreeBind=true +{% endif %} + +[Install] +WantedBy=sockets.target diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index fcc02b4..4e9faa3 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -753,62 +753,21 @@ uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Create Bitcoin P2P proxy socket unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.socket - content: | - [Unit] - Description=Bitcoin P2P Proxy Socket - - [Socket] - ListenStream={{ bitcoin_p2p_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoin-p2p-proxy socket - - - name: Create Bitcoin P2P proxy service unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.service - content: | - [Unit] - Description=Bitcoin P2P Proxy to {{ bitcoin_tailscale_hostname }} - Requires=bitcoin-p2p-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ bitcoin_tailscale_hostname }}:{{ bitcoin_p2p_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin P2P proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - enabled: yes - state: started - - - name: Allow Bitcoin P2P port through UFW - ufw: - rule: allow - port: "{{ bitcoin_p2p_port | string }}" - proto: tcp - comment: "Bitcoin P2P public access" - - - name: Verify connectivity to knots-box via Tailscale - wait_for: - host: "{{ bitcoin_tailscale_hostname }}" - port: "{{ bitcoin_p2p_port }}" - timeout: 10 - ignore_errors: yes + - name: Expose Bitcoin P2P through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: bitcoin-p2p + socket_proxy_description: "Bitcoin P2P" + socket_proxy_listen_port: "{{ bitcoin_p2p_port }}" + socket_proxy_upstream_host: "{{ bitcoin_tailscale_hostname }}" + # These four were added by hand on vipy and were NOT in this playbook; + # writing the unit without them would have dropped FreeBind, which lets + # the socket bind before the address is up. + socket_proxy_documentation: "https://github.com/bitcoin/bitcoin" + socket_proxy_free_bind: true + socket_proxy_timeout_stop_sec: 5 + socket_proxy_log_to_journal: true - name: Display public endpoint when: uptime_kuma_enabled | default(false) @@ -931,8 +890,3 @@ - /tmp/setup_bitcoin_p2p_tcp_monitor.py - /tmp/ansible_bitcoin_p2p_config.yml - handlers: - - name: Restart bitcoin-p2p-proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - state: restarted diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 860b765..1ddc75c 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -663,62 +663,17 @@ uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Create Stratum proxy socket unit - copy: - dest: /etc/systemd/system/datum-stratum-proxy.socket - content: | - [Unit] - Description=DATUM Stratum Proxy Socket - - [Socket] - ListenStream={{ datum_gateway_stratum_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: "0644" - notify: Restart datum-stratum-proxy socket - - - name: Create Stratum proxy service unit - copy: - dest: /etc/systemd/system/datum-stratum-proxy.service - content: | - [Unit] - Description=DATUM Stratum Proxy to {{ datum_tailscale_hostname }} - Requires=datum-stratum-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ datum_tailscale_hostname }}:{{ datum_gateway_stratum_port }} - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Stratum proxy socket - systemd: - name: datum-stratum-proxy.socket - enabled: yes - state: started - - - name: Allow Stratum port through UFW - ufw: - rule: allow - port: "{{ datum_gateway_stratum_port | string }}" - proto: tcp - comment: "DATUM Gateway Stratum public access" - - - name: Verify connectivity to knots-box Stratum via Tailscale - wait_for: - host: "{{ datum_tailscale_hostname }}" - port: "{{ datum_gateway_stratum_port }}" - timeout: 10 - ignore_errors: yes + - name: Expose the DATUM Stratum port through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: datum-stratum + socket_proxy_description: "DATUM Stratum" + socket_proxy_listen_port: "{{ datum_gateway_stratum_port }}" + socket_proxy_upstream_host: "{{ datum_tailscale_hostname }}" + # Matches the UFW comment already on vipy; the derived default would + # have said "DATUM Stratum" and rewritten the rule. + socket_proxy_ufw_comment: "DATUM Gateway Stratum public access" - name: Display public Stratum endpoint when: uptime_kuma_enabled | default(false) @@ -845,8 +800,3 @@ - /tmp/setup_datum_stratum_tcp_monitor.py - /tmp/ansible_datum_stratum_config.yml - handlers: - - name: Restart datum-stratum-proxy socket - systemd: - name: datum-stratum-proxy.socket - state: restarted diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index a1548a7..6ffe254 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -552,62 +552,14 @@ uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Create Fulcrum SSL proxy socket unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.socket - content: | - [Unit] - Description=Fulcrum SSL Proxy Socket - - [Socket] - ListenStream={{ fulcrum_ssl_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum-ssl-proxy socket - - - name: Create Fulcrum SSL proxy service unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.service - content: | - [Unit] - Description=Fulcrum SSL Proxy to {{ fulcrum_tailscale_hostname }} - Requires=fulcrum-ssl-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ fulcrum_tailscale_hostname }}:{{ fulcrum_ssl_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum SSL proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - enabled: yes - state: started - - - name: Allow Fulcrum SSL port through UFW - ufw: - rule: allow - port: "{{ fulcrum_ssl_port | string }}" - proto: tcp - comment: "Fulcrum SSL public access" - - - name: Verify connectivity to fulcrum-box via Tailscale - wait_for: - host: "{{ fulcrum_tailscale_hostname }}" - port: "{{ fulcrum_ssl_port }}" - timeout: 10 - ignore_errors: yes + - name: Expose Fulcrum SSL through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: fulcrum-ssl + socket_proxy_description: "Fulcrum SSL" + socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" - name: Display public endpoint when: uptime_kuma_enabled | default(false) @@ -730,9 +682,4 @@ - /tmp/setup_fulcrum_ssl_tcp_monitor.py - /tmp/ansible_fulcrum_ssl_config.yml - handlers: - - name: Restart fulcrum-ssl-proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - state: restarted From cea2523e15c656cc04b960c8eb9b6a324a18115d Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 11 Sep 2026 23:53:09 +0200 Subject: [PATCH 40/44] caddy: close out Plan 4 All five close-out greps return nothing: no sites-enabled handling outside roles/, no `systemctl reload caddy`, no caddy_sites_dir self-reference, no inline proxy unit writes. 37 playbooks syntax clean. The 14 Caddy site files and 6 proxy units on the hosts are byte-identical to the Stage 0 baseline. Seven play names still said "on vipy" while the play targeted a group. Renamed to "on the edge host" - the last place a play claimed a hostname after Plan 2. Documented the four vhosts in /etc/caddy/sites-enabled that no playbook writes (uptime-kuma, arbretstaging, bitcoininfra, scriberr) in the caddy_site README. None deleted. uptime-kuma.conf was going to be deleted as dead config. It is not dead: the louislam/uptime-kuma container is STILL RUNNING on watchtower - created 2026-02-07, restart=unless-stopped, healthy - and uptime.contrapeso.xyz returns 302, not the 502 a dead backend would give. The "decommissioning" retired the Ansible code and the vault credentials, not the service. PLAN_3 claimed "the tokens died with the server"; that is corrected there. The Caddyfile.* backups are kept: one per host, Nov-Dec 2025, not churning, and the only record of each Caddyfile before the import line was added. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/caddy_site/README.md | 24 +++++++++++++++++++ .../deploy_bitcoin_knots_playbook.yml | 2 +- .../deploy_datum_gateway_playbook.yml | 4 ++-- .../fulcrum/deploy_fulcrum_playbook.yml | 2 +- .../services/memos/deploy_memos_playbook.yml | 2 +- .../mempool/deploy_mempool_playbook.yml | 2 +- .../phoenixd/deploy_phoenixd_playbook.yml | 2 +- 7 files changed, 31 insertions(+), 7 deletions(-) diff --git a/ansible/roles/caddy_site/README.md b/ansible/roles/caddy_site/README.md index 6891070..c8e32af 100644 --- a/ansible/roles/caddy_site/README.md +++ b/ansible/roles/caddy_site/README.md @@ -92,3 +92,27 @@ upstream hostname` in every case. `datum-gateway` previously said `# Resolve via Tailscale MagicDNS`. Migrating it therefore rewrites one comment line, which Caddy ignores. Every other site renders byte-identical to what its playbook produced. + +## Sites on the hosts that this role does NOT manage + +Four vhosts exist in `/etc/caddy/sites-enabled/` that no playbook writes. They +were made by hand. The role only ever writes the one file it is told to, so it +leaves them alone — but nothing in the repo records them, and that is why they +are listed here. Checked 2026-09-11: + +| File | Host | Serves | State | +|---|---|---|---| +| `uptime-kuma.conf` | watchtower | `localhost:3001` | **HTTP 302 — still live**, see below | +| `arbretstaging.conf` | vipy | `arbret-staging-box:80` via MagicDNS | HTTP 200 | +| `bitcoininfra.conf` | vipy | static `file_server` from `/var/www/bitcoin-services-home` | HTTP 200 | +| `scriberr.conf` | vipy | `scriberr-box:8080` via MagicDNS | HTTP 502 — upstream down | + +**`uptime-kuma.conf` must not be deleted as dead config.** Uptime Kuma was +"decommissioned" in the repo — its playbooks archived and its credentials pulled +from the vault — but the container is **still running** on watchtower +(`louislam/uptime-kuma:latest`, created 2026-02-07, `restart=unless-stopped`) +and is still reachable at its public subdomain. Only the Ansible code was +retired; the service was not. See `archive/uptime_kuma/`. + +`scriberr` returning 502 is the one that looks like genuine rot: it proxies to a +`scriberr-box` that is not answering, and `scriberr-box` is not in the inventory. diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index 4e9faa3..1ae5823 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -740,7 +740,7 @@ state: restarted -- name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd +- name: Setup public Bitcoin P2P forwarding on the edge host hosts: edge become: yes vars_files: diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index 1ddc75c..0c53e9d 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -487,7 +487,7 @@ # =========================================== # Caddy Reverse Proxy for DATUM Dashboard (on vipy) # =========================================== -- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on vipy +- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on the edge host hosts: edge become: yes vars_files: @@ -650,7 +650,7 @@ # Miners connect to vipy:23334; traffic is forwarded to knots-box:23334 # over the Tailscale network, matching the Bitcoin P2P proxy pattern. # =========================================== -- name: Setup public Stratum port forwarding on vipy via systemd-socket-proxyd +- name: Setup public Stratum port forwarding on the edge host hosts: edge become: yes vars_files: diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 6ffe254..01e9f17 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -540,7 +540,7 @@ state: restarted -- name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd +- name: Setup public Fulcrum SSL forwarding on the edge host hosts: edge become: yes vars_files: diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index 83c187e..04e99cf 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -156,7 +156,7 @@ state: restarted -- name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) +- name: Configure Caddy reverse proxy for Memos on the edge host (via Tailscale) hosts: edge become: yes vars_files: diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index ba3e9e9..e9cab0e 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -603,7 +603,7 @@ - /tmp/ansible_mempool_config.yml - /tmp/mempool_push_urls.yml -- name: Configure Caddy reverse proxy for Mempool on vipy +- name: Configure Caddy reverse proxy for Mempool on the edge host hosts: edge become: yes vars_files: diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index f20ecdc..8b30a1f 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -19,7 +19,7 @@ # ⚠️ After the first run, back up {{ phoenixd_data_dir }}/seed.dat. Losing it # means losing the funds. See setup_backup_phoenixd_to_lapy.yml. -- name: Deploy phoenixd on vipy +- name: Deploy phoenixd on the edge host hosts: edge become: yes vars_files: From a8a2815c5f9099cf2d8a37baf5f273920dd315fd Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 12 Sep 2026 15:14:47 +0200 Subject: [PATCH 41/44] inventory --- ansible/inventory.ini | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/ansible/inventory.ini b/ansible/inventory.ini index 4643dbb..b9b2906 100644 --- a/ansible/inventory.ini +++ b/ansible/inventory.ini @@ -2,19 +2,20 @@ vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + [nodito_host] nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - +# Requires the tailnet to be up on the control node. [nodito_vms] -knots_box_local ansible_host=192.168.1.135 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -fulcrum_box_local ansible_host=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -mempool_box_local ansible_host=192.168.1.140 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -memos_box_local ansible_host=192.168.1.130 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -forgejo_runner_local ansible_host=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -arbret_staging_local ansible_host=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -small_backups_local ansible_host=192.168.1.148 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -nonkeiwaisi_local ansible_host=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +knots_box_local ansible_host=knots-box lan_ip=192.168.1.135 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +fulcrum_box_local ansible_host=fulcrum-box lan_ip=192.168.1.140 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +mempool_box_local ansible_host=mempool-box lan_ip=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +nonkeiwaisi_local ansible_host=nonkeiwaisi-box lan_ip=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua # Local connection to laptop: this assumes you're running ansible commands from your personal laptop [lapy] @@ -65,3 +66,6 @@ nodito_vms edge monitoring vpn_control + +[backup_store] +small_backups_local From 01b83a80ec80b44289a0637fa4b1f1aaaf9b43d8 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 12 Sep 2026 15:32:17 +0200 Subject: [PATCH 42/44] ip tricks --- ansible/group_vars/nodito_vms.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ansible/group_vars/nodito_vms.yml diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml new file mode 100644 index 0000000..2ff6994 --- /dev/null +++ b/ansible/group_vars/nodito_vms.yml @@ -0,0 +1,17 @@ +--- +# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. +# +# ansible_host is a MagicDNS name. If tailscaled is not running on the control +# node that name does not resolve, so `nc %h %p` fails fast and the second nc +# takes over on the LAN address recorded as lan_ip in inventory.ini. +# +# This is safe against the LAN addresses drifting again (which is how +# fulcrum/mempool came to be transposed): known_hosts is keyed to the MagicDNS +# NAME, so if lan_ip ever points at a different machine the host key will not +# match and ssh aborts. Verified 2026-09-12 by pointing fulcrum-box at +# mempool-box's address: "Host key verification failed." +# +# lan_ip is a convenience, not an identity. If it goes stale the fallback stops +# working; it will never connect you to the wrong box. +ansible_ssh_common_args: >- + -o ProxyCommand="sh -c 'nc -w2 %h %p 2>/dev/null || nc -w4 {{ lan_ip }} %p'" From 27e036eccd1d5fa88f57ad22ed521cb1fe46275e Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 12 Sep 2026 16:02:00 +0200 Subject: [PATCH 43/44] backup stuff --- ansible/playbooks/backups.yml | 16 ++++ ansible/roles/backup_source/README.md | 76 ++++++++++++++++ ansible/roles/backup_source/defaults/main.yml | 27 ++++++ ansible/roles/backup_source/handlers/main.yml | 4 + ansible/roles/backup_source/tasks/main.yml | 89 +++++++++++++++++++ .../backup_source/templates/backup.service.j2 | 14 +++ .../backup_source/templates/backup.sh.j2 | 69 ++++++++++++++ .../backup_source/templates/backup.timer.j2 | 11 +++ ansible/roles/backup_store/README.md | 55 ++++++++++++ ansible/roles/backup_store/defaults/main.yml | 11 +++ ansible/roles/backup_store/handlers/main.yml | 5 ++ ansible/roles/backup_store/tasks/main.yml | 53 +++++++++++ .../templates/pull-backups.service.j2 | 10 +++ .../backup_store/templates/pull-backups.sh.j2 | 43 +++++++++ .../templates/pull-backups.timer.j2 | 9 ++ .../headscale/setup_backup_headscale.yml | 21 +++++ 16 files changed, 513 insertions(+) create mode 100644 ansible/playbooks/backups.yml create mode 100644 ansible/roles/backup_source/README.md create mode 100644 ansible/roles/backup_source/defaults/main.yml create mode 100644 ansible/roles/backup_source/handlers/main.yml create mode 100644 ansible/roles/backup_source/tasks/main.yml create mode 100644 ansible/roles/backup_source/templates/backup.service.j2 create mode 100644 ansible/roles/backup_source/templates/backup.sh.j2 create mode 100644 ansible/roles/backup_source/templates/backup.timer.j2 create mode 100644 ansible/roles/backup_store/README.md create mode 100644 ansible/roles/backup_store/defaults/main.yml create mode 100644 ansible/roles/backup_store/handlers/main.yml create mode 100644 ansible/roles/backup_store/tasks/main.yml create mode 100644 ansible/roles/backup_store/templates/pull-backups.service.j2 create mode 100644 ansible/roles/backup_store/templates/pull-backups.sh.j2 create mode 100644 ansible/roles/backup_store/templates/pull-backups.timer.j2 create mode 100644 ansible/services/headscale/setup_backup_headscale.yml diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml new file mode 100644 index 0000000..842942e --- /dev/null +++ b/ansible/playbooks/backups.yml @@ -0,0 +1,16 @@ +- name: Configure the offsite backup pull + hosts: backup_store + gather_facts: yes + + tasks: + - name: Ensure the box pulls every source on a timer + ansible.builtin.include_role: + name: backup_store + vars: + backup_store_sources: + - name: arbret + source: "arbret@prd-arbret:/opt/arbret/backups/" + retention_days: 90 + - name: headscale + source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" + retention_days: 90 diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md new file mode 100644 index 0000000..065e1fc --- /dev/null +++ b/ansible/roles/backup_source/README.md @@ -0,0 +1,76 @@ +# `backup_source` + +Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a +local directory, prune, on a systemd timer. `small-backups-box` pulls the +directory later (see `backup_store`). + +Modelled on `prd-arbret`, which has been doing exactly this correctly since +before the rest of the estate was migrated. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 +``` + +Produces `/opt/backups/headscale/headscale_.tar.gz.age`, +`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`. + +## Why the source encrypts, not the destination + +`age -r ` is asymmetric and the host holds only the **public** key, so +a compromised host cannot read its own backups — or anyone else's. The scripts +this replaces encrypted with GPG *on the laptop, after the data had already +crossed the network*, which protects the artefact at rest but not in transit. + +The matching identity lives only on lapy and is escrowed. **Lose it and every +artefact everywhere becomes noise**, including arbret's. + +## `backup_source_dump_command` writes to STDOUT + +The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` +with relative paths in `tar` rather than absolute ones: it avoids tar's "removing +leading /" and makes the restore target explicit. + +## The trap is the reason this role exists + +When `backup_source_stop_service` is set, the script stops the unit and installs +an EXIT trap that starts it again. Without it, a failed dump leaves the service +down until the next timer fires — **every hand-written script this replaced had +that bug**, and it was only ever masked because their `systemctl stop` failed +first, before anything was stopped. + +Verified on spacey: with the dump forced to fail, the log shows +`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit +failed rather than hiding it), and headscale is `active` afterwards. + +If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed — +which is correct, because nothing was stopped. + +## `.partial` + +The dump writes `.partial` and only `mv`s it into place on success, so +a truncated file is never mistaken for a backup. A failure inside the pipeline +does leave one behind, and the prune glob cannot match it (it ends `.partial`, +not `.tar.gz.age`), so the script clears stale partials at the **start** of each +run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run. + +## `backup_source_stop_service` may be a bare name + +`headscale` and `headscale.service` both work. The unit template normalises it, +because systemd rejects a bare name in `After=` with +`Failed to add dependency ... Invalid argument` — which it logs and then ignores, +so the unit appears to work while carrying no ordering at all. + +## Retention is two-tier + +`backup_source_retention_days` is **local** and short — these hosts are +disk-constrained. The long tail lives on `small-backups-box`, which decides its +own retention per source. Losing the local copy is expected and fine. diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml new file mode 100644 index 0000000..77d17e0 --- /dev/null +++ b/ansible/roles/backup_source/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# Required +backup_source_name: "" # "headscale" -> headscale_.tar.gz.age +backup_source_description: "" # "Headscale" +backup_source_dump_command: "" # must write the payload to STDOUT + +# Placement +backup_source_dir: "/opt/backups/{{ backup_source_name }}" +backup_source_artifact_suffix: "tar.gz.age" + +# Encryption. Asymmetric: the host holds only the public key and cannot decrypt +# what it produces. +backup_source_recipient: "{{ age_backup_recipient }}" + +# The unprivileged account small-backups-box pulls as. It owns the dump +# directory and nothing else; it deliberately has no sudo. +backup_source_pull_user: backup-pull +backup_source_pull_key: "{{ backup_pull_public_key }}" + +# Safety +backup_source_stop_service: "" # local unit stopped for the dump, restored by a trap + +# Retention here is LOCAL and short; small-backups-box keeps the long tail. +backup_source_retention_days: 7 + +# Schedule. The box pulls at 04:00, so dumps must land before that. +backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml new file mode 100644 index 0000000..37a7f2b --- /dev/null +++ b/ansible/roles/backup_source/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd for backup units + ansible.builtin.systemd: + daemon_reload: yes diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml new file mode 100644 index 0000000..54d6ac1 --- /dev/null +++ b/ansible/roles/backup_source/tasks/main.yml @@ -0,0 +1,89 @@ +--- +- name: Assert backup_source parameters are sane + ansible.builtin.assert: + that: + - backup_source_name | length > 0 + - backup_source_description | length > 0 + - backup_source_dump_command | length > 0 + - backup_source_recipient | length > 0 + - backup_source_recipient is match('^age1[0-9a-z]{58}$') + fail_msg: >- + backup_source: '{{ backup_source_name | default("") }}' needs a name, + description, dump command and a valid age recipient (age1... 62 chars). + quiet: true + +# Declared here rather than assumed. Stage 1 installed it by hand; this is what +# makes a rebuilt host get it too. +- name: Ensure age is installed + ansible.builtin.apt: + name: age + state: present + update_cache: yes + cache_valid_time: 3600 + +# The pull account: unprivileged, no sudo, exists only so small-backups-box can +# read the dump directory. Trust points one way — the box can read backups, and +# can do nothing else on this host. +- name: "Ensure the {{ backup_source_pull_user }} account exists" + ansible.builtin.user: + name: "{{ backup_source_pull_user }}" + system: yes + shell: /bin/sh # rsync-over-ssh needs a shell; nologin breaks it + home: "/var/lib/{{ backup_source_pull_user }}" + create_home: yes + password: '!' # no password login, ever + when: backup_source_pull_user | length > 0 + +- name: "Authorise the backup box's key for {{ backup_source_pull_user }}" + ansible.posix.authorized_key: + user: "{{ backup_source_pull_user }}" + key: "{{ backup_source_pull_key }}" + key_options: "restrict" # no pty, no forwarding, no user rc + exclusive: yes + state: present + when: backup_source_pull_user | length > 0 + +# The shared container above the per-service directories. It must be traversable +# or the pull account cannot reach its own directory. The script's `mkdir -p` +# runs under `umask 077` and would otherwise create this 0700. +- name: "Ensure {{ backup_source_dir | dirname }} is traversable" + ansible.builtin.file: + path: "{{ backup_source_dir | dirname }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: "Ensure {{ backup_source_dir }} exists" + ansible.builtin.file: + path: "{{ backup_source_dir }}" + state: directory + owner: root + group: "{{ backup_source_pull_user | default('root', true) }}" + mode: '0750' + +- name: "Install the {{ backup_source_name }} backup script" + ansible.builtin.template: + src: backup.sh.j2 + dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh" + owner: root + group: root + mode: '0750' + validate: "bash -n %s" + +- name: "Install the {{ backup_source_name }}-backup systemd units" + ansible.builtin.template: + src: "backup.{{ item }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + notify: Reload systemd for backup units + +- name: "Enable the {{ backup_source_name }}-backup timer" + ansible.builtin.systemd: + name: "{{ backup_source_name }}-backup.timer" + enabled: yes + state: started + daemon_reload: yes diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 new file mode 100644 index 0000000..91ad063 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description={{ backup_source_description }} backup +{% if backup_source_stop_service %} +{# systemd rejects a bare name here ("Failed to add dependency ... Invalid + argument"), so normalise to a full unit name. #} +After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }} +{% endif %} + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 new file mode 100644 index 0000000..35e4398 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source) +# +# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}. +# The host holds only the age PUBLIC key, so it cannot read its own backups. +set -euo pipefail +umask 077 + +BACKUP_DIR="{{ backup_source_dir }}" +RETENTION_DAYS={{ backup_source_retention_days }} +RECIPIENT="{{ backup_source_recipient }}" +SUFFIX="{{ backup_source_artifact_suffix }}" +NAME="{{ backup_source_name }}" +{% if backup_source_stop_service %} +SERVICE="{{ backup_source_stop_service }}" +{% endif %} + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}" + +die() { echo "FATAL: $*" >&2; exit 1; } +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } + +# --- Pre-flight --- +[[ -n "$RECIPIENT" ]] || die "no age recipient configured" +command -v age >/dev/null || die "age is not installed" + +# Mode must agree with what the role sets, or each undoes the other every run. +mkdir -p "$BACKUP_DIR" +{% if backup_source_pull_user %} +chown root:{{ backup_source_pull_user }} "$BACKUP_DIR" +chmod 750 "$BACKUP_DIR" +{% else %} +chmod 700 "$BACKUP_DIR" +{% endif %} + +# A run that died mid-dump leaves a .partial. It is not a backup, and the prune +# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them +# here or they accumulate forever. +rm -f "${BACKUP_DIR}/${NAME}_"*.partial + +{% if backup_source_stop_service %} +# --- Stop the service, and guarantee it comes back --- +# The trap is the point: without it a failed dump leaves the service down until +# the next timer fires. Every hand-written script this replaced had that bug. +log "Stopping ${SERVICE}..." +systemctl stop "$SERVICE" +trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT +{% endif %} + +# --- Dump straight into age; plaintext never touches the disk --- +log "Writing ${ARTIFACT}..." +{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +mv "${ARTIFACT}.partial" "$ARTIFACT" +{% if backup_source_pull_user %} +# Readable by the pull account and nobody else. The contents are age-encrypted +# regardless, so this is depth rather than the actual protection. +chown root:{{ backup_source_pull_user }} "$ARTIFACT" +chmod 640 "$ARTIFACT" +{% else %} +chmod 600 "$ARTIFACT" +{% endif %} +log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))" + +# --- Prune --- +log "Pruning local artefacts older than ${RETENTION_DAYS} days..." +find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete + +log "Done." diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2 new file mode 100644 index 0000000..d9a7e6a --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description={{ backup_source_description }} backup + +[Timer] +OnCalendar={{ backup_source_on_calendar }} +# Persistent: a window missed while the host was down runs on next boot. cron on +# a laptop had no equivalent, which is how two backups went unnoticed for months. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md new file mode 100644 index 0000000..13d9f5a --- /dev/null +++ b/ansible/roles/backup_store/README.md @@ -0,0 +1,55 @@ +# `backup_store` + +Pulls already-encrypted backup artefacts from every source host onto +`small-backups-box`, on a timer, and expires them per source. + +Generalises the hand-written `pull-backups.sh` that had one hardcoded source +(`arbret`). That job's behaviour is preserved exactly: same source path, same +90 days, same destination directory. + +## This host holds no key + +Everything pulled here is ciphertext produced by `backup_source` on the source +host. The box cannot read any of it — the age identity lives only on lapy. That +is deliberate: the machine holding every backup should not also be able to open +them. + +## One failing source must not stop the others + +The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own +function, failures are counted, and the script exits non-zero at the end so +systemd marks the unit failed. A dead host costs you that one source, not the +whole run. + +This is the specific failure the whole plan exists to prevent: the laptop jobs +aborted on first error and then silently produced empty directories for nine +months. + +## Trust points one way + +The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated +account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised +with `restrict`. That account can read one directory and do nothing else — no +sudo, no pty, no forwarding. A compromised backup box cannot reach into +production. + +## Addressing: names, never IPs + +Sources are addressed by name. The job this replaced hardcoded spacey's IP; the +droplet was later rebuilt, the address was recycled to a stranger, and the +backup failed silently from 2025-12-01 while the directory listing still looked +healthy. + +Two kinds of name are in play: + +- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a + headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without + it the box cannot even resolve the peer, let alone reach it. +- **spacey** is *not* a tailnet member — it is the headscale control server — so + its backup is pulled over the public internet via `headscale.contrapeso.xyz`, + which follows the host if the droplet is rebuilt. + +## Retention here is the long tail + +Sources keep a few days locally; this box keeps 90 (or whatever the source entry +says). Losing the source's local copy is expected. diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml new file mode 100644 index 0000000..c0c128b --- /dev/null +++ b/ansible/roles/backup_store/defaults/main.yml @@ -0,0 +1,11 @@ +--- +backup_store_dir: "{{ ansible_env.HOME }}/backups" +backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull" +backup_store_on_calendar: "*-*-* 04:00:00" + +# One entry per source. `retention_days` is the LONG tail; the source keeps its +# own short local retention. +# - name: headscale +# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" +# retention_days: 90 +backup_store_sources: [] diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml new file mode 100644 index 0000000..632be35 --- /dev/null +++ b/ansible/roles/backup_store/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload systemd for pull-backups + ansible.builtin.systemd: + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml new file mode 100644 index 0000000..d138830 --- /dev/null +++ b/ansible/roles/backup_store/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Assert backup_store sources are sane + ansible.builtin.assert: + that: + - backup_store_sources | length > 0 + - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length + fail_msg: "backup_store: every source needs name, source and retention_days" + quiet: true + +- name: Ensure rsync is installed + ansible.builtin.apt: + name: rsync + state: present + update_cache: yes + cache_valid_time: 3600 + become: yes + +- name: Ensure the backup store directory exists + ansible.builtin.file: + path: "{{ backup_store_dir }}" + state: directory + mode: '0700' + +- name: Install the pull-backups script + ansible.builtin.template: + src: pull-backups.sh.j2 + dest: /usr/local/bin/pull-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + +- name: Install the pull-backups systemd units + ansible.builtin.template: + src: "pull-backups.{{ item }}.j2" + dest: "/etc/systemd/system/pull-backups.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + become: yes + notify: Reload systemd for pull-backups + +- name: Enable the pull-backups timer + ansible.builtin.systemd: + name: pull-backups.timer + enabled: yes + state: started + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2 new file mode 100644 index 0000000..c65516c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Pull encrypted backups from production + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/pull-backups.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier=pull-backups diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 new file mode 100644 index 0000000..267900c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Pull encrypted backups from production — managed by Ansible (roles/backup_store) +# +# Everything here is already ciphertext: this host only moves and expires files, +# and holds no key that can read them. +set -uo pipefail # deliberately NOT -e; see the loop below + +SSH_KEY="{{ backup_store_ssh_key }}" +STORE="{{ backup_store_dir }}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } +fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); } + +failures=0 + +# One source failing must not stop the others. The whole point of this box is +# that a single dead host cannot silently take the rest of the backups with it — +# which is exactly how the laptop-based jobs failed unnoticed for nine months. +{% for src in backup_store_sources %} +# --- {{ src.name }} --- +pull_{{ src.name | replace('-', '_') }}() { + local dir="${STORE}/{{ src.name }}" + mkdir -p "$dir" + log "Pulling {{ src.name }} from {{ src.source }}..." + if rsync -az --timeout=120 \ + -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ + "{{ src.source }}" "$dir/"; then + log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" + else + fail "{{ src.name }}: rsync failed" + return 1 + fi + log " {{ src.name }}: pruning older than {{ src.retention_days }} days" + find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete +} +pull_{{ src.name | replace('-', '_') }} || true + +{% endfor %} +if [ "$failures" -gt 0 ]; then + log "FAILED: $failures source(s) did not pull" + exit 1 +fi +log "All sources pulled." diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2 new file mode 100644 index 0000000..336e1ca --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Daily offsite backup pull + +[Timer] +OnCalendar={{ backup_store_on_calendar }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml new file mode 100644 index 0000000..9c72d1d --- /dev/null +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -0,0 +1,21 @@ +--- +- name: Configure the Headscale backup on the vpn_control host + hosts: vpn_control + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./headscale_vars.yml + + tasks: + - name: Ensure Headscale dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + # -C / with relative paths: avoids tar's "removing leading /" and makes + # the restore target explicit. + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" From 394f2519ff261f518473f6df171c736d51a164a2 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 12 Sep 2026 16:20:42 +0200 Subject: [PATCH 44/44] age backups everywhere --- ansible/group_vars/all/main.yml | 7 +++ ansible/group_vars/nodito_vms.yml | 23 ++++++--- ansible/playbooks/backups.yml | 12 +++++ ansible/roles/backup_source/README.md | 50 +++++++++++++++++++ ansible/roles/backup_source/defaults/main.yml | 8 ++- ansible/roles/backup_source/tasks/main.yml | 2 + .../backup_source/templates/backup.sh.j2 | 12 +++-- .../services/forgejo/setup_backup_forgejo.yml | 25 ++++++++++ .../services/lnbits/setup_backup_lnbits.yml | 25 ++++++++++ ansible/services/memos/setup_backup_memos.yml | 26 ++++++++++ .../vaultwarden/setup_backup_vaultwarden.yml | 25 ++++++++++ 11 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 ansible/services/forgejo/setup_backup_forgejo.yml create mode 100644 ansible/services/lnbits/setup_backup_lnbits.yml create mode 100644 ansible/services/memos/setup_backup_memos.yml create mode 100644 ansible/services/vaultwarden/setup_backup_vaultwarden.yml diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 36d35f8..0be0162 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -7,3 +7,10 @@ root_domain: contrapeso.xyz # playbooks are kept deliberately — the check logic is meant to be rewired to # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. uptime_kuma_enabled: false + +# age recipient for all backup artefacts +age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" + +# Public key small-backups-box pulls with +# Authorised on each source host for an unprivileged, dedicated user only +backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml index 2ff6994..1072b51 100644 --- a/ansible/group_vars/nodito_vms.yml +++ b/ansible/group_vars/nodito_vms.yml @@ -2,16 +2,23 @@ # Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. # # ansible_host is a MagicDNS name. If tailscaled is not running on the control -# node that name does not resolve, so `nc %h %p` fails fast and the second nc -# takes over on the LAN address recorded as lan_ip in inventory.ini. +# node that name does not resolve, the probe fails, and the LAN address recorded +# as lan_ip in inventory.ini takes over. # -# This is safe against the LAN addresses drifting again (which is how -# fulcrum/mempool came to be transposed): known_hosts is keyed to the MagicDNS -# NAME, so if lan_ip ever points at a different machine the host key will not -# match and ssh aborts. Verified 2026-09-12 by pointing fulcrum-box at -# mempool-box's address: "Host key verification failed." +# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`: +# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a +# single `nc -w5` silently tears down the SSH session after five quiet seconds. +# That produced intermittent "Data could not be sent to remote host" failures on +# exactly the long, quiet operations (apt) where a dropped connection costs most. +# `nc -z` probes, then `exec nc` carries the session with no timeout at all. +# +# Safe against the LAN addresses drifting again (which is how fulcrum/mempool +# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if +# lan_ip ever points at a different machine the host key will not match and ssh +# aborts. Verified by pointing fulcrum-box at mempool-box's address: +# "Host key verification failed." # # lan_ip is a convenience, not an identity. If it goes stale the fallback stops # working; it will never connect you to the wrong box. ansible_ssh_common_args: >- - -o ProxyCommand="sh -c 'nc -w2 %h %p 2>/dev/null || nc -w4 {{ lan_ip }} %p'" + -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'" diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 842942e..0686ab6 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -14,3 +14,15 @@ - name: headscale source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" retention_days: 90 + - name: memos + source: "backup-pull@memos-box:/opt/backups/memos/" + retention_days: 90 + - name: vaultwarden + source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/" + retention_days: 90 + - name: lnbits + source: "backup-pull@prd-vipy:/opt/backups/lnbits/" + retention_days: 90 + - name: forgejo + source: "backup-pull@prd-vipy:/opt/backups/forgejo/" + retention_days: 14 diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md index 065e1fc..a7794ab 100644 --- a/ansible/roles/backup_source/README.md +++ b/ansible/roles/backup_source/README.md @@ -39,6 +39,56 @@ The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` with relative paths in `tar` rather than absolute ones: it avoids tar's "removing leading /" and makes the restore target explicit. +## Services that are not systemd + +`backup_source_stop_service` runs `systemctl stop/start`. For anything else, +give the pair explicitly — vaultwarden is a docker compose stack, so +`systemctl stop vaultwarden` silently does nothing: + +```yaml +backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" +backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" +``` + +The same EXIT trap wraps both forms. The assert refuses a stop command without a +matching start command, because that combination fails in the one way you would +not notice: the service stops and never comes back. + +## More than one thing to back up + +`tar` takes several paths, so multiple files or directories are normally **one** +artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together, +lnbits captures its data directory and its `.env`. + +Prefer one artefact. A backup should be a consistent snapshot, and two artefacts +written by two runs can drift — you can end up restoring an `.env` that does not +match the database it configures. Pulling a single file back out needs no +unpacking: + +```bash +age -d -i | tar -xzO opt/lnbits/lnbits/.env +``` + +If you genuinely need separate artefacts, call the role twice with different +`backup_source_name`s rather than extending it — but only one call may set +`backup_source_stop_service`, or the service is stopped twice per night. + +The case this shape cannot express is a **database dump plus a file tree** +(`pg_dump` and a media directory, say): you cannot merge those into one stream +without staging plaintext on disk, which is exactly what this design avoids. +None of the current services need it — all are file trees, all stopped for the +dump. A future one that does should use two role calls. + +## Everything here is sqlite, so everything stops + +All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files +present). A live copy of a WAL-mode database can be torn or stale, so each is +stopped for the duration. Measured downtime: under a second for headscale and +memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos +and database. That last one is the real cost of a consistent snapshot; if it +becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy, +not skipping the stop. + ## The trap is the reason this role exists When `backup_source_stop_service` is set, the script stops the unit and installs diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml index 77d17e0..7de6006 100644 --- a/ansible/roles/backup_source/defaults/main.yml +++ b/ansible/roles/backup_source/defaults/main.yml @@ -17,8 +17,12 @@ backup_source_recipient: "{{ age_backup_recipient }}" backup_source_pull_user: backup-pull backup_source_pull_key: "{{ backup_pull_public_key }}" -# Safety -backup_source_stop_service: "" # local unit stopped for the dump, restored by a trap +# Safety. Give either a systemd unit, or an explicit pair of commands for +# services that are not systemd-managed (vaultwarden is a docker compose stack). +# Whichever is used, a trap guarantees the restart. +backup_source_stop_service: "" # systemd unit stopped for the dump +backup_source_stop_command: "" # overrides stop_service when set +backup_source_start_command: "" # required alongside stop_command # Retention here is LOCAL and short; small-backups-box keeps the long tail. backup_source_retention_days: 7 diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml index 54d6ac1..4283b84 100644 --- a/ansible/roles/backup_source/tasks/main.yml +++ b/ansible/roles/backup_source/tasks/main.yml @@ -7,9 +7,11 @@ - backup_source_dump_command | length > 0 - backup_source_recipient | length > 0 - backup_source_recipient is match('^age1[0-9a-z]{58}$') + - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0) fail_msg: >- backup_source: '{{ backup_source_name | default("") }}' needs a name, description, dump command and a valid age recipient (age1... 62 chars). + backup_source_stop_command must be paired with backup_source_start_command. quiet: true # Declared here rather than assumed. Stage 1 installed it by hand; this is what diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 index 35e4398..18fabd6 100644 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -11,8 +11,10 @@ RETENTION_DAYS={{ backup_source_retention_days }} RECIPIENT="{{ backup_source_recipient }}" SUFFIX="{{ backup_source_artifact_suffix }}" NAME="{{ backup_source_name }}" -{% if backup_source_stop_service %} -SERVICE="{{ backup_source_stop_service }}" +{% if backup_source_stop_service or backup_source_stop_command %} +STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }} +START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }} +SERVICE="{{ backup_source_stop_service or backup_source_description }}" # label for the log only {% endif %} TIMESTAMP=$(date +%Y%m%d_%H%M%S) @@ -39,13 +41,13 @@ chmod 700 "$BACKUP_DIR" # here or they accumulate forever. rm -f "${BACKUP_DIR}/${NAME}_"*.partial -{% if backup_source_stop_service %} +{% if backup_source_stop_service or backup_source_stop_command %} # --- Stop the service, and guarantee it comes back --- # The trap is the point: without it a failed dump leaves the service down until # the next timer fires. Every hand-written script this replaced had that bug. log "Stopping ${SERVICE}..." -systemctl stop "$SERVICE" -trap 'log "Restarting ${SERVICE}..."; systemctl start "${SERVICE}" || true' EXIT +eval "$STOP_CMD" +trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT {% endif %} # --- Dump straight into age; plaintext never touches the disk --- diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml new file mode 100644 index 0000000..d6aafef --- /dev/null +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -0,0 +1,25 @@ +--- +# Forgejo backup: dumps locally on vipy, encrypted with age. +# +# The biggest artefact in the estate (~2.7 G) and the reason retention here is +# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14. +# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the +# old job did the same. +- name: Configure the Forgejo backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./forgejo_vars.yml + + tasks: + - name: Ensure Forgejo dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: forgejo + backup_source_description: "Forgejo" + backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo" + backup_source_stop_service: forgejo + backup_source_retention_days: 2 + backup_source_on_calendar: "*-*-* 02:30:00" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml new file mode 100644 index 0000000..46abf31 --- /dev/null +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -0,0 +1,25 @@ +--- +# LNBits backup: dumps locally on vipy, encrypted with age. +# The old job produced TWO gpg artefacts (data, then .env separately). They are +# folded into one tar here so the wallet database and the .env that configures +# it are always the same point in time; two artefacts written by two runs can +# drift. Pulling one file back out needs no unpacking: +# age -d -i | tar -xzO opt/lnbits/lnbits/.env +- name: Configure the LNBits backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./lnbits_vars.yml + + tasks: + - name: Ensure LNBits dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: lnbits + backup_source_description: "LNBits" + backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env" + backup_source_stop_service: lnbits + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:20:00" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml new file mode 100644 index 0000000..2131924 --- /dev/null +++ b/ansible/services/memos/setup_backup_memos.yml @@ -0,0 +1,26 @@ +--- +# Memos backup: dumps locally on memos-box, encrypted with age. +# Replaces the lapy pull, which had been writing EMPTY directories since +# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned +# to a different machine that has no rsync. +- name: Configure the Memos backup on its own host + hosts: memos + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./memos_vars.yml + + tasks: + - name: Ensure Memos dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: memos + backup_source_description: "Memos" + backup_source_dump_command: "tar -czf - -C / var/lib/memos" + # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a + # consistent database rather than a torn mid-write copy. The old rsync + # job did not stop it. + backup_source_stop_service: memos + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml new file mode 100644 index 0000000..8430475 --- /dev/null +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -0,0 +1,25 @@ +--- +# Vaultwarden backup: dumps locally on vipy, encrypted with age. +# Previously rsynced to lapy in the CLEAR; the artefact now never exists +# unencrypted, on disk or on the wire. +- name: Configure the Vaultwarden backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./vaultwarden_vars.yml + + tasks: + - name: Ensure Vaultwarden dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: vaultwarden + backup_source_description: "Vaultwarden" + backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data" + # Not systemd — a docker compose stack — so stop/start explicitly. + # sqlite in WAL mode, hence stopping at all. + backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" + backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:10:00"