diff --git a/.gitignore b/.gitignore index 6c0a084..9ca7b8e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,13 +9,12 @@ crash.log *.tfvars *.tfvars.json -test-inventory.ini -inventory.ini venv/* .env -# Secrets and sensitive files -*_secrets.yml -*_secrets.yaml -secrets/ -.secrets/ +# Secrets are ansible-vault encrypted and ARE committed. +# Anything matching *_secrets.plain.yml is a working decryption — never commit those. +*_secrets.plain.yml + +# Vault password — never commit +ansible/.vault_pass \ No newline at end of file diff --git a/01_infra_setup.md b/01_infra_setup.md index 6ef0978..a8ba9c0 100644 --- a/01_infra_setup.md +++ b/01_infra_setup.md @@ -162,6 +162,12 @@ Note that, by applying these playbooks, both the root user and the `counterweigh ```bash cp ansible/infra_secrets.yml.example ansible/infra_secrets.yml ``` + > **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server + > deployment was removed from this repo; what it monitored and how it was set up is + > preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in + > the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic + > survives for whatever replaces it. The credentials below no longer exist in the vault. + * Edit `ansible/infra_secrets.yml` and add your Uptime Kuma credentials: ```yaml uptime_kuma_username: "admin" diff --git a/02_vps_core_services_setup.md b/02_vps_core_services_setup.md index 19cd122..5d75de9 100644 --- a/02_vps_core_services_setup.md +++ b/02_vps_core_services_setup.md @@ -49,6 +49,12 @@ Checklist: ## Uptime Kuma +> **DEPRECATED (2026-09-11).** Uptime Kuma has been decommissioned. The server +> deployment was removed from this repo; what it monitored and how it was set up is +> preserved in [`archive/uptime_kuma/`](archive/uptime_kuma/). The monitoring blocks in +> the playbooks are kept but inert (`uptime_kuma_enabled: false`) so the check logic +> survives for whatever replaces it. The credentials below no longer exist in the vault. + Uptime Kuma gets used to monitor the availability of services, keep track of their uptime and notify issues. ### Deploy diff --git a/README.md b/README.md index f343cbc..89ef092 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ My repo documenting my personal infra, along with artifacts, scripts, etc. Go through the different numbered markdowns in the repo root to do the different parts. +## How to edit secrets + +`ansible-vault edit ansible/your_file_with_secrets.yml` + +Assumes that you've set `ansible/.vault_pass` with `chmod 600`. + ## Overview ### Services @@ -16,7 +22,7 @@ Go through the different numbered markdowns in the repo root to do the different + Plan install + File based config + Crossbackup to Desky via rsync -* Uptime Kuma +* ~~Uptime Kuma~~ — decommissioned 2026-09-11, see `archive/uptime_kuma/` + Deployed on Vipy + Crossbackup to Desky via rsync * Vaultwarden diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..aa10181 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,14 @@ +[defaults] +inventory = inventory.ini +roles_path = roles +collections_path = collections +interpreter_python = auto_silent +stdout_callback = yaml +retry_files_enabled = False +host_key_checking = True +forks = 10 +vault_password_file = .vault_pass + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=300s diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml new file mode 100644 index 0000000..0be0162 --- /dev/null +++ b/ansible/group_vars/all/main.yml @@ -0,0 +1,16 @@ +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: contrapeso.xyz + +# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the +# playbooks are kept deliberately — the check logic is meant to be rewired to +# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. +uptime_kuma_enabled: false + +# age recipient for all backup artefacts +age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" + +# Public key small-backups-box pulls with +# Authorised on each source host for an unprivileged, dedicated user only +backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml new file mode 100644 index 0000000..d1f18ec --- /dev/null +++ b/ansible/group_vars/all/vault.yml @@ -0,0 +1,60 @@ +$ANSIBLE_VAULT;1.1;AES256 +61356165613635386631393135656434646436303665313031346566323336313138353433316463 +3363323534613064643132663335623238366431393062340a346538396662306537663163623366 +38626166383933616331623231373137306562623637313263333237633661663436666266616433 +3862346438643638650a306634333535653633613534646630386131333236366538333765323333 +38656163303837303732663561373232393132343331663164656262393730326434373731333636 +36613538646431396536363936336562616431656665653965373864633366663836353434626434 +34383932373461333564303439623565383661646365386665393831383463663662356536356236 +36373964666236626465366161636135393734356536633466383262326537343833636630343738 +62353066316131613737373162643363653662656261363465386364323962656537373061373032 +33343763353464383438363438343965653532393831343930393562633630383932653862623637 +38386239353237356631646436356166373961333464396639383538383662326534343339313330 +65373339356364636634616532363832386631323062363530313861336238353261353334306235 +62616338396431316537346638656365356564346666366366343638356261623664393263323937 +30343261363562383332323462336435376664386134646562643836363834313237373631353731 +35653535663864643266313332356635363262363533663232656531373130336539633066376139 +32316665393831663035623962656364363831333563366135636164346335383738363336663566 +63356532643563393939383635386462663561386434323939303431653438653131363538383034 +33623933333464363032656636643033326162626163353633343062633966343332383138363963 +36383831663562616533316436366566323061386535343538393861383462333166343562316633 +30623665623035393537393965626363323132656433313339396233356666346634316332616336 +37363635363330323230373332326565343530653335383437373230366563366237633665626331 +66623336626230663361636439316337393865383035326136653264666438666566646132353036 +38616264313833316536623238633339373466613866626366383835656638623863323838653030 +63303938376164653966356435386333363731656666313234663535666165646233313137343563 +36386437393139656438333262383437656666343831313239323961373637653163643664356565 +62346431393133656530316262303763646165643336396661666431383436323562336137653031 +62626538373839613734396366653065306534636630346338316237616161613037616364356431 +38643965376133616161336633383664326230383435363334353137303162663738313331346238 +39356161616533616134356231323530306338333162343363353531303263636632613036386638 +63633136386232306234323936303563646466313935326631396565383432386130656638616266 +32383334363237336539396665336366643764633131643663643137376438323666326435626461 +64346633636431393137633537306431646564386565303933636434386462346630626537346438 +32623333666133303061646564366366326665363163396262633164323631636337346130303239 +38373936663337356134666132303165393365663763396362623434633737373538653566646134 +62316164396438303532616266313062326666633130656338653139376634306664633031333037 +31636166306565353334633435656233336233363664306264626237623366336161303134353433 +63383739666462623336386537346662633666626466393039653439346436653937633537396436 +35393339383066326630353066623132333034656539363561346462626265363263303535343961 +39316461616630326539613731303039613736393633373338646266323938326162373831346336 +30656130343463366534323030646238313465306266383034623065636665623366333063383736 +38373063393837306462303564643962373334343139626338623935336435643730646532633630 +66363730386636633639346463363365343239373265303738353732653633653437636130636664 +39386365353334653765343335303263363461313965383664326563333734626533376436626530 +63353163323637303730353564383733653365613635353764333266393532653663326533646132 +39313763373735323835626437306435373238653432393936643165663663656665316132653330 +35383038353532656434366336346235363563636264303734633138323963396562306232646236 +38366536306561653937336333373434336164663336613839353439356435333833396363636437 +38323934613735643363656233333037336465336564313966623063376566663030303230323262 +65643564666534326234306164343365383632333061316238623565353538313538396364313337 +35336230313339643736653238386231343661623337306236383665356632366236356335323530 +66373831616239376231636361333430343433303233393066323865663434643433303832373262 +36366534646161323130393931626362626139663139643263366639656531613436313533363130 +65643736363833333939613566663339623964333262643863333030623138633464386238613934 +31323639333234336264313663376465323737353766643839303665313737336534386665363034 +30326439666433373232306136306365343764643434306561343339353132346430646436343362 +32333765616262363930353435616563333736313533653339656231316230346166363335356638 +37333561366339613136306438306130343230663732333862663838396463623661303961336433 +65646332373139303462303633346432366530643130366133363937653739653036366136373434 +343864373734666431326239373866313734 diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml new file mode 100644 index 0000000..1072b51 --- /dev/null +++ b/ansible/group_vars/nodito_vms.yml @@ -0,0 +1,24 @@ +--- +# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. +# +# ansible_host is a MagicDNS name. If tailscaled is not running on the control +# node that name does not resolve, the probe fails, and the LAN address recorded +# as lan_ip in inventory.ini takes over. +# +# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`: +# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a +# single `nc -w5` silently tears down the SSH session after five quiet seconds. +# That produced intermittent "Data could not be sent to remote host" failures on +# exactly the long, quiet operations (apt) where a dropped connection costs most. +# `nc -z` probes, then `exec nc` carries the session with no timeout at all. +# +# Safe against the LAN addresses drifting again (which is how fulcrum/mempool +# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if +# lan_ip ever points at a different machine the host key will not match and ssh +# aborts. Verified by pointing fulcrum-box at mempool-box's address: +# "Host key verification failed." +# +# lan_ip is a convenience, not an identity. If it goes stale the fallback stops +# working; it will never connect you to the wrong box. +ansible_ssh_common_args: >- + -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'" diff --git a/ansible/host_vars/nodito/main.yml b/ansible/host_vars/nodito/main.yml new file mode 100644 index 0000000..c0002f3 --- /dev/null +++ b/ansible/host_vars/nodito/main.yml @@ -0,0 +1,28 @@ +# Nodito CPU Temperature Monitoring Configuration + +# Temperature Monitoring Configuration +temp_threshold_celsius: 80 +temp_check_interval_minutes: 1 + +# Script Configuration +monitoring_script_dir: /opt/nodito-monitoring +monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" +log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" + +# System Configuration +systemd_service_name: nodito-cpu-temp-monitor + +# ZFS Pool Configuration +zfs_pool_name: "proxmox-tank-1" +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror +zfs_pool_mountpoint: "/var/lib/vz" + +# UPS Configuration (CyberPower CP900EPFCLCD via USB) +ups_name: cyberpower +ups_desc: "CyberPower CP900EPFCLCD" +ups_driver: usbhid-ups +ups_port: auto +ups_user: counterweight +ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power +ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index 13e5149..c6eed18 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,5 +1,5 @@ - name: Secure Debian - hosts: all + hosts: managed vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index e83cbcb..9f37c70 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,5 +1,5 @@ - name: Secure Debian - hosts: all + hosts: managed vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/410_disk_usage_alerts.yml b/ansible/infra/410_disk_usage_alerts.yml index de02f53..dcd9bdc 100644 --- a/ansible/infra/410_disk_usage_alerts.yml +++ b/ansible/infra/410_disk_usage_alerts.yml @@ -1,5 +1,18 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Deploy Disk Usage Monitoring - hosts: all + hosts: managed become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/420_system_healthcheck.yml b/ansible/infra/420_system_healthcheck.yml index 2580ff0..05532a7 100644 --- a/ansible/infra/420_system_healthcheck.yml +++ b/ansible/infra/420_system_healthcheck.yml @@ -1,5 +1,18 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Deploy System Healthcheck Monitoring - hosts: all + hosts: managed become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/430_cpu_temp_alerts.yml b/ansible/infra/430_cpu_temp_alerts.yml index 3b87102..5c9e855 100644 --- a/ansible/infra/430_cpu_temp_alerts.yml +++ b/ansible/infra/430_cpu_temp_alerts.yml @@ -1,5 +1,18 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Deploy CPU Temperature Monitoring - hosts: nodito_host + hosts: hypervisor become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/900_install_rsync.yml b/ansible/infra/900_install_rsync.yml index c0b7318..6c6b10e 100644 --- a/ansible/infra/900_install_rsync.yml +++ b/ansible/infra/900_install_rsync.yml @@ -1,5 +1,5 @@ - name: Install rsync - hosts: all + hosts: managed vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/910_docker_playbook.yml b/ansible/infra/910_docker_playbook.yml index f137b6a..62b7147 100644 --- a/ansible/infra/910_docker_playbook.yml +++ b/ansible/infra/910_docker_playbook.yml @@ -1,5 +1,5 @@ - name: Install Docker and Docker Compose on Debian 12 - hosts: all + hosts: managed become: yes tasks: diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 8d06d44..5b5e4ce 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -1,5 +1,5 @@ - name: Join machine to headscale mesh network - hosts: all + hosts: managed become: yes vars_files: - ../infra_vars.yml diff --git a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml index 02c6679..86f693f 100644 --- a/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml +++ b/ansible/infra/nodito/30_proxmox_bootstrap_playbook.yml @@ -1,5 +1,5 @@ - name: Bootstrap Nodito SSH Key Access - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../infra_vars.yml diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml index b0be2ef..378a674 100644 --- a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -1,5 +1,5 @@ - name: Switch Proxmox VE from Enterprise to Community Repositories - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../infra_vars.yml diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index cb72328..8cdde6a 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -1,5 +1,5 @@ - name: Setup ZFS RAID 1 Pool for Proxmox Storage - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../infra_vars.yml @@ -171,8 +171,21 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup ZFS Pool Health Monitoring and Monthly Scrubs - hosts: nodito + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index e8f8332..3b687ee 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -1,5 +1,5 @@ - name: Create Proxmox template from Debian cloud image (no VM clone) - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index 02468d5..f39fb56 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -1,5 +1,5 @@ - name: Setup NUT (Network UPS Tools) for CyberPower UPS - hosts: nodito_host + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml @@ -250,8 +250,21 @@ - nut-monitor +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup UPS Heartbeat Monitoring with Uptime Kuma - hosts: nodito + hosts: hypervisor become: true vars_files: - ../../infra_vars.yml diff --git a/ansible/infra/nodito/nodito_secrets.yml b/ansible/infra/nodito/nodito_secrets.yml new file mode 100644 index 0000000..c34b1e8 --- /dev/null +++ b/ansible/infra/nodito/nodito_secrets.yml @@ -0,0 +1,12 @@ +$ANSIBLE_VAULT;1.1;AES256 +39386134336137333139343861353965353665626161303634323563386563356364373265616163 +3237623438636161323366313363356335613566303338630a663732316663333136653737663935 +34373466316538663962636134303530323837316563336136653636643430363566376363323666 +3531613636393331310a306633613030343138656237663061653438336562323135633732356563 +33396136656564353465636361613161326465303166373131343562363338303966666362643532 +33383561366539393930633766363533313363653733393263376634316362643863376635366638 +36396333633231616662323465323932656565396138313264613832346261616231333265636439 +61666334613662373839613833613663333436373365376534643662656335316536303739616437 +65333635613330353536353162646234323266316338643435653864666364313734386665303830 +34383961633734646134373866623330663038366130306265656466653562643764346162313333 +353664363335303433393330306332666336 diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml new file mode 100644 index 0000000..4803d86 --- /dev/null +++ b/ansible/infra_secrets.yml @@ -0,0 +1,60 @@ +$ANSIBLE_VAULT;1.1;AES256 +34616531626537376463326332306434383163393761363536633133363161373631376437653234 +3831656536313531343433376264643261396634633965370a303931336236323766313065636535 +31343338616364386132623232373266663665353563666535383232666262663062323864303932 +3439373734316639650a633732333064633335383833663261326464336538636666323063646331 +66653464636534663935616461636132613162303530663237376661343338396133323431623561 +30303563303465633231306339326561313533333536376433393130376634303932643339656336 +30616130343733626162366561393939393766623138616537393032376665396139333561623830 +33623130336235633034666561383165616439316334323664623661343438663733373833306335 +30396165313731636435636263353133326135346236323030653734353831626238666663656364 +62386164363361373464383730333733626333333736306535336635613634646535383237326332 +37636532663933663739303634333235343137333363316430643263326231613635643633636437 +30336462633233396430633939616661643261666136656363363461343230653738613166346434 +63376363666462346539343063343562623962303130616535306439653134633630353963373337 +64313239353162363862643432666438383330636638333464323731643163643236313535313030 +65656231383738356537326661663163376634613031396436646633376139313237343534653065 +64323532346133313265346334353530393633396339316330366536646565353836396662303866 +65373030393463306664623235626437613965303837313365373632643935656630303936373837 +64663539623764656636613263346638376665626262663430333231633735653563643864343835 +62303637383332653366383066326536316336306539623230353066343739316430356437313365 +33323031613130643832636133636263303565623330306135333762363036633737343933623437 +33633436363062643436376632336235666330656265316333323361316430656631383734343238 +65616366666536363866343361666563643336306532666332643230656264303565353032393932 +36613738326239376161383064646634623639356439663965323237303361343838373737636235 +39656465333063633136643463356564636537326339653033633063366265636138656466646264 +32643961343863656464353439346531633036613138353333353631626665396239326465303632 +39336664333038636435366333623336353963323535316436646136313739373537376537353735 +66333265633232346536313933333836643439656666636236626266323333383935383565616139 +38626133356364613562393664643563346539366364636332613162663337356232333735393431 +30343635393462356431646336636663333735343164616261343836363366616162613563613066 +66333034613234626431356536323832393466363135653231336238356334353638613838626430 +63636630313533363563376161663637353362383431663130646666646433353864373736613662 +62343666633862643839666532656132653939663437373164393536653739376265303130353636 +62616538326330333136626238306133363634326365366637363961333635356133613564373131 +64626139656633663065353862343135393233333231386166346532656534623361623061336434 +37616565626134333863376532393861376132323434656433613731613834633963386635353336 +39653263346637303836663763653264396139383363616461393333333362616332663863363834 +61613832326466646133626337663364333566326633313664613236346135653332373035313034 +62356530313065393737393634313166313039616635363365633031336434396463326462316236 +62353838636533303664663132353838636632353733316639643964666139383539333761663531 +37366530373165653063303032383461326535613336343635626538373635356266396666346133 +61383434323565306438356634646338666232363238393932353365363461376130323363363236 +34643862653937353461383265663933646264373365623465313666643662633334366437646434 +34343332613832366235663563353435323738356339383338333361383561336366376238393237 +38656265663164376666616366623062393530366261623361383365376235643035353838313235 +32396430376139313430316439333539343266393965353030636638656661316137313437663832 +64366661323865383262323036613230316233666566386133363633303931323464663238323861 +62333132346165353830653062353763306232366563323430383933653163373562326536393362 +66353931376262383931613931303034356537363137366235643430323465623162363031373034 +64653563666533363062333831336463376530306639616631303266386136666233653638653732 +31313537393838333730303235363661326333646531653935633935373362303732653565656438 +36626330623063373832333739643666396135653838643164376264393863646433353036643731 +34623636623366353466333237373265323936643134613230326565303663306431373634356334 +37613732393262626135306632623336613364323632316636623037656538313236666562633438 +37306361333130376538366232393637646263656239303431623963626265386634313735373162 +64393733303132383464386330626135336237303563326238633437346164333738646431333730 +62616533363638643261306565373835303832313539656564386132393863373039653938616261 +36343031356361336563316230343433633033353130373933313361326164633765316433616162 +62356338393634653330623933313433343630386264396337373532376335316362333164363963 +363663666365336564663232363462613162 diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index d539282..c95234d 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -1,8 +1,6 @@ # Uptime Kuma login credentials # Used by the disk monitoring playbook to create monitors automatically -uptime_kuma_username: "admin" -uptime_kuma_password: "your_password_here" # ntfy credentials # Used for notification channel setup in Uptime Kuma diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml index 952df93..36d35f8 100644 --- a/ansible/infra_vars.yml +++ b/ansible/infra_vars.yml @@ -2,3 +2,8 @@ new_user: counterweight ssh_port: 22 allow_ssh_from: "any" root_domain: contrapeso.xyz + +# Uptime Kuma was decommissioned on 2026-09-11. The monitoring blocks in the +# playbooks are kept deliberately — the check logic is meant to be rewired to +# whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. +uptime_kuma_enabled: false diff --git a/ansible/inventory.ini b/ansible/inventory.ini new file mode 100644 index 0000000..b9b2906 --- /dev/null +++ b/ansible/inventory.ini @@ -0,0 +1,71 @@ +[vps] +vipy ansible_host=167.172.107.33 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +watchtower ansible_host=164.92.239.72 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +spacey ansible_host=64.227.112.128 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +[nodito_host] +nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +# Requires the tailnet to be up on the control node. +[nodito_vms] +knots_box_local ansible_host=knots-box lan_ip=192.168.1.135 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +fulcrum_box_local ansible_host=fulcrum-box lan_ip=192.168.1.140 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +mempool_box_local ansible_host=mempool-box lan_ip=192.168.1.142 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +nonkeiwaisi_local ansible_host=nonkeiwaisi-box lan_ip=192.168.1.151 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +# Local connection to laptop: this assumes you're running ansible commands from your personal laptop +[lapy] +localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C + +[arbret] +prd-arbret ansible_host=167.99.242.62 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +[edge] +vipy + +[monitoring] +watchtower + +[vpn_control] +spacey + +[hypervisor] +nodito + +[bitcoin] +knots_box_local + +[electrum] +fulcrum_box_local + +[mempool] +mempool_box_local + +[memos] +memos_box_local + +[ci_runner] +forgejo_runner_local + +[control] +localhost + +# Every machine Ansible may configure as a server. +# Deliberately EXCLUDES [control] (your laptop) and [arbret]. +[managed:children] +vps +nodito_host +nodito_vms + +# Hosts that run Caddy and therefore have /etc/caddy/sites-enabled. +[caddy:children] +edge +monitoring +vpn_control + +[backup_store] +small_backups_local diff --git a/ansible/inventory.ini.example b/ansible/inventory.ini.example deleted file mode 100644 index bde96dd..0000000 --- a/ansible/inventory.ini.example +++ /dev/null @@ -1,16 +0,0 @@ -[vps] -vipy ansible_host=your.services.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key -watchtower ansible_host=your.monitoring.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key -spacey ansible_host=your.headscale.vps.ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key - -[nodito_host] -nodito ansible_host=your.proxmox.ip.here ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key ansible_ssh_pass=your_root_password - -[nodito_vms] -# Example node, replace with your VM names and addresses -# memos_box ansible_host=192.168.1.150 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/your-key - -# Local connection to laptop: this assumes you're running ansible commands from your personal laptop -# Make sure to adjust the username -[lapy] -localhost ansible_connection=local ansible_user=your laptop user gpg_recipient=your_email@example.com gpg_key_id=your_gpg_key_id_here \ No newline at end of file diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml new file mode 100644 index 0000000..0686ab6 --- /dev/null +++ b/ansible/playbooks/backups.yml @@ -0,0 +1,28 @@ +- name: Configure the offsite backup pull + hosts: backup_store + gather_facts: yes + + tasks: + - name: Ensure the box pulls every source on a timer + ansible.builtin.include_role: + name: backup_store + vars: + backup_store_sources: + - name: arbret + source: "arbret@prd-arbret:/opt/arbret/backups/" + retention_days: 90 + - name: headscale + source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" + retention_days: 90 + - name: memos + source: "backup-pull@memos-box:/opt/backups/memos/" + retention_days: 90 + - name: vaultwarden + source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/" + retention_days: 90 + - name: lnbits + source: "backup-pull@prd-vipy:/opt/backups/lnbits/" + retention_days: 90 + - name: forgejo + source: "backup-pull@prd-vipy:/opt/backups/forgejo/" + retention_days: 14 diff --git a/ansible/requirements.yml b/ansible/requirements.yml index dd7eef4..16ef1fd 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,11 +1,10 @@ ---- # Ansible Galaxy Collections Requirements # Install with: ansible-galaxy collection install -r requirements.yml -collections: - # Uptime Kuma Ansible Collection - # Used by: infra/41_disk_usage_alerts.yml - # Provides modules to manage Uptime Kuma monitors programmatically - - name: lucasheld.uptime_kuma - version: ">=1.0.0" +# No collections are currently required. +# +# lucasheld.uptime_kuma was pinned here but never used — every monitor was created +# by hand-rolled Python instead. Removed 2026-09-11 along with Uptime Kuma itself. +# See archive/uptime_kuma/. +collections: [] diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md new file mode 100644 index 0000000..a7794ab --- /dev/null +++ b/ansible/roles/backup_source/README.md @@ -0,0 +1,126 @@ +# `backup_source` + +Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a +local directory, prune, on a systemd timer. `small-backups-box` pulls the +directory later (see `backup_store`). + +Modelled on `prd-arbret`, which has been doing exactly this correctly since +before the rest of the estate was migrated. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 +``` + +Produces `/opt/backups/headscale/headscale_.tar.gz.age`, +`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`. + +## Why the source encrypts, not the destination + +`age -r ` is asymmetric and the host holds only the **public** key, so +a compromised host cannot read its own backups — or anyone else's. The scripts +this replaces encrypted with GPG *on the laptop, after the data had already +crossed the network*, which protects the artefact at rest but not in transit. + +The matching identity lives only on lapy and is escrowed. **Lose it and every +artefact everywhere becomes noise**, including arbret's. + +## `backup_source_dump_command` writes to STDOUT + +The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` +with relative paths in `tar` rather than absolute ones: it avoids tar's "removing +leading /" and makes the restore target explicit. + +## Services that are not systemd + +`backup_source_stop_service` runs `systemctl stop/start`. For anything else, +give the pair explicitly — vaultwarden is a docker compose stack, so +`systemctl stop vaultwarden` silently does nothing: + +```yaml +backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" +backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" +``` + +The same EXIT trap wraps both forms. The assert refuses a stop command without a +matching start command, because that combination fails in the one way you would +not notice: the service stops and never comes back. + +## More than one thing to back up + +`tar` takes several paths, so multiple files or directories are normally **one** +artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together, +lnbits captures its data directory and its `.env`. + +Prefer one artefact. A backup should be a consistent snapshot, and two artefacts +written by two runs can drift — you can end up restoring an `.env` that does not +match the database it configures. Pulling a single file back out needs no +unpacking: + +```bash +age -d -i | tar -xzO opt/lnbits/lnbits/.env +``` + +If you genuinely need separate artefacts, call the role twice with different +`backup_source_name`s rather than extending it — but only one call may set +`backup_source_stop_service`, or the service is stopped twice per night. + +The case this shape cannot express is a **database dump plus a file tree** +(`pg_dump` and a media directory, say): you cannot merge those into one stream +without staging plaintext on disk, which is exactly what this design avoids. +None of the current services need it — all are file trees, all stopped for the +dump. A future one that does should use two role calls. + +## Everything here is sqlite, so everything stops + +All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files +present). A live copy of a WAL-mode database can be torn or stale, so each is +stopped for the duration. Measured downtime: under a second for headscale and +memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos +and database. That last one is the real cost of a consistent snapshot; if it +becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy, +not skipping the stop. + +## The trap is the reason this role exists + +When `backup_source_stop_service` is set, the script stops the unit and installs +an EXIT trap that starts it again. Without it, a failed dump leaves the service +down until the next timer fires — **every hand-written script this replaced had +that bug**, and it was only ever masked because their `systemctl stop` failed +first, before anything was stopped. + +Verified on spacey: with the dump forced to fail, the log shows +`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit +failed rather than hiding it), and headscale is `active` afterwards. + +If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed — +which is correct, because nothing was stopped. + +## `.partial` + +The dump writes `.partial` and only `mv`s it into place on success, so +a truncated file is never mistaken for a backup. A failure inside the pipeline +does leave one behind, and the prune glob cannot match it (it ends `.partial`, +not `.tar.gz.age`), so the script clears stale partials at the **start** of each +run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run. + +## `backup_source_stop_service` may be a bare name + +`headscale` and `headscale.service` both work. The unit template normalises it, +because systemd rejects a bare name in `After=` with +`Failed to add dependency ... Invalid argument` — which it logs and then ignores, +so the unit appears to work while carrying no ordering at all. + +## Retention is two-tier + +`backup_source_retention_days` is **local** and short — these hosts are +disk-constrained. The long tail lives on `small-backups-box`, which decides its +own retention per source. Losing the local copy is expected and fine. diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml new file mode 100644 index 0000000..7de6006 --- /dev/null +++ b/ansible/roles/backup_source/defaults/main.yml @@ -0,0 +1,31 @@ +--- +# Required +backup_source_name: "" # "headscale" -> headscale_.tar.gz.age +backup_source_description: "" # "Headscale" +backup_source_dump_command: "" # must write the payload to STDOUT + +# Placement +backup_source_dir: "/opt/backups/{{ backup_source_name }}" +backup_source_artifact_suffix: "tar.gz.age" + +# Encryption. Asymmetric: the host holds only the public key and cannot decrypt +# what it produces. +backup_source_recipient: "{{ age_backup_recipient }}" + +# The unprivileged account small-backups-box pulls as. It owns the dump +# directory and nothing else; it deliberately has no sudo. +backup_source_pull_user: backup-pull +backup_source_pull_key: "{{ backup_pull_public_key }}" + +# Safety. Give either a systemd unit, or an explicit pair of commands for +# services that are not systemd-managed (vaultwarden is a docker compose stack). +# Whichever is used, a trap guarantees the restart. +backup_source_stop_service: "" # systemd unit stopped for the dump +backup_source_stop_command: "" # overrides stop_service when set +backup_source_start_command: "" # required alongside stop_command + +# Retention here is LOCAL and short; small-backups-box keeps the long tail. +backup_source_retention_days: 7 + +# Schedule. The box pulls at 04:00, so dumps must land before that. +backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml new file mode 100644 index 0000000..37a7f2b --- /dev/null +++ b/ansible/roles/backup_source/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd for backup units + ansible.builtin.systemd: + daemon_reload: yes diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml new file mode 100644 index 0000000..4283b84 --- /dev/null +++ b/ansible/roles/backup_source/tasks/main.yml @@ -0,0 +1,91 @@ +--- +- name: Assert backup_source parameters are sane + ansible.builtin.assert: + that: + - backup_source_name | length > 0 + - backup_source_description | length > 0 + - backup_source_dump_command | length > 0 + - backup_source_recipient | length > 0 + - backup_source_recipient is match('^age1[0-9a-z]{58}$') + - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0) + fail_msg: >- + backup_source: '{{ backup_source_name | default("") }}' needs a name, + description, dump command and a valid age recipient (age1... 62 chars). + backup_source_stop_command must be paired with backup_source_start_command. + quiet: true + +# Declared here rather than assumed. Stage 1 installed it by hand; this is what +# makes a rebuilt host get it too. +- name: Ensure age is installed + ansible.builtin.apt: + name: age + state: present + update_cache: yes + cache_valid_time: 3600 + +# The pull account: unprivileged, no sudo, exists only so small-backups-box can +# read the dump directory. Trust points one way — the box can read backups, and +# can do nothing else on this host. +- name: "Ensure the {{ backup_source_pull_user }} account exists" + ansible.builtin.user: + name: "{{ backup_source_pull_user }}" + system: yes + shell: /bin/sh # rsync-over-ssh needs a shell; nologin breaks it + home: "/var/lib/{{ backup_source_pull_user }}" + create_home: yes + password: '!' # no password login, ever + when: backup_source_pull_user | length > 0 + +- name: "Authorise the backup box's key for {{ backup_source_pull_user }}" + ansible.posix.authorized_key: + user: "{{ backup_source_pull_user }}" + key: "{{ backup_source_pull_key }}" + key_options: "restrict" # no pty, no forwarding, no user rc + exclusive: yes + state: present + when: backup_source_pull_user | length > 0 + +# The shared container above the per-service directories. It must be traversable +# or the pull account cannot reach its own directory. The script's `mkdir -p` +# runs under `umask 077` and would otherwise create this 0700. +- name: "Ensure {{ backup_source_dir | dirname }} is traversable" + ansible.builtin.file: + path: "{{ backup_source_dir | dirname }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: "Ensure {{ backup_source_dir }} exists" + ansible.builtin.file: + path: "{{ backup_source_dir }}" + state: directory + owner: root + group: "{{ backup_source_pull_user | default('root', true) }}" + mode: '0750' + +- name: "Install the {{ backup_source_name }} backup script" + ansible.builtin.template: + src: backup.sh.j2 + dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh" + owner: root + group: root + mode: '0750' + validate: "bash -n %s" + +- name: "Install the {{ backup_source_name }}-backup systemd units" + ansible.builtin.template: + src: "backup.{{ item }}.j2" + dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + notify: Reload systemd for backup units + +- name: "Enable the {{ backup_source_name }}-backup timer" + ansible.builtin.systemd: + name: "{{ backup_source_name }}-backup.timer" + enabled: yes + state: started + daemon_reload: yes diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 new file mode 100644 index 0000000..91ad063 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description={{ backup_source_description }} backup +{% if backup_source_stop_service %} +{# systemd rejects a bare name here ("Failed to add dependency ... Invalid + argument"), so normalise to a full unit name. #} +After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }} +{% endif %} + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 new file mode 100644 index 0000000..18fabd6 --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.sh.j2 @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source) +# +# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}. +# The host holds only the age PUBLIC key, so it cannot read its own backups. +set -euo pipefail +umask 077 + +BACKUP_DIR="{{ backup_source_dir }}" +RETENTION_DAYS={{ backup_source_retention_days }} +RECIPIENT="{{ backup_source_recipient }}" +SUFFIX="{{ backup_source_artifact_suffix }}" +NAME="{{ backup_source_name }}" +{% if backup_source_stop_service or backup_source_stop_command %} +STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }} +START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }} +SERVICE="{{ backup_source_stop_service or backup_source_description }}" # label for the log only +{% endif %} + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}" + +die() { echo "FATAL: $*" >&2; exit 1; } +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } + +# --- Pre-flight --- +[[ -n "$RECIPIENT" ]] || die "no age recipient configured" +command -v age >/dev/null || die "age is not installed" + +# Mode must agree with what the role sets, or each undoes the other every run. +mkdir -p "$BACKUP_DIR" +{% if backup_source_pull_user %} +chown root:{{ backup_source_pull_user }} "$BACKUP_DIR" +chmod 750 "$BACKUP_DIR" +{% else %} +chmod 700 "$BACKUP_DIR" +{% endif %} + +# A run that died mid-dump leaves a .partial. It is not a backup, and the prune +# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them +# here or they accumulate forever. +rm -f "${BACKUP_DIR}/${NAME}_"*.partial + +{% if backup_source_stop_service or backup_source_stop_command %} +# --- Stop the service, and guarantee it comes back --- +# The trap is the point: without it a failed dump leaves the service down until +# the next timer fires. Every hand-written script this replaced had that bug. +log "Stopping ${SERVICE}..." +eval "$STOP_CMD" +trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT +{% endif %} + +# --- Dump straight into age; plaintext never touches the disk --- +log "Writing ${ARTIFACT}..." +{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +mv "${ARTIFACT}.partial" "$ARTIFACT" +{% if backup_source_pull_user %} +# Readable by the pull account and nobody else. The contents are age-encrypted +# regardless, so this is depth rather than the actual protection. +chown root:{{ backup_source_pull_user }} "$ARTIFACT" +chmod 640 "$ARTIFACT" +{% else %} +chmod 600 "$ARTIFACT" +{% endif %} +log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))" + +# --- Prune --- +log "Pruning local artefacts older than ${RETENTION_DAYS} days..." +find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete + +log "Done." diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2 new file mode 100644 index 0000000..d9a7e6a --- /dev/null +++ b/ansible/roles/backup_source/templates/backup.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description={{ backup_source_description }} backup + +[Timer] +OnCalendar={{ backup_source_on_calendar }} +# Persistent: a window missed while the host was down runs on next boot. cron on +# a laptop had no equivalent, which is how two backups went unnoticed for months. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md new file mode 100644 index 0000000..13d9f5a --- /dev/null +++ b/ansible/roles/backup_store/README.md @@ -0,0 +1,55 @@ +# `backup_store` + +Pulls already-encrypted backup artefacts from every source host onto +`small-backups-box`, on a timer, and expires them per source. + +Generalises the hand-written `pull-backups.sh` that had one hardcoded source +(`arbret`). That job's behaviour is preserved exactly: same source path, same +90 days, same destination directory. + +## This host holds no key + +Everything pulled here is ciphertext produced by `backup_source` on the source +host. The box cannot read any of it — the age identity lives only on lapy. That +is deliberate: the machine holding every backup should not also be able to open +them. + +## One failing source must not stop the others + +The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own +function, failures are counted, and the script exits non-zero at the end so +systemd marks the unit failed. A dead host costs you that one source, not the +whole run. + +This is the specific failure the whole plan exists to prevent: the laptop jobs +aborted on first error and then silently produced empty directories for nine +months. + +## Trust points one way + +The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated +account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised +with `restrict`. That account can read one directory and do nothing else — no +sudo, no pty, no forwarding. A compromised backup box cannot reach into +production. + +## Addressing: names, never IPs + +Sources are addressed by name. The job this replaced hardcoded spacey's IP; the +droplet was later rebuilt, the address was recycled to a stranger, and the +backup failed silently from 2025-12-01 while the directory listing still looked +healthy. + +Two kinds of name are in play: + +- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a + headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without + it the box cannot even resolve the peer, let alone reach it. +- **spacey** is *not* a tailnet member — it is the headscale control server — so + its backup is pulled over the public internet via `headscale.contrapeso.xyz`, + which follows the host if the droplet is rebuilt. + +## Retention here is the long tail + +Sources keep a few days locally; this box keeps 90 (or whatever the source entry +says). Losing the source's local copy is expected. diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml new file mode 100644 index 0000000..c0c128b --- /dev/null +++ b/ansible/roles/backup_store/defaults/main.yml @@ -0,0 +1,11 @@ +--- +backup_store_dir: "{{ ansible_env.HOME }}/backups" +backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull" +backup_store_on_calendar: "*-*-* 04:00:00" + +# One entry per source. `retention_days` is the LONG tail; the source keeps its +# own short local retention. +# - name: headscale +# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" +# retention_days: 90 +backup_store_sources: [] diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml new file mode 100644 index 0000000..632be35 --- /dev/null +++ b/ansible/roles/backup_store/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload systemd for pull-backups + ansible.builtin.systemd: + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml new file mode 100644 index 0000000..d138830 --- /dev/null +++ b/ansible/roles/backup_store/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Assert backup_store sources are sane + ansible.builtin.assert: + that: + - backup_store_sources | length > 0 + - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length + - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length + fail_msg: "backup_store: every source needs name, source and retention_days" + quiet: true + +- name: Ensure rsync is installed + ansible.builtin.apt: + name: rsync + state: present + update_cache: yes + cache_valid_time: 3600 + become: yes + +- name: Ensure the backup store directory exists + ansible.builtin.file: + path: "{{ backup_store_dir }}" + state: directory + mode: '0700' + +- name: Install the pull-backups script + ansible.builtin.template: + src: pull-backups.sh.j2 + dest: /usr/local/bin/pull-backups.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + become: yes + +- name: Install the pull-backups systemd units + ansible.builtin.template: + src: "pull-backups.{{ item }}.j2" + dest: "/etc/systemd/system/pull-backups.{{ item }}" + owner: root + group: root + mode: '0644' + loop: [service, timer] + become: yes + notify: Reload systemd for pull-backups + +- name: Enable the pull-backups timer + ansible.builtin.systemd: + name: pull-backups.timer + enabled: yes + state: started + daemon_reload: yes + become: yes diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2 new file mode 100644 index 0000000..c65516c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Pull encrypted backups from production + +[Service] +Type=oneshot +User={{ ansible_user_id }} +ExecStart=/usr/local/bin/pull-backups.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier=pull-backups diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 new file mode 100644 index 0000000..267900c --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.sh.j2 @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Pull encrypted backups from production — managed by Ansible (roles/backup_store) +# +# Everything here is already ciphertext: this host only moves and expires files, +# and holds no key that can read them. +set -uo pipefail # deliberately NOT -e; see the loop below + +SSH_KEY="{{ backup_store_ssh_key }}" +STORE="{{ backup_store_dir }}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } +fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); } + +failures=0 + +# One source failing must not stop the others. The whole point of this box is +# that a single dead host cannot silently take the rest of the backups with it — +# which is exactly how the laptop-based jobs failed unnoticed for nine months. +{% for src in backup_store_sources %} +# --- {{ src.name }} --- +pull_{{ src.name | replace('-', '_') }}() { + local dir="${STORE}/{{ src.name }}" + mkdir -p "$dir" + log "Pulling {{ src.name }} from {{ src.source }}..." + if rsync -az --timeout=120 \ + -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ + "{{ src.source }}" "$dir/"; then + log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" + else + fail "{{ src.name }}: rsync failed" + return 1 + fi + log " {{ src.name }}: pruning older than {{ src.retention_days }} days" + find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete +} +pull_{{ src.name | replace('-', '_') }} || true + +{% endfor %} +if [ "$failures" -gt 0 ]; then + log "FAILED: $failures source(s) did not pull" + exit 1 +fi +log "All sources pulled." diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2 new file mode 100644 index 0000000..336e1ca --- /dev/null +++ b/ansible/roles/backup_store/templates/pull-backups.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Daily offsite backup pull + +[Timer] +OnCalendar={{ backup_store_on_calendar }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/caddy_site/README.md b/ansible/roles/caddy_site/README.md new file mode 100644 index 0000000..c8e32af --- /dev/null +++ b/ansible/roles/caddy_site/README.md @@ -0,0 +1,118 @@ +# `caddy_site` + +Writes one Caddy site file into `{{ caddy_sites_dir }}`, makes sure the main +Caddyfile imports that directory, validates the result, and reloads Caddy once. + +Replaces the four-task block that was copy-pasted into 10 playbooks. + +Runs on any host in the `[caddy]` group — `edge` (vipy), `monitoring` +(watchtower) and `vpn_control` (spacey). + +## Usage + +```yaml +- ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: forgejo # -> forgejo.conf + caddy_site_domain: "{{ forgejo_domain }}" + caddy_site_upstream: "localhost:{{ forgejo_port }}" +``` + +Use `include_role`, not a `roles:` block, so the call stays in task order next +to the tasks it depends on. Variables passed this way are scoped to the include +and do not leak into later calls — so **every call must pass everything it +needs**; nothing carries over. + +## Shapes + +Pick exactly one of `caddy_site_upstream`, `caddy_site_root`, `caddy_site_body`. + +| Want | Set | +|---|---| +| `reverse_proxy host:port` | `caddy_site_upstream` | +| static `root *` + `file_server` | `caddy_site_root` | +| anything else | `caddy_site_body` (raw, indented 4 for you) | + +`caddy_site_upstream` accepts two modifiers, which add a block to the +`reverse_proxy`: + +- `caddy_site_headers_up: {"X-Forwarded-Host": "..."}` +- `caddy_site_resolvers: "100.100.100.100"` — Tailscale MagicDNS + +and `caddy_site_basic_auth` wraps the site in a `basic_auth` block. + +## `caddy_site_basic_auth` is a LIST, not a dict + +```yaml +caddy_site_basic_auth: + - user: "{{ datum_dashboard_username }}" + hash: "{{ datum_dashboard_password_hash }}" +``` + +**Ansible does not template dictionary keys.** With `{ "{{ user }}": "hash" }` +the value is rendered and the key is not, so the literal string +`{{ datum_dashboard_username }}` lands in the config file. Found while building +this role; the `assert` refuses a mapping so it cannot happen again. + +## Secrets and `--diff` + +Rendered site files can carry credentials — `datum-gateway.conf` holds a bcrypt +hash — and `--diff` prints rendered content. The template task therefore sets +`diff: "{{ caddy_site_reveal | bool }}"`, default `false`, so `--diff` runs are +safe everywhere. Pass `-e caddy_site_reveal=true` to see what moved on a site +you know is not secret. + +## Validation + +`validate: "caddy validate --adapter caddyfile --config %s"` runs against the +rendered temp file before it is moved into place. Verified on vipy that a single +site fragment validates cleanly (rc=0, `Valid configuration`) and that a +malformed one is rejected (rc=1, with the syntax error and line number). A +failed validate leaves the live file untouched, so a broken config can no longer +reach a running Caddy. + +What it cannot catch is a conflict with the global `/etc/caddy/Caddyfile`. + +## The reload is a handler + +`Reload caddy` fires **once, at the end of the play**, however many sites +notified it. The code this replaced ran `command: systemctl reload caddy` +immediately, mid-play. If a later task in the same play needs the new config to +be live, flush first: + +```yaml +- ansible.builtin.meta: flush_handlers +``` + +## Known intentional difference + +The `resolvers` block is commented `# Use Tailscale MagicDNS to resolve the +upstream hostname` in every case. `datum-gateway` previously said `# Resolve via +Tailscale MagicDNS`. Migrating it therefore rewrites one comment line, which +Caddy ignores. Every other site renders byte-identical to what its playbook +produced. + +## Sites on the hosts that this role does NOT manage + +Four vhosts exist in `/etc/caddy/sites-enabled/` that no playbook writes. They +were made by hand. The role only ever writes the one file it is told to, so it +leaves them alone — but nothing in the repo records them, and that is why they +are listed here. Checked 2026-09-11: + +| File | Host | Serves | State | +|---|---|---|---| +| `uptime-kuma.conf` | watchtower | `localhost:3001` | **HTTP 302 — still live**, see below | +| `arbretstaging.conf` | vipy | `arbret-staging-box:80` via MagicDNS | HTTP 200 | +| `bitcoininfra.conf` | vipy | static `file_server` from `/var/www/bitcoin-services-home` | HTTP 200 | +| `scriberr.conf` | vipy | `scriberr-box:8080` via MagicDNS | HTTP 502 — upstream down | + +**`uptime-kuma.conf` must not be deleted as dead config.** Uptime Kuma was +"decommissioned" in the repo — its playbooks archived and its credentials pulled +from the vault — but the container is **still running** on watchtower +(`louislam/uptime-kuma:latest`, created 2026-02-07, `restart=unless-stopped`) +and is still reachable at its public subdomain. Only the Ansible code was +retired; the service was not. See `archive/uptime_kuma/`. + +`scriberr` returning 502 is the one that looks like genuine rot: it proxies to a +`scriberr-box` that is not answering, and `scriberr-box` is not in the inventory. diff --git a/ansible/roles/caddy_site/defaults/main.yml b/ansible/roles/caddy_site/defaults/main.yml new file mode 100644 index 0000000..cafa3dc --- /dev/null +++ b/ansible/roles/caddy_site/defaults/main.yml @@ -0,0 +1,23 @@ +--- +# Required +caddy_site_name: "" # file basename -> .conf +caddy_site_domain: "" # site address line; may hold several, comma separated + +# Pick exactly one shape +caddy_site_upstream: "" # "localhost:3000" -> reverse_proxy +caddy_site_root: "" # filesystem path -> root * + file_server +caddy_site_body: "" # raw escape hatch for one-off sites; wins over both + +# reverse_proxy modifiers +caddy_site_resolvers: "" # "100.100.100.100" for Tailscale MagicDNS +caddy_site_headers_up: {} # {"X-Forwarded-Host": "wallet.example.com"} +# A LIST, not a dict: Ansible does not template dict *keys*, so a Jinja +# expression for the username silently passes through as literal text. +caddy_site_basic_auth: [] # [{user: "{{ x_user }}", hash: "{{ x_hash }}"}] + +# Placement. caddy_sites_dir comes from services_config.yml; this is the fallback. +caddy_sites_dir: /etc/caddy/sites-enabled + +# Rendered site files can carry credentials (basic_auth hashes), so --diff is +# suppressed by default. Pass -e caddy_site_reveal=true to see what moved. +caddy_site_reveal: false diff --git a/ansible/roles/caddy_site/handlers/main.yml b/ansible/roles/caddy_site/handlers/main.yml new file mode 100644 index 0000000..4bfa7ff --- /dev/null +++ b/ansible/roles/caddy_site/handlers/main.yml @@ -0,0 +1,12 @@ +--- +# Fires once at the end of the play, however many sites notified it. +# Anything later in the same play that needs the new config live must be +# preceded by `- ansible.builtin.meta: flush_handlers`. +# become is explicit because handlers do not inherit it from the task that +# notified them. headscale's play runs become: no and elevates per task, so +# without this the reload would run unprivileged and fail. +- name: Reload caddy + become: true + ansible.builtin.systemd: + name: caddy + state: reloaded diff --git a/ansible/roles/caddy_site/tasks/main.yml b/ansible/roles/caddy_site/tasks/main.yml new file mode 100644 index 0000000..5ad3976 --- /dev/null +++ b/ansible/roles/caddy_site/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: Assert caddy_site parameters are sane + ansible.builtin.assert: + that: + - caddy_site_name | length > 0 + - caddy_site_domain | length > 0 + - (caddy_site_upstream | length > 0) or (caddy_site_root | length > 0) or (caddy_site_body | length > 0) + - caddy_site_basic_auth is not mapping + fail_msg: >- + caddy_site: '{{ caddy_site_name | default("") }}' needs a name, a domain and + one of caddy_site_upstream / caddy_site_root / caddy_site_body. + caddy_site_basic_auth must be a LIST of {user, hash} — Ansible does not template dict keys. + quiet: true + +- name: Ensure Caddy sites-enabled directory exists + ansible.builtin.file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Ensure Caddyfile imports sites-enabled + ansible.builtin.lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + create: yes + mode: '0644' + backup: yes + +# `validate` runs `caddy validate` against the rendered temp file before it is +# moved into place: verified on vipy that a single site fragment validates +# cleanly (rc=0, "Valid configuration") and that a malformed one is rejected +# (rc=1). A failed validate leaves the live file untouched. +- name: "Write Caddy site '{{ caddy_site_name }}'" + ansible.builtin.template: + src: site.conf.j2 + dest: "{{ caddy_sites_dir }}/{{ caddy_site_name }}.conf" + owner: root + group: root + mode: '0644' + validate: "caddy validate --adapter caddyfile --config %s" + diff: "{{ caddy_site_reveal | bool }}" + notify: Reload caddy diff --git a/ansible/roles/caddy_site/templates/site.conf.j2 b/ansible/roles/caddy_site/templates/site.conf.j2 new file mode 100644 index 0000000..5d34c01 --- /dev/null +++ b/ansible/roles/caddy_site/templates/site.conf.j2 @@ -0,0 +1,34 @@ +{{ caddy_site_domain }} { +{% if caddy_site_body %} +{{ caddy_site_body | trim | indent(4, first=True) }} +{% else %} +{% if caddy_site_basic_auth %} + basic_auth { +{% for cred in caddy_site_basic_auth %} + {{ cred.user }} {{ cred.hash }} +{% endfor %} + } +{% endif %} +{% if caddy_site_root %} + root * {{ caddy_site_root }} + file_server +{% endif %} +{% if caddy_site_upstream %} +{% if caddy_site_headers_up or caddy_site_resolvers %} + reverse_proxy {{ caddy_site_upstream }} { +{% for key, value in caddy_site_headers_up.items() %} + header_up {{ key }} {{ value }} +{% endfor %} +{% if caddy_site_resolvers %} + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers {{ caddy_site_resolvers }} + } +{% endif %} + } +{% else %} + reverse_proxy {{ caddy_site_upstream }} +{% endif %} +{% endif %} +{% endif %} +} diff --git a/ansible/roles/socket_proxy/README.md b/ansible/roles/socket_proxy/README.md new file mode 100644 index 0000000..cebb95f --- /dev/null +++ b/ansible/roles/socket_proxy/README.md @@ -0,0 +1,62 @@ +# `socket_proxy` + +Exposes a service running on a private Tailscale host through a public TCP port +on an edge machine, using `systemd-socket-proxyd`. Writes a `.socket` and a +`.service` unit, enables the socket, opens the UFW port, and checks the upstream +is reachable. + +## Usage + +```yaml +- ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: fulcrum-ssl # -> fulcrum-ssl-proxy.{socket,service} + socket_proxy_description: "Fulcrum SSL" # -> "Fulcrum SSL Proxy Socket" + socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" +``` + +`socket_proxy_upstream_port` defaults to `socket_proxy_listen_port`, which is +what all three current callers want. + +## Optional unit settings + +These exist because the **live** `bitcoin-p2p-proxy` units on vipy carried +settings the playbook never wrote. Somebody added them by hand, so running +`deploy_bitcoin_knots_playbook.yml` would have silently removed them: + +| Variable | Emits | Why it matters | +|---|---|---| +| `socket_proxy_free_bind` | `FreeBind=true` in `[Socket]` | Lets the socket bind to an address that is not up yet. Without it the socket can fail to start on boot. | +| `socket_proxy_documentation` | `Documentation=` in both units | Cosmetic. | +| `socket_proxy_timeout_stop_sec` | `TimeoutStopSec=` | Bounds how long a stop can hang. | +| `socket_proxy_log_to_journal` | `StandardOutput=journal` + `StandardError=journal` | Cosmetic on modern systemd, which defaults to the journal anyway. | + +Only `bitcoin-p2p` passes any of them. + +## `socket_proxy_ufw_comment` + +Defaults to `" public access"`, which reproduces the live rule +comment for bitcoin-p2p and fulcrum-ssl. **datum-stratum must pass it +explicitly** — its live comment is `DATUM Gateway Stratum public access` while +the derived default would be `DATUM Stratum public access`, and UFW treats the +comment as part of the rule, so the mismatch rewrites the rule on every run. + +## The upstream check never fails the play + +`wait_for` on the upstream carries `failed_when: false`. The proxy is correctly +configured whether or not the backend happens to be up, and this is the one task +that depends on another machine. The original plays used `ignore_errors: yes`, +which prints a red "ignoring" line; `failed_when: false` is the quieter +equivalent. + +## Restarts + +The handler restarts the `.socket`, not the `.service` — that is what picks up a +changed unit; the service is started by the socket on the next connection. + +**Restarting a socket drops connections that are currently open through it.** +For bitcoin-p2p that means peers reconnect; for datum-stratum it means a mining +client has to reconnect and may lose in-flight shares. The handler only fires +when a unit file actually changes. diff --git a/ansible/roles/socket_proxy/defaults/main.yml b/ansible/roles/socket_proxy/defaults/main.yml new file mode 100644 index 0000000..6a512ea --- /dev/null +++ b/ansible/roles/socket_proxy/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# Required +socket_proxy_name: "" # "bitcoin-p2p" -> bitcoin-p2p-proxy.{socket,service} +socket_proxy_description: "" # "Bitcoin P2P" -> "Bitcoin P2P Proxy Socket" +socket_proxy_listen_port: 0 # public port on the edge host +socket_proxy_upstream_host: "" # Tailscale hostname, e.g. "knots-box" + +# Optional +socket_proxy_upstream_port: "" # defaults to socket_proxy_listen_port +socket_proxy_documentation: "" # Documentation= in both units +socket_proxy_free_bind: false # FreeBind=true: bind before the address is up +socket_proxy_timeout_stop_sec: "" # TimeoutStopSec= +socket_proxy_log_to_journal: false # StandardOutput/StandardError=journal + +# Firewall +socket_proxy_ufw_proto: tcp +socket_proxy_ufw_comment: "" # defaults to " public access" diff --git a/ansible/roles/socket_proxy/handlers/main.yml b/ansible/roles/socket_proxy/handlers/main.yml new file mode 100644 index 0000000..7fdf421 --- /dev/null +++ b/ansible/roles/socket_proxy/handlers/main.yml @@ -0,0 +1,8 @@ +--- +# Restarting the .socket is what picks up a changed unit; the .service is +# started by the socket on the next connection. +- name: Restart socket proxy + ansible.builtin.systemd: + name: "{{ socket_proxy_name }}-proxy.socket" + state: restarted + daemon_reload: yes diff --git a/ansible/roles/socket_proxy/tasks/main.yml b/ansible/roles/socket_proxy/tasks/main.yml new file mode 100644 index 0000000..4315555 --- /dev/null +++ b/ansible/roles/socket_proxy/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Assert socket_proxy parameters are sane + ansible.builtin.assert: + that: + - socket_proxy_name | length > 0 + - socket_proxy_description | length > 0 + - socket_proxy_listen_port | int > 0 + - socket_proxy_upstream_host | length > 0 + fail_msg: >- + socket_proxy: '{{ socket_proxy_name | default("") }}' needs a name, + a description, a listen port and an upstream host. + quiet: true + +- name: "Create the {{ socket_proxy_name }}-proxy socket unit" + ansible.builtin.template: + src: proxy.socket.j2 + dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.socket" + owner: root + group: root + mode: '0644' + notify: Restart socket proxy + +- name: "Create the {{ socket_proxy_name }}-proxy service unit" + ansible.builtin.template: + src: proxy.service.j2 + dest: "/etc/systemd/system/{{ socket_proxy_name }}-proxy.service" + owner: root + group: root + mode: '0644' + notify: Restart socket proxy + +- name: "Enable and start the {{ socket_proxy_name }}-proxy socket" + ansible.builtin.systemd: + name: "{{ socket_proxy_name }}-proxy.socket" + enabled: yes + state: started + daemon_reload: yes + +- name: "Allow the {{ socket_proxy_name }} port through UFW" + community.general.ufw: + rule: allow + port: "{{ socket_proxy_listen_port | string }}" + proto: "{{ socket_proxy_ufw_proto }}" + comment: "{{ socket_proxy_ufw_comment | default(socket_proxy_description ~ ' public access', true) }}" + +# Reachability of the upstream over Tailscale. Deliberately non-fatal: the proxy +# is still correctly configured if the backend happens to be down, and this is +# the one check that depends on another machine being up. +- name: "Verify {{ socket_proxy_upstream_host }} is reachable over Tailscale" + ansible.builtin.wait_for: + host: "{{ socket_proxy_upstream_host }}" + port: "{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }}" + timeout: 10 + failed_when: false diff --git a/ansible/roles/socket_proxy/templates/proxy.service.j2 b/ansible/roles/socket_proxy/templates/proxy.service.j2 new file mode 100644 index 0000000..e59d251 --- /dev/null +++ b/ansible/roles/socket_proxy/templates/proxy.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description={{ socket_proxy_description }} Proxy to {{ socket_proxy_upstream_host }} +{% if socket_proxy_documentation %} +Documentation={{ socket_proxy_documentation }} +{% endif %} +Requires={{ socket_proxy_name }}-proxy.socket +After=network.target + +[Service] +Type=notify +ExecStart=/lib/systemd/systemd-socket-proxyd {{ socket_proxy_upstream_host }}:{{ socket_proxy_upstream_port | default(socket_proxy_listen_port, true) }} +{% if socket_proxy_timeout_stop_sec %} +TimeoutStopSec={{ socket_proxy_timeout_stop_sec }} +{% endif %} +{% if socket_proxy_log_to_journal %} +StandardOutput=journal +StandardError=journal +{% endif %} diff --git a/ansible/roles/socket_proxy/templates/proxy.socket.j2 b/ansible/roles/socket_proxy/templates/proxy.socket.j2 new file mode 100644 index 0000000..0dc1721 --- /dev/null +++ b/ansible/roles/socket_proxy/templates/proxy.socket.j2 @@ -0,0 +1,14 @@ +[Unit] +Description={{ socket_proxy_description }} Proxy Socket +{% if socket_proxy_documentation %} +Documentation={{ socket_proxy_documentation }} +{% endif %} + +[Socket] +ListenStream={{ socket_proxy_listen_port }} +{% if socket_proxy_free_bind %} +FreeBind=true +{% endif %} + +[Install] +WantedBy=sockets.target diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index d818945..1ae5823 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -1,5 +1,5 @@ - name: Build and Deploy Bitcoin Knots from Source - hosts: knots_box_local + hosts: bitcoin become: yes vars_files: - ../../infra_vars.yml @@ -439,7 +439,18 @@ debug: msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Bitcoin Knots health check and push script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh content: | @@ -534,6 +545,7 @@ mode: '0644' - name: Create systemd service for Bitcoin Knots health check + when: uptime_kuma_enabled | default(false) copy: dest: /etc/systemd/system/bitcoin-knots-healthcheck.service content: | @@ -566,6 +578,7 @@ state: started - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -655,6 +668,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -667,6 +681,7 @@ mode: '0644' - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_bitcoin_knots_monitor.py delegate_to: localhost become: no @@ -707,6 +722,7 @@ when: uptime_kuma_push_url | default('') != '' - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -718,13 +734,14 @@ handlers: - name: Restart bitcoind + when: uptime_kuma_enabled | default(false) systemd: name: bitcoind state: restarted -- name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd - hosts: vipy +- name: Setup public Bitcoin P2P forwarding on the edge host + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -736,64 +753,24 @@ uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Create Bitcoin P2P proxy socket unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.socket - content: | - [Unit] - Description=Bitcoin P2P Proxy Socket - - [Socket] - ListenStream={{ bitcoin_p2p_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoin-p2p-proxy socket - - - name: Create Bitcoin P2P proxy service unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.service - content: | - [Unit] - Description=Bitcoin P2P Proxy to {{ bitcoin_tailscale_hostname }} - Requires=bitcoin-p2p-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ bitcoin_tailscale_hostname }}:{{ bitcoin_p2p_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin P2P proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - enabled: yes - state: started - - - name: Allow Bitcoin P2P port through UFW - ufw: - rule: allow - port: "{{ bitcoin_p2p_port | string }}" - proto: tcp - comment: "Bitcoin P2P public access" - - - name: Verify connectivity to knots-box via Tailscale - wait_for: - host: "{{ bitcoin_tailscale_hostname }}" - port: "{{ bitcoin_p2p_port }}" - timeout: 10 - ignore_errors: yes + - name: Expose Bitcoin P2P through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: bitcoin-p2p + socket_proxy_description: "Bitcoin P2P" + socket_proxy_listen_port: "{{ bitcoin_p2p_port }}" + socket_proxy_upstream_host: "{{ bitcoin_tailscale_hostname }}" + # These four were added by hand on vipy and were NOT in this playbook; + # writing the unit without them would have dropped FreeBind, which lets + # the socket bind before the address is up. + socket_proxy_documentation: "https://github.com/bitcoin/bitcoin" + socket_proxy_free_bind: true + socket_proxy_timeout_stop_sec: 5 + socket_proxy_log_to_journal: true - name: Display public endpoint + when: uptime_kuma_enabled | default(false) debug: msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" @@ -801,6 +778,7 @@ # Uptime Kuma TCP Monitor for Public P2P # =========================================== - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -873,6 +851,7 @@ mode: '0755' - name: Create temporary config for TCP monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -887,6 +866,7 @@ mode: '0644' - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py delegate_to: localhost become: no @@ -900,6 +880,7 @@ when: tcp_monitor_setup.stdout is defined - name: Clean up TCP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -909,8 +890,3 @@ - /tmp/setup_bitcoin_p2p_tcp_monitor.py - /tmp/ansible_bitcoin_p2p_config.yml - handlers: - - name: Restart bitcoin-p2p-proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - state: restarted diff --git a/ansible/services/caddy_playbook.yml b/ansible/services/caddy_playbook.yml index de98c8f..29e74b2 100644 --- a/ansible/services/caddy_playbook.yml +++ b/ansible/services/caddy_playbook.yml @@ -1,5 +1,5 @@ - name: Install and configure Caddy on Debian 12 - hosts: vps + hosts: caddy become: yes tasks: diff --git a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml index f33ded7..0c53e9d 100644 --- a/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml +++ b/ansible/services/datum-gateway/deploy_datum_gateway_playbook.yml @@ -25,7 +25,7 @@ # bitcoin_rpc_password - Shared with the bitcoin-knots deployment - name: Deploy DATUM Gateway on knots_box_local - hosts: knots_box_local + hosts: bitcoin become: yes vars_files: - ../../infra_vars.yml @@ -206,7 +206,18 @@ # =========================================== # Health Check Script + Systemd Timer # =========================================== + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create DATUM Gateway health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/datum-gateway-healthcheck-push.sh content: | @@ -287,6 +298,7 @@ daemon_reload: yes - name: Enable and start datum-gateway health check timer + when: uptime_kuma_enabled | default(false) systemd: name: datum-gateway-healthcheck.timer enabled: yes @@ -296,6 +308,7 @@ # Uptime Kuma Push Monitor Setup # =========================================== - name: Create Uptime Kuma push monitor setup script for DATUM Gateway + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -396,6 +409,7 @@ mode: "0755" - name: Create temporary config for push monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -408,6 +422,7 @@ mode: "0644" - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_datum_gateway_monitor.py delegate_to: localhost become: no @@ -421,6 +436,7 @@ when: monitor_setup.stdout is defined - name: Read push URL from file + when: uptime_kuma_enabled | default(false) slurp: src: /tmp/datum_gateway_push_url.txt delegate_to: localhost @@ -442,6 +458,7 @@ notify: Restart datum-gateway health check timer - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -460,6 +477,7 @@ daemon_reload: yes - name: Restart datum-gateway health check timer + when: uptime_kuma_enabled | default(false) systemd: name: datum-gateway-healthcheck.timer state: restarted @@ -469,8 +487,8 @@ # =========================================== # Caddy Reverse Proxy for DATUM Dashboard (on vipy) # =========================================== -- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on vipy - hosts: vipy +- name: Configure Caddy reverse proxy for DATUM Gateway dashboard on the edge host + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -480,65 +498,31 @@ vars: datum_gateway_subdomain: "{{ subdomains.datum_gateway }}" datum_gateway_domain: "{{ datum_gateway_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: "0755" + - name: Publish the DATUM Gateway dashboard through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: datum-gateway + caddy_site_domain: "{{ datum_gateway_domain }}" + caddy_site_upstream: "knots-box:{{ datum_gateway_api_port }}" + caddy_site_resolvers: "100.100.100.100" + caddy_site_basic_auth: + - user: "{{ datum_dashboard_username }}" + hash: "{{ datum_dashboard_password_hash }}" - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: "import sites-enabled/*" - insertafter: EOF - state: present - backup: yes - create: yes - mode: "0644" - - - name: Create Caddy reverse proxy config for DATUM Gateway dashboard - copy: - dest: "{{ caddy_sites_dir }}/datum-gateway.conf" - content: | - {{ datum_gateway_domain }} { - basic_auth { - {{ datum_dashboard_username }} {{ datum_dashboard_password_hash }} - } - reverse_proxy knots-box:{{ datum_gateway_api_port }} { - # Resolve via Tailscale MagicDNS - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: "0644" - - - name: Validate Caddy config - command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile - register: caddy_validate + # The role validates the site fragment on its own. This re-validates the + # whole assembled Caddyfile, which is the only thing that catches a + # conflict between this site and another. Kept from the hand-rolled + # version; the other nine services never had it. + - name: Validate the assembled Caddyfile + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile changed_when: false - - name: Display Caddy validation output - debug: - msg: "{{ caddy_validate.stdout_lines + caddy_validate.stderr_lines }}" - - - name: Reload Caddy - command: systemctl reload caddy - register: caddy_reload - - - name: Display Caddy reload output - debug: - msg: "{{ caddy_reload.stdout_lines + caddy_reload.stderr_lines }}" - - name: Display DATUM Gateway dashboard URL + when: uptime_kuma_enabled | default(false) debug: msg: "DATUM Gateway dashboard: https://{{ datum_gateway_domain }}" @@ -546,6 +530,7 @@ # Uptime Kuma HTTP Monitor for Public Dashboard # =========================================== - name: Create Uptime Kuma HTTP monitor setup script for DATUM dashboard + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -621,6 +606,7 @@ mode: "0755" - name: Create temporary config for HTTP monitor + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -634,6 +620,7 @@ mode: "0644" - name: Run Uptime Kuma HTTP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_datum_http_monitor.py delegate_to: localhost become: no @@ -647,6 +634,7 @@ when: http_monitor_setup.stdout is defined - name: Clean up HTTP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -662,8 +650,8 @@ # Miners connect to vipy:23334; traffic is forwarded to knots-box:23334 # over the Tailscale network, matching the Bitcoin P2P proxy pattern. # =========================================== -- name: Setup public Stratum port forwarding on vipy via systemd-socket-proxyd - hosts: vipy +- name: Setup public Stratum port forwarding on the edge host + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -675,64 +663,20 @@ uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Create Stratum proxy socket unit - copy: - dest: /etc/systemd/system/datum-stratum-proxy.socket - content: | - [Unit] - Description=DATUM Stratum Proxy Socket - - [Socket] - ListenStream={{ datum_gateway_stratum_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: "0644" - notify: Restart datum-stratum-proxy socket - - - name: Create Stratum proxy service unit - copy: - dest: /etc/systemd/system/datum-stratum-proxy.service - content: | - [Unit] - Description=DATUM Stratum Proxy to {{ datum_tailscale_hostname }} - Requires=datum-stratum-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ datum_tailscale_hostname }}:{{ datum_gateway_stratum_port }} - owner: root - group: root - mode: "0644" - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Stratum proxy socket - systemd: - name: datum-stratum-proxy.socket - enabled: yes - state: started - - - name: Allow Stratum port through UFW - ufw: - rule: allow - port: "{{ datum_gateway_stratum_port | string }}" - proto: tcp - comment: "DATUM Gateway Stratum public access" - - - name: Verify connectivity to knots-box Stratum via Tailscale - wait_for: - host: "{{ datum_tailscale_hostname }}" - port: "{{ datum_gateway_stratum_port }}" - timeout: 10 - ignore_errors: yes + - name: Expose the DATUM Stratum port through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: datum-stratum + socket_proxy_description: "DATUM Stratum" + socket_proxy_listen_port: "{{ datum_gateway_stratum_port }}" + socket_proxy_upstream_host: "{{ datum_tailscale_hostname }}" + # Matches the UFW comment already on vipy; the derived default would + # have said "DATUM Stratum" and rewritten the rule. + socket_proxy_ufw_comment: "DATUM Gateway Stratum public access" - name: Display public Stratum endpoint + when: uptime_kuma_enabled | default(false) debug: msg: "DATUM Stratum public endpoint: {{ ansible_host }}:{{ datum_gateway_stratum_port }}" @@ -740,6 +684,7 @@ # Uptime Kuma TCP Monitor for Public Stratum # =========================================== - name: Create Uptime Kuma TCP monitor setup script for Stratum + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -816,6 +761,7 @@ mode: "0755" - name: Create temporary config for Stratum TCP monitor + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -830,6 +776,7 @@ mode: "0644" - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_datum_stratum_tcp_monitor.py delegate_to: localhost become: no @@ -843,6 +790,7 @@ when: tcp_monitor_setup.stdout is defined - name: Clean up Stratum TCP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -852,8 +800,3 @@ - /tmp/setup_datum_stratum_tcp_monitor.py - /tmp/ansible_datum_stratum_config.yml - handlers: - - name: Restart datum-stratum-proxy socket - systemd: - name: datum-stratum-proxy.socket - state: restarted diff --git a/ansible/services/forgejo-runner/SETUP.md b/ansible/services/forgejo-runner/SETUP.md index a66d295..5cb2a3f 100644 --- a/ansible/services/forgejo-runner/SETUP.md +++ b/ansible/services/forgejo-runner/SETUP.md @@ -25,4 +25,6 @@ ansible-playbook ansible/services/forgejo-runner/deploy_forgejo_runner_playbook. 1. On the VM: `systemctl status forgejo-runner` should show active 2. In Forgejo: **Site Administration** > **Actions** > **Runners** should show the runner as online -3. In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings +3. ~~In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings~~ + *(Uptime Kuma was decommissioned 2026-09-11 — this check no longer applies. The + healthcheck timer still runs on the host but pushes nowhere. See `archive/uptime_kuma/`.)* diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml index a194178..bdc8428 100644 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -1,5 +1,5 @@ - name: Install Forgejo Runner on Debian 13 - hosts: forgejo_runner_local + hosts: ci_runner become: yes vars_files: - ../../infra_vars.yml @@ -147,7 +147,18 @@ register: runner_active changed_when: false + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Assert runner is running + when: uptime_kuma_enabled | default(false) assert: that: - runner_active.stdout == "active" @@ -155,6 +166,7 @@ # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── - name: Create Uptime Kuma push monitor setup script + when: uptime_kuma_enabled | default(false) copy: dest: /tmp/setup_forgejo_runner_monitor.py content: | @@ -238,6 +250,7 @@ become: no - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: > {{ ansible_playbook_python }} /tmp/setup_forgejo_runner_monitor.py @@ -256,10 +269,12 @@ changed_when: false - name: Parse monitor setup result + when: uptime_kuma_enabled | default(false) set_fact: monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - name: Set push URL + when: uptime_kuma_enabled | default(false) set_fact: uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" @@ -272,6 +287,7 @@ mode: '0755' - name: Create forgejo-runner healthcheck script + when: uptime_kuma_enabled | default(false) copy: dest: "{{ healthcheck_script_path }}" content: | @@ -385,6 +401,7 @@ Timeout: {{ healthcheck_timeout_seconds }}s - name: Clean up temporary monitor setup script + when: uptime_kuma_enabled | default(false) file: path: /tmp/setup_forgejo_runner_monitor.py state: absent diff --git a/ansible/services/forgejo/deploy_forgejo_playbook.yml b/ansible/services/forgejo/deploy_forgejo_playbook.yml index e17d08f..db78e95 100644 --- a/ansible/services/forgejo/deploy_forgejo_playbook.yml +++ b/ansible/services/forgejo/deploy_forgejo_playbook.yml @@ -1,5 +1,5 @@ - name: Install Forgejo on Debian 12 with Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./forgejo_vars.yml vars: forgejo_subdomain: "{{ subdomains.forgejo }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" forgejo_domain: "{{ forgejo_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -88,37 +87,26 @@ enabled: yes state: started - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for forgejo - copy: - dest: "{{ caddy_sites_dir }}/forgejo.conf" - content: | - {{ forgejo_domain }} { - reverse_proxy localhost:{{ forgejo_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish Forgejo through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: forgejo + caddy_site_domain: "{{ forgejo_domain }}" + caddy_site_upstream: "localhost:{{ forgejo_port }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Forgejo + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -193,6 +181,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -206,6 +195,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_forgejo_monitor.py delegate_to: localhost become: no @@ -214,6 +204,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/forgejo/forgejo_vars.yml b/ansible/services/forgejo/forgejo_vars.yml index 0bbb5a5..2a24133 100644 --- a/ansible/services/forgejo/forgejo_vars.yml +++ b/ansible/services/forgejo/forgejo_vars.yml @@ -12,7 +12,7 @@ forgejo_user: "git" # (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml new file mode 100644 index 0000000..d6aafef --- /dev/null +++ b/ansible/services/forgejo/setup_backup_forgejo.yml @@ -0,0 +1,25 @@ +--- +# Forgejo backup: dumps locally on vipy, encrypted with age. +# +# The biggest artefact in the estate (~2.7 G) and the reason retention here is +# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14. +# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the +# old job did the same. +- name: Configure the Forgejo backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./forgejo_vars.yml + + tasks: + - name: Ensure Forgejo dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: forgejo + backup_source_description: "Forgejo" + backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo" + backup_source_stop_service: forgejo + backup_source_retention_days: 2 + backup_source_on_calendar: "*-*-* 02:30:00" diff --git a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml index b90f0fb..c3ff8d6 100644 --- a/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml +++ b/ansible/services/forgejo/setup_backup_forgejo_to_lapy.yml @@ -1,6 +1,6 @@ --- - name: Configure local backup for Forgejo from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 1255cd6..01e9f17 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Fulcrum Electrum Server - hosts: fulcrum_box_local + hosts: electrum become: yes vars_files: - ../../infra_vars.yml @@ -259,7 +259,18 @@ debug: msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Fulcrum health check and push script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/fulcrum-healthcheck-push.sh content: | @@ -332,6 +343,7 @@ mode: '0644' - name: Create systemd service for Fulcrum health check + when: uptime_kuma_enabled | default(false) copy: dest: /etc/systemd/system/fulcrum-healthcheck.service content: | @@ -364,6 +376,7 @@ state: started - name: Create Uptime Kuma push monitor setup script for Fulcrum + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -453,6 +466,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -465,6 +479,7 @@ mode: '0644' - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_fulcrum_monitor.py delegate_to: localhost become: no @@ -505,6 +520,7 @@ when: uptime_kuma_push_url | default('') != '' - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -518,13 +534,14 @@ handlers: - name: Restart fulcrum + when: uptime_kuma_enabled | default(false) systemd: name: fulcrum state: restarted -- name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd - hosts: vipy +- name: Setup public Fulcrum SSL forwarding on the edge host + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -535,64 +552,17 @@ uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Create Fulcrum SSL proxy socket unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.socket - content: | - [Unit] - Description=Fulcrum SSL Proxy Socket - - [Socket] - ListenStream={{ fulcrum_ssl_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum-ssl-proxy socket - - - name: Create Fulcrum SSL proxy service unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.service - content: | - [Unit] - Description=Fulcrum SSL Proxy to {{ fulcrum_tailscale_hostname }} - Requires=fulcrum-ssl-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ fulcrum_tailscale_hostname }}:{{ fulcrum_ssl_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum SSL proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - enabled: yes - state: started - - - name: Allow Fulcrum SSL port through UFW - ufw: - rule: allow - port: "{{ fulcrum_ssl_port | string }}" - proto: tcp - comment: "Fulcrum SSL public access" - - - name: Verify connectivity to fulcrum-box via Tailscale - wait_for: - host: "{{ fulcrum_tailscale_hostname }}" - port: "{{ fulcrum_ssl_port }}" - timeout: 10 - ignore_errors: yes + - name: Expose Fulcrum SSL through a socket proxy + ansible.builtin.include_role: + name: socket_proxy + vars: + socket_proxy_name: fulcrum-ssl + socket_proxy_description: "Fulcrum SSL" + socket_proxy_listen_port: "{{ fulcrum_ssl_port }}" + socket_proxy_upstream_host: "{{ fulcrum_tailscale_hostname }}" - name: Display public endpoint + when: uptime_kuma_enabled | default(false) debug: msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" @@ -600,6 +570,7 @@ # Uptime Kuma TCP Monitor for Public SSL Port # =========================================== - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -672,6 +643,7 @@ mode: '0755' - name: Create temporary config for TCP monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -686,6 +658,7 @@ mode: '0644' - name: Run Uptime Kuma TCP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py delegate_to: localhost become: no @@ -699,6 +672,7 @@ when: tcp_monitor_setup.stdout is defined - name: Clean up TCP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -708,9 +682,4 @@ - /tmp/setup_fulcrum_ssl_tcp_monitor.py - /tmp/ansible_fulcrum_ssl_config.yml - handlers: - - name: Restart fulcrum-ssl-proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - state: restarted diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 1bcf5bf..527e4c2 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy headscale and configure Caddy reverse proxy - hosts: spacey + hosts: vpn_control become: no vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./headscale_vars.yml vars: headscale_subdomain: "{{ subdomains.headscale }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" headscale_base_domain: "tailnet.{{ root_domain }}" headscale_namespace: "{{ service_settings.headscale.namespace }}" @@ -230,41 +229,47 @@ port: '3478' proto: udp - - name: Ensure Caddy sites-enabled directory exists - become: yes - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - become: yes - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for headscale - become: yes - copy: - dest: "{{ caddy_sites_dir }}/headscale.conf" - content: | - {{ headscale_domain }} { - reverse_proxy localhost:{{ headscale_port }} + - name: Publish headscale through Caddy + ansible.builtin.include_role: + name: caddy_site + # This play is become: no and elevates per task. `apply` is how an + # include_role passes become down to the role's tasks - `become:` on + # the include itself is rejected. The role's handler sets its own. + apply: + become: yes + vars: + caddy_site_name: headscale + caddy_site_domain: "{{ headscale_domain }}" + # Raw body, and it must stay raw: the /admin* route in front of + # Headplane is not expressible as a plain reverse_proxy. The previous + # version of this task wrote only `reverse_proxy localhost:8080`, which + # would have deleted the admin route and its auth on the next run. + caddy_site_body: | + @headplane { + path /admin* } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - become: yes - command: systemctl reload caddy + handle @headplane { + basicauth { + {{ headscale_ui_username }} {{ headscale_ui_password_hash }} + } + reverse_proxy http://localhost:{{ headplane_port }} + } + # Headscale API is protected by its own API key authentication + # All API operations require a valid Bearer token in the Authorization header + reverse_proxy * http://localhost:{{ headscale_port }} + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Headscale + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -339,6 +344,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -352,6 +358,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_headscale_monitor.py delegate_to: localhost become: no @@ -360,6 +367,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/headscale/headscale_vars.yml b/ansible/services/headscale/headscale_vars.yml index 653c175..05fc040 100644 --- a/ansible/services/headscale/headscale_vars.yml +++ b/ansible/services/headscale/headscale_vars.yml @@ -13,7 +13,7 @@ headscale_data_dir: /var/lib/headscale # Namespace now configured in services_config.yml under service_settings.headscale.namespace # Remote access -remote_host_name: "spacey" +remote_host_name: "{{ groups['vpn_control'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" @@ -22,3 +22,6 @@ remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" # Local backup local_backup_dir: "{{ lookup('env', 'HOME') }}/headscale-backups" backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/headscale_backup.sh" + +# Headplane (headscale admin UI), proxied at /admin* behind Caddy basic auth +headplane_port: 3000 diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml new file mode 100644 index 0000000..9c72d1d --- /dev/null +++ b/ansible/services/headscale/setup_backup_headscale.yml @@ -0,0 +1,21 @@ +--- +- name: Configure the Headscale backup on the vpn_control host + hosts: vpn_control + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./headscale_vars.yml + + tasks: + - name: Ensure Headscale dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: headscale + backup_source_description: "Headscale" + # -C / with relative paths: avoids tar's "removing leading /" and makes + # the restore target explicit. + backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" + backup_source_stop_service: headscale + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml index 5f9a764..acc82ce 100644 --- a/ansible/services/headscale/setup_backup_headscale_to_lapy.yml +++ b/ansible/services/headscale/setup_backup_headscale_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Headscale from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/lnbits/deploy_lnbits_playbook.yml b/ansible/services/lnbits/deploy_lnbits_playbook.yml index e5d546b..65bdfbf 100644 --- a/ansible/services/lnbits/deploy_lnbits_playbook.yml +++ b/ansible/services/lnbits/deploy_lnbits_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy LNBits with Poetry and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./lnbits_vars.yml vars: lnbits_subdomain: "{{ subdomains.lnbits }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" lnbits_domain: "{{ lnbits_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -147,41 +146,28 @@ enabled: yes state: started - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for lnbits - copy: - dest: "{{ caddy_sites_dir }}/lnbits.conf" - content: | - {{ lnbits_domain }} { - reverse_proxy localhost:{{ lnbits_port }} { - header_up X-Forwarded-Host {{ lnbits_domain }} - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish LNBits through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: lnbits + caddy_site_domain: "{{ lnbits_domain }}" + caddy_site_upstream: "localhost:{{ lnbits_port }}" + caddy_site_headers_up: + X-Forwarded-Host: "{{ lnbits_domain }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for LNBits + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -256,6 +242,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -269,6 +256,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_lnbits_monitor.py delegate_to: localhost become: no @@ -277,6 +265,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/lnbits/lnbits_vars.yml b/ansible/services/lnbits/lnbits_vars.yml index bdb97df..eabd6fc 100644 --- a/ansible/services/lnbits/lnbits_vars.yml +++ b/ansible/services/lnbits/lnbits_vars.yml @@ -6,7 +6,7 @@ lnbits_port: 8765 # (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml new file mode 100644 index 0000000..46abf31 --- /dev/null +++ b/ansible/services/lnbits/setup_backup_lnbits.yml @@ -0,0 +1,25 @@ +--- +# LNBits backup: dumps locally on vipy, encrypted with age. +# The old job produced TWO gpg artefacts (data, then .env separately). They are +# folded into one tar here so the wallet database and the .env that configures +# it are always the same point in time; two artefacts written by two runs can +# drift. Pulling one file back out needs no unpacking: +# age -d -i | tar -xzO opt/lnbits/lnbits/.env +- name: Configure the LNBits backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./lnbits_vars.yml + + tasks: + - name: Ensure LNBits dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: lnbits + backup_source_description: "LNBits" + backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env" + backup_source_stop_service: lnbits + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:20:00" diff --git a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml index 5d10dec..48296cb 100644 --- a/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml +++ b/ansible/services/lnbits/setup_backup_lnbits_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for LNBits from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index da56bd6..04e99cf 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Memos on memos-box - hosts: memos_box_local + hosts: memos become: yes vars_files: - ../../infra_vars.yml @@ -139,14 +139,25 @@ msg: "Memos is running on port {{ memos_port }}. Access via Tailscale at http://{{ memos_tailscale_hostname }}:{{ memos_port }}" handlers: + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Restart memos + when: uptime_kuma_enabled | default(false) systemd: name: memos state: restarted -- name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) - hosts: vipy +- name: Configure Caddy reverse proxy for Memos on the edge host (via Tailscale) + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -155,47 +166,21 @@ - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for memos (via Tailscale) - copy: - dest: "{{ caddy_sites_dir }}/memos.conf" - content: | - {{ memos_domain }} { - reverse_proxy {{ memos_tailscale_hostname }}:{{ memos_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish Memos through Caddy (via Tailscale) + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: memos + caddy_site_domain: "{{ memos_domain }}" + caddy_site_upstream: "{{ memos_tailscale_hostname }}:{{ memos_port }}" + caddy_site_resolvers: "100.100.100.100" - name: Create Uptime Kuma monitor setup script for Memos + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -275,6 +260,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -288,6 +274,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_memos_monitor.py delegate_to: localhost become: no @@ -296,6 +283,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml new file mode 100644 index 0000000..2131924 --- /dev/null +++ b/ansible/services/memos/setup_backup_memos.yml @@ -0,0 +1,26 @@ +--- +# Memos backup: dumps locally on memos-box, encrypted with age. +# Replaces the lapy pull, which had been writing EMPTY directories since +# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned +# to a different machine that has no rsync. +- name: Configure the Memos backup on its own host + hosts: memos + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./memos_vars.yml + + tasks: + - name: Ensure Memos dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: memos + backup_source_description: "Memos" + backup_source_dump_command: "tar -czf - -C / var/lib/memos" + # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a + # consistent database rather than a torn mid-write copy. The old rsync + # job did not stop it. + backup_source_stop_service: memos + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml index 6d9c161..c32c332 100644 --- a/ansible/services/memos/setup_backup_memos_to_lapy.yml +++ b/ansible/services/memos/setup_backup_memos_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Memos from memos-box - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 658180a..e9cab0e 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Mempool Block Explorer with Docker - hosts: mempool_box_local + hosts: mempool become: yes vars_files: - ../../infra_vars.yml @@ -226,7 +226,18 @@ delay: 5 ignore_errors: yes + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Display deployment status + when: uptime_kuma_enabled | default(false) debug: msg: - "Mempool deployment complete!" @@ -239,6 +250,7 @@ # Health Check Scripts for Uptime Kuma Push Monitors # =========================================== - name: Create Mempool MariaDB health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh content: | @@ -273,6 +285,7 @@ mode: '0755' - name: Create Mempool backend health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/mempool-backend-healthcheck-push.sh content: | @@ -307,6 +320,7 @@ mode: '0755' - name: Create Mempool frontend health check script + when: uptime_kuma_enabled | default(false) copy: dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh content: | @@ -396,6 +410,7 @@ daemon_reload: yes - name: Enable and start health check timers + when: uptime_kuma_enabled | default(false) systemd: name: "mempool-{{ item }}-healthcheck.timer" enabled: yes @@ -409,6 +424,7 @@ # Uptime Kuma Push Monitor Setup # =========================================== - name: Create Uptime Kuma push monitor setup script for Mempool + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -492,6 +508,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -507,6 +524,7 @@ mode: '0644' - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_mempool_monitors.py delegate_to: localhost become: no @@ -520,6 +538,7 @@ when: monitor_setup.stdout is defined - name: Read push URLs from file + when: uptime_kuma_enabled | default(false) slurp: src: /tmp/mempool_push_urls.yml delegate_to: localhost @@ -573,6 +592,7 @@ when: push_urls is defined - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: @@ -583,8 +603,8 @@ - /tmp/ansible_mempool_config.yml - /tmp/mempool_push_urls.yml -- name: Configure Caddy reverse proxy for Mempool on vipy - hosts: vipy +- name: Configure Caddy reverse proxy for Mempool on the edge host + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -594,49 +614,19 @@ vars: mempool_subdomain: "{{ subdomains.mempool }}" mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for Mempool - copy: - dest: "{{ caddy_sites_dir }}/mempool.conf" - content: | - {{ mempool_domain }} { - reverse_proxy mempool-box:{{ mempool_frontend_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - systemd: - name: caddy - state: reloaded + - name: Publish Mempool through Caddy (via Tailscale) + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: mempool + caddy_site_domain: "{{ mempool_domain }}" + caddy_site_upstream: "mempool-box:{{ mempool_frontend_port }}" + caddy_site_resolvers: "100.100.100.100" - name: Display Mempool URL + when: uptime_kuma_enabled | default(false) debug: msg: "Mempool is now available at https://{{ mempool_domain }}" @@ -644,6 +634,7 @@ # Uptime Kuma HTTP Monitor for Public Endpoint # =========================================== - name: Create Uptime Kuma HTTP monitor setup script for Mempool + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -714,6 +705,7 @@ mode: '0755' - name: Create temporary config for HTTP monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -727,6 +719,7 @@ mode: '0644' - name: Run Uptime Kuma HTTP monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_mempool_http_monitor.py delegate_to: localhost become: no @@ -740,6 +733,7 @@ when: http_monitor_setup.stdout is defined - name: Clean up HTTP monitor temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index b8c0064..7379d5f 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy ntfy-emergency-app with Docker Compose and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./ntfy_emergency_app_vars.yml vars: ntfy_emergency_app_subdomain: "{{ subdomains.ntfy_emergency_app }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" ntfy_emergency_app_domain: "{{ ntfy_emergency_app_subdomain }}.{{ root_domain }}" ntfy_service_domain: "{{ subdomains.ntfy }}.{{ root_domain }}" ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" @@ -49,37 +48,26 @@ args: chdir: "{{ ntfy_emergency_app_dir }}" - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for ntfy-emergency-app - copy: - dest: "{{ caddy_sites_dir }}/ntfy-emergency-app.conf" - content: | - {{ ntfy_emergency_app_domain }} { - reverse_proxy localhost:{{ ntfy_emergency_app_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish ntfy-emergency-app through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: ntfy-emergency-app + caddy_site_domain: "{{ ntfy_emergency_app_domain }}" + caddy_site_upstream: "localhost:{{ ntfy_emergency_app_port }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -159,6 +147,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -172,6 +161,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_ntfy_emergency_app_monitor.py delegate_to: localhost become: no @@ -180,6 +170,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml index 415bc4d..a3bb480 100644 --- a/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml +++ b/ansible/services/ntfy-emergency-app/ntfy_emergency_app_vars.yml @@ -9,7 +9,7 @@ ntfy_emergency_app_topic: "emergencia" ntfy_emergency_app_ui_message: "Leave Pablo a message, he will respond as soon as possible" # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/ntfy/deploy_ntfy_playbook.yml b/ansible/services/ntfy/deploy_ntfy_playbook.yml index 0729baa..61fafe1 100644 --- a/ansible/services/ntfy/deploy_ntfy_playbook.yml +++ b/ansible/services/ntfy/deploy_ntfy_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy ntfy and configure Caddy reverse proxy - hosts: watchtower + hosts: monitoring become: yes vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./ntfy_vars.yml vars: ntfy_subdomain: "{{ subdomains.ntfy }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" ntfy_domain: "{{ ntfy_subdomain }}.{{ root_domain }}" tasks: @@ -76,42 +75,23 @@ shell: | (echo "{{ ntfy_password }}"; echo "{{ ntfy_password }}") | ntfy user add --role=admin "{{ ntfy_username }}" - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' + - name: Publish ntfy through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: ntfy + caddy_site_domain: "{{ ntfy_domain }}, http://{{ ntfy_domain }}" + # Raw body: ntfy needs a plain-HTTP listener for its CLI/app clients, + # with only GETs to the docs and topic paths redirected to HTTPS. + caddy_site_body: | + reverse_proxy 127.0.0.1:{{ ntfy_port }} - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for ntfy - copy: - dest: "{{ caddy_sites_dir }}/ntfy.conf" - content: | - {{ ntfy_domain }}, http://{{ ntfy_domain }} { - reverse_proxy 127.0.0.1:{{ ntfy_port }} - - @httpget { - protocol http - method GET - path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) - } - redir @httpget https://{host}{uri} + @httpget { + protocol http + method GET + path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + redir @httpget https://{host}{uri} handlers: - name: Restart ntfy diff --git a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml index 5ba03f1..2d3d22a 100644 --- a/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml +++ b/ansible/services/ntfy/setup_ntfy_uptime_kuma_notification.yml @@ -1,5 +1,18 @@ +# ═════════════════════════════════════════════════════════════════════════════ +# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# +# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username +# and uptime_kuma_password were removed from the vault, so the "Validate Uptime +# Kuma configuration" assert stops it before anything is installed or changed. +# +# It is kept because the CHECK LOGIC is the durable part — what gets measured, +# the thresholds, and the systemd timer plumbing. When something replaces Uptime +# Kuma, only the push transport needs rewriting; the rest still applies. +# +# What was being monitored: archive/uptime_kuma/MONITORS.md +# ═════════════════════════════════════════════════════════════════════════════ - name: Setup ntfy as Uptime Kuma Notification Channel - hosts: watchtower + hosts: monitoring become: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml index f4ee8ec..96d030f 100644 --- a/ansible/services/personal-blog/deploy_personal_blog_playbook.yml +++ b/ansible/services/personal-blog/deploy_personal_blog_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy personal blog static site with Caddy file server - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./personal_blog_vars.yml vars: personal_blog_subdomain: "{{ subdomains.personal_blog }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" personal_blog_domain: "{{ personal_blog_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -51,38 +50,26 @@ group: www-data mode: '0664' - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy file server configuration for personal blog - copy: - dest: "{{ caddy_sites_dir }}/personal-blog.conf" - content: | - {{ personal_blog_domain }} { - root * {{ personal_blog_web_root }} - file_server - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish the blog through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: personal-blog + caddy_site_domain: "{{ personal_blog_domain }}" + caddy_site_root: "{{ personal_blog_web_root }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Personal Blog + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -157,6 +144,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -170,6 +158,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_personal_blog_monitor.py delegate_to: localhost become: no @@ -178,6 +167,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/personal-blog/personal_blog_vars.yml b/ansible/services/personal-blog/personal_blog_vars.yml index 59e0921..a1f34e3 100644 --- a/ansible/services/personal-blog/personal_blog_vars.yml +++ b/ansible/services/personal-blog/personal_blog_vars.yml @@ -4,7 +4,7 @@ personal_blog_web_root: "/var/www/pablohere.contrapeso.xyz" # Remote access for deployment -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml index 2b90d68..99f9b34 100644 --- a/ansible/services/personal-blog/setup_deploy_alias_lapy.yml +++ b/ansible/services/personal-blog/setup_deploy_alias_lapy.yml @@ -1,5 +1,5 @@ - name: Configure deployment alias for personal blog in lapy .bashrc - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml index 65ad7c9..8b30a1f 100644 --- a/ansible/services/phoenixd/deploy_phoenixd_playbook.yml +++ b/ansible/services/phoenixd/deploy_phoenixd_playbook.yml @@ -19,8 +19,8 @@ # ⚠️ After the first run, back up {{ phoenixd_data_dir }}/seed.dat. Losing it # means losing the funds. See setup_backup_phoenixd_to_lapy.yml. -- name: Deploy phoenixd on vipy - hosts: vipy +- name: Deploy phoenixd on the edge host + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -253,7 +253,18 @@ # =========================================== # Health Check Script + Systemd Timer # =========================================== + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create phoenixd health check script + when: uptime_kuma_enabled | default(false) copy: dest: "{{ phoenixd_healthcheck_script_path }}" content: | @@ -340,6 +351,7 @@ daemon_reload: yes - name: Enable and start phoenixd health check timer + when: uptime_kuma_enabled | default(false) systemd: name: "{{ phoenixd_healthcheck_service_name }}.timer" enabled: yes @@ -349,6 +361,7 @@ # Uptime Kuma Push Monitor Setup # =========================================== - name: Create Uptime Kuma push monitor setup script for phoenixd + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -443,6 +456,7 @@ mode: "0755" - name: Create temporary config for push monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -455,6 +469,7 @@ mode: "0644" - name: Run Uptime Kuma push monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_phoenixd_monitor.py delegate_to: localhost become: no @@ -468,6 +483,7 @@ when: monitor_setup.stdout is defined - name: Read push URL from file + when: uptime_kuma_enabled | default(false) slurp: src: /tmp/phoenixd_push_url.txt delegate_to: localhost @@ -489,6 +505,7 @@ notify: Restart phoenixd health check timer - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/phoenixd/phoenixd_vars.yml b/ansible/services/phoenixd/phoenixd_vars.yml index 57218dc..f67c302 100644 --- a/ansible/services/phoenixd/phoenixd_vars.yml +++ b/ansible/services/phoenixd/phoenixd_vars.yml @@ -41,7 +41,7 @@ phoenixd_healthcheck_service_name: phoenixd-healthcheck phoenixd_monitor_name: "Phoenixd" # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml index cc008e6..7474af1 100644 --- a/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml +++ b/ansible/services/phoenixd/setup_backup_phoenixd_to_lapy.yml @@ -13,7 +13,7 @@ # Because both files are static, phoenixd does not need to be stopped. - name: Configure local backup for phoenixd from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml index 0340538..74e87d8 100644 --- a/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml +++ b/ansible/services/vaultwarden/deploy_vaultwarden_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Vaultwarden with Docker Compose and configure Caddy reverse proxy - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml @@ -8,7 +8,6 @@ - ./vaultwarden_vars.yml vars: vaultwarden_subdomain: "{{ subdomains.vaultwarden }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" vaultwarden_domain: "{{ vaultwarden_subdomain }}.{{ root_domain }}" uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" @@ -81,37 +80,26 @@ name: fail2ban state: restarted - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Create Caddy reverse proxy configuration for vaultwarden - copy: - dest: "{{ caddy_sites_dir }}/vaultwarden.conf" - content: | - {{ vaultwarden_domain }} { - reverse_proxy localhost:{{ vaultwarden_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy + - name: Publish Vaultwarden through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: vaultwarden + caddy_site_domain: "{{ vaultwarden_domain }}" + caddy_site_upstream: "localhost:{{ vaultwarden_port }}" + # ═════════════════════════════════════════════════════════════════════════ + # DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. + # + # Every task below is inert: uptime_kuma_enabled is false in + # group_vars/all/main.yml, so they all skip and the deployment above still + # runs normally. Kept because the health-check logic is the durable part — + # when a replacement exists, rewire the push transport and flip the flag. + # + # What was being monitored: archive/uptime_kuma/MONITORS.md + # ═════════════════════════════════════════════════════════════════════════ - name: Create Uptime Kuma monitor setup script for Vaultwarden + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -191,6 +179,7 @@ mode: '0755' - name: Create temporary config for monitor setup + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no copy: @@ -204,6 +193,7 @@ mode: '0644' - name: Run Uptime Kuma monitor setup + when: uptime_kuma_enabled | default(false) command: python3 /tmp/setup_vaultwarden_monitor.py delegate_to: localhost become: no @@ -212,6 +202,7 @@ ignore_errors: yes - name: Clean up temporary files + when: uptime_kuma_enabled | default(false) delegate_to: localhost become: no file: diff --git a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml index b041e8e..eebb214 100644 --- a/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml +++ b/ansible/services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml @@ -1,5 +1,5 @@ - name: Disable Vaultwarden Signups - hosts: vipy + hosts: edge become: yes vars_files: - ../../infra_vars.yml diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml new file mode 100644 index 0000000..8430475 --- /dev/null +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml @@ -0,0 +1,25 @@ +--- +# Vaultwarden backup: dumps locally on vipy, encrypted with age. +# Previously rsynced to lapy in the CLEAR; the artefact now never exists +# unencrypted, on disk or on the wire. +- name: Configure the Vaultwarden backup on the edge host + hosts: edge + become: yes + vars_files: + - ../../group_vars/all/main.yml + - ./vaultwarden_vars.yml + + tasks: + - name: Ensure Vaultwarden dumps itself, encrypted, on a timer + ansible.builtin.include_role: + name: backup_source + vars: + backup_source_name: vaultwarden + backup_source_description: "Vaultwarden" + backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data" + # Not systemd — a docker compose stack — so stop/start explicitly. + # sqlite in WAL mode, hence stopping at all. + backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" + backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" + backup_source_retention_days: 7 + backup_source_on_calendar: "*-*-* 02:10:00" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml index 064d633..096f9d3 100644 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml +++ b/ansible/services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Vaultwarden from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/vaultwarden/vaultwarden_vars.yml b/ansible/services/vaultwarden/vaultwarden_vars.yml index 75e527d..e60f1d4 100644 --- a/ansible/services/vaultwarden/vaultwarden_vars.yml +++ b/ansible/services/vaultwarden/vaultwarden_vars.yml @@ -6,7 +6,7 @@ vaultwarden_port: 8222 # (caddy_sites_dir and subdomain now in services_config.yml) # Remote access -remote_host_name: "vipy" +remote_host_name: "{{ groups['edge'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/ansible/services_config.yml b/ansible/services_config.yml index 1bc9db0..a20c853 100644 --- a/ansible/services_config.yml +++ b/ansible/services_config.yml @@ -5,6 +5,9 @@ subdomains: # Monitoring Services (on watchtower) ntfy: ntfy + # DEPRECATED 2026-09-11 — Uptime Kuma is decommissioned and this subdomain no + # longer resolves to anything. Kept only because the deprecated monitoring + # blocks still template it into uptime_kuma_api_url. See archive/uptime_kuma/. uptime_kuma: uptime # VPN Infrastructure (on spacey) diff --git a/archive/uptime_kuma/MONITORS.md b/archive/uptime_kuma/MONITORS.md new file mode 100644 index 0000000..53b1d89 --- /dev/null +++ b/archive/uptime_kuma/MONITORS.md @@ -0,0 +1,152 @@ +# Uptime Kuma — monitor inventory (archived) + +Captured from the live instance at `https://uptime.contrapeso.xyz` on 2026-09-11, +immediately before decommissioning. This is the **authoritative** record: most of +these monitors existed only in the Uptime Kuma UI and were never described by any +playbook in this repo. + +**75 monitors total** — 16 group, 8 http, 3 port, 48 push. All were active. + +Push tokens are deliberately **not** recorded here: they are live credentials, and +anything holding one could report a false 'up'. They die with the server. + +--- + +## arbret - production *(7 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| arbret.com - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbret.com - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbret.com - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | +| arbret.com - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | +| arbret.com - health | push | — | 120s | Healthy when GET /api/health returns status ok | +| arbret.com - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | +| arbret.com - postgresql | push | — | 120s | Healthy when postgresql.service is active | + +## arbret - staging *(7 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| arbretstaging.contrapeso.xyz - arbret-analytics | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbretstaging.contrapeso.xyz - arbret-backup | push | — | 120s | Healthy when timer is scheduled and last run succeeded | +| arbretstaging.contrapeso.xyz - arbret-server | push | — | 120s | Healthy when arbret-server.service is active | +| arbretstaging.contrapeso.xyz - arbret-worker | push | — | 120s | Healthy when arbret-worker.service is active | +| arbretstaging.contrapeso.xyz - health | push | — | 120s | Healthy when GET /api/health returns status ok | +| arbretstaging.contrapeso.xyz - https | push | — | 120s | Healthy when HTTPS front-door returns 200 | +| arbretstaging.contrapeso.xyz - postgresql | push | — | 120s | Healthy when postgresql.service is active | + +## arbret-staging-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-arbret-staging-box-root | push | — | 960s | upside-down, Disk Usage: arbret-staging-box (/) - Alerts when usage excee | +| system-healthcheck-arbret-staging-box | push | — | 90s | System Healthcheck: arbret-staging-box - Regular healthcheck | + +## forgejo-runner-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-forgejo-runner-box-root | push | — | 960s | upside-down, Disk Usage: forgejo-runner-box (/) - Alerts when usage excee | +| system-healthcheck-forgejo-runner-box | push | — | 90s | System Healthcheck: forgejo-runner-box - Regular healthcheck | + +## fulcrum-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-fulcrum-box-root | push | — | 960s | upside-down, Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-fulcrum-box | push | — | 90s | System Healthcheck: fulcrum-box - Regular healthcheck ping e | + +## knots-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-knots-box-root | push | — | 960s | upside-down, Disk Usage: knots-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-knots-box | push | — | 90s | System Healthcheck: knots-box - Regular healthcheck ping eve | + +## memos-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-memos-box-root | push | — | 960s | upside-down, Disk Usage: memos-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-memos-box | push | — | 90s | System Healthcheck: memos-box - Regular healthcheck ping eve | + +## mempool-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-mempool-box-root | push | — | 960s | upside-down, Disk Usage: mempool-box (/) - Alerts when usage exceeds 80% | +| system-healthcheck-mempool-box | push | — | 90s | System Healthcheck: mempool-box - Regular healthcheck ping e | + +## nodito - infra *(4 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| UPS ONLINE | push | https:// | 90s | — | +| cpu-temp-nodito | push | — | 120s | upside-down, CPU Temperature: nodito - Alerts when temperature exceeds 80 | +| system-healthcheck-nodito | push | — | 300s | System Healthcheck: nodito - Regular healthcheck ping every | +| zfs-health-nodito | push | — | 90000s | ZFS Pool Health: nodito - Daily health check for pool proxmo | + +## nonkeiwaisi-box - infra *(0 monitors)* + +_(empty)_ + +## prd-arbret - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-arbret-root | push | — | 960s | upside-down, Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-arbret | push | — | 90s | System Healthcheck: prd-arbret - Regular healthcheck ping ev | + +## prd-spacey - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-spacey-root | push | — | 960s | upside-down, Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-spacey | push | — | 90s | System Healthcheck: prd-spacey - Regular healthcheck ping ev | + +## prd-vipy - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-vipy-root | push | — | 960s | upside-down, Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80% | +| system-healthcheck-prd-vipy | push | — | 90s | System Healthcheck: prd-vipy - Regular healthcheck ping ever | + +## prd-watchtower - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-prd-watchtower-root | push | — | 960s | upside-down, Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 8 | +| system-healthcheck-prd-watchtower | push | — | 90s | System Healthcheck: prd-watchtower - Regular healthcheck pin | + +## services *(19 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| Forgejo | http | https://forgejo.contrapeso.xyz/api/healthz | 90s | — | +| Headscale | http | https://headscale.contrapeso.xyz/health | 60s | — | +| LNBits | http | https://wallet.contrapeso.xyz/api/v1/health | 60s | — | +| Memos | http | https://memos.contrapeso.xyz/healthz | 60s | — | +| Mempool Public Access | http | https://mempool.contrapeso.xyz | 60s | — | +| Personal Blog | http | https://pablohere.contrapeso.xyz | 60s | — | +| Vaultwarden | http | https://vault.contrapeso.xyz/alive | 60s | — | +| ntfy-emergency-app | http | https://avisame.contrapeso.xyz | 60s | — | +| Bitcoin Knots P2P Public | port | 167.172.107.33:8333 | 60s | — | +| DATUM Stratum (public) | port | 167.172.107.33:23334 | 60s | — | +| Fulcrum SSL Public | port | 167.172.107.33:50002 | 60s | — | +| Bitcoin Knots | push | — | 90s | — | +| DATUM Gateway | push | — | 90s | — | +| Fulcrum | push | — | 90s | — | +| Mempool Backend | push | — | 180s | — | +| Mempool Frontend | push | — | 90s | — | +| Mempool MariaDB | push | — | 90s | — | +| Phoenixd | push | — | 90s | — | +| forgejo-runner-healthcheck | push | — | 90s | Forgejo Runner healthcheck - ping every 60s | + +## small-backups-box - infra *(2 monitors)* + +| Monitor | Type | Target | Interval | Notes | +|---|---|---|---|---| +| disk-usage-small-backups-box-root | push | — | 960s | upside-down, Disk Usage: small-backups-box (/) - Alerts when usage exceed | +| system-healthcheck-small-backups-box | push | — | 90s | System Healthcheck: small-backups-box - Regular healthcheck | + diff --git a/archive/uptime_kuma/README.md b/archive/uptime_kuma/README.md new file mode 100644 index 0000000..5405ae0 --- /dev/null +++ b/archive/uptime_kuma/README.md @@ -0,0 +1,51 @@ +# Uptime Kuma — archived + +Uptime Kuma was the monitoring stack for this infrastructure until **2026-09-11**, when +it was decommissioned. Everything that referenced it has been removed from the live +playbooks; this folder is the record of what it was, kept so the setup can be understood +later without digging through git history. + +## Contents + +| File | What it is | +|---|---| +| `MONITORS.md` | Every monitor that existed, grouped as it was in the UI. The authoritative record. | +| `monitors.json` | The same data, machine-readable, as returned by the API. | +| `deploy_uptime_kuma_playbook.yml` | How the server itself was deployed (Docker Compose on `monitoring`, behind Caddy). | +| `uptime_kuma_vars.yml` | Variables the deploy playbook needed — it is unreadable without these. | +| `setup_backup_uptime_kuma_to_lapy.yml` | How its data was backed up, and to where. Useful when disposing of the old volumes. | + +## Why the inventory was captured from the live server, not the repo + +The playbooks only ever created **17** monitors. The live instance had **75**. The +difference was created by hand in the UI and existed nowhere else — so a repo-derived +list would have silently lost two thirds of the picture. `MONITORS.md` is a snapshot of +the real thing, taken immediately before removal. + +Push tokens are deliberately excluded. They are live credentials — anything holding one +can report a false "up" — and they become meaningless once the server is gone. + +## What removal did NOT do + +Removing the playbook code does not touch the machines. **48 push monitors** were driven +by scripts and systemd timers installed *on the hosts*, which keep firing on their +schedule and curling an endpoint that no longer answers. They are harmless but they are +still there, writing logs and failing quietly. + +Left behind, per host: + +- `/opt/disk-monitoring` + `disk-usage-monitor.{service,timer}` — on all 12 `managed` hosts +- `/opt/system-healthcheck` + `system-healthcheck.{service,timer}` — on all 12 `managed` hosts +- `/opt/nodito-monitoring` + `nodito-cpu-temp-monitor.{service,timer}` — `nodito` +- `/opt/zfs-monitoring` — `nodito` +- `/opt/ups-monitoring` — `nodito` +- `bitcoin-knots-healthcheck.{service,timer}` — `bitcoin` +- `datum-gateway-healthcheck.{service,timer}` — `bitcoin` +- `fulcrum-healthcheck.{service,timer}` — `electrum` +- `mempool-{backend,frontend,mariadb}-healthcheck.service` — `mempool` +- phoenixd and forgejo-runner healthcheck units — `edge`, `ci_runner` + +Note `nut-monitor.service` on `nodito` is **NUT's own daemon**, not a monitoring +leftover — do not remove it with the rest. + +Cleaning these up is a separate decommissioning pass and was not part of the removal. diff --git a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml b/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml similarity index 99% rename from ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml rename to archive/uptime_kuma/deploy_uptime_kuma_playbook.yml index 4af3858..8754dce 100644 --- a/ansible/services/uptime_kuma/deploy_uptime_kuma_playbook.yml +++ b/archive/uptime_kuma/deploy_uptime_kuma_playbook.yml @@ -1,5 +1,5 @@ - name: Deploy Uptime Kuma with Docker Compose and configure Caddy reverse proxy - hosts: watchtower + hosts: monitoring become: yes vars_files: - ../../infra_vars.yml diff --git a/archive/uptime_kuma/monitors.json b/archive/uptime_kuma/monitors.json new file mode 100644 index 0000000..d1b0b0a --- /dev/null +++ b/archive/uptime_kuma/monitors.json @@ -0,0 +1,1202 @@ +[ + { + "id": 1, + "name": "services", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 3, + "name": "Vaultwarden", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://vault.contrapeso.xyz/alive", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 4, + "name": "prd-spacey - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 5, + "name": "prd-watchtower - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 6, + "name": "prd-vipy - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 7, + "name": "disk-usage-prd-vipy-root", + "type": "push", + "active": true, + "parent": 6, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-vipy (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 8, + "name": "disk-usage-prd-watchtower-root", + "type": "push", + "active": true, + "parent": 5, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-watchtower (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 9, + "name": "disk-usage-prd-spacey-root", + "type": "push", + "active": true, + "parent": 4, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-spacey (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 10, + "name": "system-healthcheck-prd-vipy", + "type": "push", + "active": true, + "parent": 6, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-vipy - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 11, + "name": "system-healthcheck-prd-spacey", + "type": "push", + "active": true, + "parent": 4, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-spacey - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 12, + "name": "system-healthcheck-prd-watchtower", + "type": "push", + "active": true, + "parent": 5, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-watchtower - Regular healthcheck ping every 60s", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 13, + "name": "Forgejo", + "type": "http", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": "https://forgejo.contrapeso.xyz/api/healthz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 10, + "resendInterval": 0 + }, + { + "id": 14, + "name": "Personal Blog", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://pablohere.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 15, + "name": "LNBits", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://wallet.contrapeso.xyz/api/v1/health", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 16, + "name": "ntfy-emergency-app", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://avisame.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 17, + "name": "Headscale", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://headscale.contrapeso.xyz/health", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 18, + "name": "knots-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 19, + "name": "disk-usage-knots-box-root", + "type": "push", + "active": true, + "parent": 18, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: knots-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 20, + "name": "system-healthcheck-knots-box", + "type": "push", + "active": true, + "parent": 18, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: knots-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 21, + "name": "nodito - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 22, + "name": "system-healthcheck-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 300, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: nodito - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 23, + "name": "cpu-temp-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "CPU Temperature: nodito - Alerts when temperature exceeds 80\u00b0C", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 24, + "name": "Bitcoin Knots", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 25, + "name": "fulcrum-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 26, + "name": "disk-usage-fulcrum-box-root", + "type": "push", + "active": true, + "parent": 25, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: fulcrum-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 27, + "name": "system-healthcheck-fulcrum-box", + "type": "push", + "active": true, + "parent": 25, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: fulcrum-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 28, + "name": "Fulcrum", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 35, + "name": "mempool-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 36, + "name": "disk-usage-mempool-box-root", + "type": "push", + "active": true, + "parent": 35, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: mempool-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 37, + "name": "system-healthcheck-mempool-box", + "type": "push", + "active": true, + "parent": 35, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: mempool-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 38, + "name": "Mempool MariaDB", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 39, + "name": "Mempool Backend", + "type": "push", + "active": true, + "parent": 1, + "interval": 180, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 40, + "name": "Mempool Frontend", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 41, + "name": "Mempool Public Access", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://mempool.contrapeso.xyz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 42, + "name": "memos-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 43, + "name": "disk-usage-memos-box-root", + "type": "push", + "active": true, + "parent": 42, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: memos-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 44, + "name": "system-healthcheck-memos-box", + "type": "push", + "active": true, + "parent": 42, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: memos-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 45, + "name": "Memos", + "type": "http", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": "https://memos.contrapeso.xyz/healthz", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 46, + "name": "Bitcoin Knots P2P Public", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 8333, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 47, + "name": "Fulcrum SSL Public", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 50002, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 49, + "name": "zfs-health-nodito", + "type": "push", + "active": true, + "parent": 21, + "interval": 90000, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "ZFS Pool Health: nodito - Daily health check for pool proxmox-tank-1", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 50, + "name": "UPS ONLINE", + "type": "push", + "active": true, + "parent": 21, + "interval": 90, + "retries": null, + "url": "https://", + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 51, + "name": "forgejo-runner-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 52, + "name": "disk-usage-forgejo-runner-box-root", + "type": "push", + "active": true, + "parent": 51, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: forgejo-runner-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 53, + "name": "system-healthcheck-forgejo-runner-box", + "type": "push", + "active": true, + "parent": 51, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: forgejo-runner-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 55, + "name": "forgejo-runner-healthcheck", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Forgejo Runner healthcheck - ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 59, + "name": "arbret-staging-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 60, + "name": "disk-usage-arbret-staging-box-root", + "type": "push", + "active": true, + "parent": 59, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: arbret-staging-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 61, + "name": "system-healthcheck-arbret-staging-box", + "type": "push", + "active": true, + "parent": 59, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: arbret-staging-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 66, + "name": "arbret - staging", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 67, + "name": "arbretstaging.contrapeso.xyz - health", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when GET /api/health returns status ok", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 68, + "name": "arbretstaging.contrapeso.xyz - https", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when HTTPS front-door returns 200", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 69, + "name": "arbretstaging.contrapeso.xyz - postgresql", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when postgresql.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 70, + "name": "arbretstaging.contrapeso.xyz - arbret-server", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-server.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 71, + "name": "arbretstaging.contrapeso.xyz - arbret-worker", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-worker.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 72, + "name": "arbretstaging.contrapeso.xyz - arbret-analytics", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 73, + "name": "arbretstaging.contrapeso.xyz - arbret-backup", + "type": "push", + "active": true, + "parent": 66, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 74, + "name": "prd-arbret - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 85, + "name": "disk-usage-prd-arbret-root", + "type": "push", + "active": true, + "parent": 74, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: prd-arbret (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 86, + "name": "system-healthcheck-prd-arbret", + "type": "push", + "active": true, + "parent": 74, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: prd-arbret - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 87, + "name": "arbret - production", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 88, + "name": "arbret.com - health", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when GET /api/health returns status ok", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 89, + "name": "arbret.com - https", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when HTTPS front-door returns 200", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 90, + "name": "arbret.com - postgresql", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when postgresql.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 91, + "name": "arbret.com - arbret-server", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-server.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 92, + "name": "arbret.com - arbret-worker", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when arbret-worker.service is active", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 93, + "name": "arbret.com - arbret-analytics", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 94, + "name": "arbret.com - arbret-backup", + "type": "push", + "active": true, + "parent": 87, + "interval": 120, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "Healthy when timer is scheduled and last run succeeded", + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 95, + "name": "small-backups-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 96, + "name": "disk-usage-small-backups-box-root", + "type": "push", + "active": true, + "parent": 95, + "interval": 960, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": true, + "description": "Disk Usage: small-backups-box (/) - Alerts when usage exceeds 80%", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 97, + "name": "system-healthcheck-small-backups-box", + "type": "push", + "active": true, + "parent": 95, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": "System Healthcheck: small-backups-box - Regular healthcheck ping every 60s", + "maxretries": 1, + "resendInterval": 0 + }, + { + "id": 109, + "name": "DATUM Gateway", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 111, + "name": "DATUM Stratum (public)", + "type": "port", + "active": true, + "parent": 1, + "interval": 60, + "retries": null, + "url": null, + "hostname": "167.172.107.33", + "port": 23334, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 112, + "name": "Phoenixd", + "type": "push", + "active": true, + "parent": 1, + "interval": 90, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 3, + "resendInterval": 0 + }, + { + "id": 113, + "name": "nonkeiwaisi-box - infra", + "type": "group", + "active": true, + "parent": null, + "interval": 60, + "retries": null, + "url": null, + "hostname": null, + "port": null, + "upsideDown": false, + "description": null, + "maxretries": 1, + "resendInterval": 0 + } +] diff --git a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml b/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml similarity index 99% rename from ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml rename to archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml index 9ae9713..54c0b79 100644 --- a/ansible/services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml +++ b/archive/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml @@ -1,5 +1,5 @@ - name: Configure local backup for Uptime Kuma from remote - hosts: lapy + hosts: control gather_facts: no vars_files: - ../../infra_vars.yml diff --git a/ansible/services/uptime_kuma/uptime_kuma_vars.yml b/archive/uptime_kuma/uptime_kuma_vars.yml similarity index 92% rename from ansible/services/uptime_kuma/uptime_kuma_vars.yml rename to archive/uptime_kuma/uptime_kuma_vars.yml index 3263f49..33ffb6d 100644 --- a/ansible/services/uptime_kuma/uptime_kuma_vars.yml +++ b/archive/uptime_kuma/uptime_kuma_vars.yml @@ -4,7 +4,7 @@ uptime_kuma_data_dir: "{{ uptime_kuma_dir }}/data" uptime_kuma_port: 3001 # Remote access -remote_host_name: "watchtower" +remote_host_name: "{{ groups['monitoring'] | first }}" remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" diff --git a/requirements.txt b/requirements.txt index 972dc4f..653523e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,7 @@ packaging==25.0 pycparser==2.22 PyYAML==6.0.2 resolvelib==1.0.1 +# Only needed by the deprecated Uptime Kuma monitoring blocks, which are kept for +# reference but never run (uptime_kuma_enabled: false). Drop this once they are +# rewired to a replacement. See archive/uptime_kuma/. uptime-kuma-api>=1.2.1