diff --git a/ansible/host_vars/vipy/main.yml b/ansible/host_vars/vipy/main.yml index 3b077df..93415da 100644 --- a/ansible/host_vars/vipy/main.yml +++ b/ansible/host_vars/vipy/main.yml @@ -13,3 +13,8 @@ monitored_services: - lnbits - caddy - phoenixd + # Deployed by the arbret repo, not this one (see arbret_handoff.md); listed + # here because "what runs on this host" is still a property of the host. + - arbret-server + - arbret-worker + - postgresql diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index 2dc4689..3ff5b23 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -35,6 +35,11 @@ - {sub: "{{ subdomains.monero_explorer }}", host: vipy} - {sub: "{{ subdomains.music }}", host: vipy} - {sub: "{{ subdomains.jar_lid_archive_staging }}", host: vipy} + # Not under root_domain, so named in full. arbret.com's zone is NOT in + # Porkbun (mynymdns) - this is the check that would have caught it still + # pointing at prd-arbret's recycled IP after the 2026-10-17 rebuild. + - {fqdn: arbret.com, host: vipy} + - {fqdn: www.arbret.com, host: vipy} # A public resolver on purpose: this must test what the internet sees, not # what a local cache or the tailnet's MagicDNS happens to answer. @@ -68,6 +73,13 @@ # tailnet hop, gunicorn, and the database and bucket behind it (503 if # either is down). "/" would only prove the login redirect works. - {name: jars-staging, sub: "{{ subdomains.jar_lid_archive_staging }}", path: "/health/", status: 200} + # arbret (runs on vipy since 2026-10-17; deployed by the arbret repo, not + # this one). /api/health answers 200 + "status":"ok" when healthy and 503 + # "degraded" otherwise, DB down included - per the arbret owner. The body + # condition guards against a 200 that is not actually healthy. + - {name: arbret, fqdn: arbret.com, path: "/", status: 200} + - {name: arbret-health, fqdn: arbret.com, path: "/api/health", status: 200, + conditions: ['[BODY].status == ok']} # Ports published from the edge host by socket_proxy. public_tcp: @@ -107,11 +119,11 @@ - name: Build the DNS endpoints ansible.builtin.set_fact: dns_endpoints: "{{ dns_endpoints | default([]) + [{ - 'name': item.sub ~ '.' ~ root_domain, + 'name': (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain), 'group': 'dns', 'url': dns_resolver, 'interval': item.interval | default('6h'), - 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, + 'dns': {'query-type': 'A', 'query-name': (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain)}, 'conditions': ['[DNS_RCODE] == NOERROR', '[BODY] == ' ~ hostvars[item.host].ansible_host], 'alerts': [{'type': 'signal', 'failure-threshold': 1, @@ -128,10 +140,11 @@ http_endpoints: "{{ http_endpoints | default([]) + [{ 'name': item.name, 'group': 'public', - 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, + 'url': 'https://' ~ (item.fqdn if item.fqdn is defined else item.sub ~ '.' ~ root_domain) ~ item.path, 'interval': item.interval | default('5m'), 'conditions': ['[STATUS] == ' ~ item.status, - '[CERTIFICATE_EXPIRATION] > 168h'], + '[CERTIFICATE_EXPIRATION] > 168h'] + + item.conditions | default([]), 'alerts': [{'type': 'signal', 'failure-threshold': item.threshold | default(3), 'success-threshold': 2, 'send-on-resolved': true, 'minimum-reminder-interval': '6h'}]}] }}" diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index a46682a..f8cbb91 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -59,6 +59,15 @@ - {name: xmrblocks, host: monerod_box_local} - {name: navidrome, host: music_box_local} - {name: jar-lid-archive-staging, host: jar_lid_staging_local} + # arbret (on vipy, deployed by the arbret repo). `host` here is the KEY in + # gatus_push_tokens, and arbret has a dedicated `arbret` token rather than + # vipy's - so it cannot write vipy's own results, and vice versa. + # Windows are two runs plus slack, as for the 5m probes above: + # analytics hourly at :00 -> 130m + # beancount daily at 03:00 -> 26h + # Cadence lives in the arbret repo; tell us if it changes. + - {name: arbret-analytics, host: arbret, heartbeat: '130m'} + - {name: arbret-beancount, host: arbret, heartbeat: '26h'} # The alert every probe gets. failure-threshold is 1 - the heartbeat window # is the tolerance, see the note above - unless a probe sets diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 875f10a..791d51d 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -12,8 +12,10 @@ backup_store_check_push_base: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" backup_store_check_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" backup_store_sources: + # On vipy since 2026-10-17 (prd-arbret is gone). The arbret repo writes + # the dump to the backup_source contract - see arbret_handoff.md. - name: arbret - source: "arbret@prd-arbret:/opt/arbret/backups/" + source: "backup-pull@prd-vipy:/opt/backups/arbret/" retention_days: 90 - name: headscale source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" @@ -52,8 +54,8 @@ # LATENCY and DIAGNOSIS: the store only learns at the next 04:00 pull, and it # cannot tell you whether the dump broke or the pull did. # -# arbret has no dump endpoint: prd-arbret lives in [arbret], which `managed` -# deliberately excludes, so nothing of ours runs there. It is store-checked only. +# arbret's dump is written by the arbret repo, not by backup_source, and it +# pushes with its own `arbret` token (the `host` below is the token key). # ───────────────────────────────────────────────────────────────────────────── - name: Register the backup checks with Gatus hosts: observability @@ -67,6 +69,7 @@ - {name: lnbits, host: vipy} - {name: forgejo, host: vipy} - {name: jar-lid-archive-staging, host: jar_lid_staging_local} + - {name: arbret, host: arbret} store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo, jar-lid-archive-staging] tasks: diff --git a/ansible/services/arbret/deploy_arbret_edge_playbook.yml b/ansible/services/arbret/deploy_arbret_edge_playbook.yml new file mode 100644 index 0000000..3449e95 --- /dev/null +++ b/ansible/services/arbret/deploy_arbret_edge_playbook.yml @@ -0,0 +1,59 @@ +--- +# arbret's footprint on the edge host - and ONLY that. +# +# arbret itself (app, worker, Postgres, timers, its backup dump) is deployed by +# the arbret owner's own Ansible repo since prd-arbret was merged into vipy on +# 2026-10-17. This playbook owns the parts of the shared host that are ours: +# the public front door, the tailnet port, and the backup directory contract. +# See arbret_handoff.md at the repo root for the division of labour. +- name: Publish arbret on the edge host + hosts: edge + become: yes + vars: + # Local port the arbret app binds on 127.0.0.1, per the arbret owner. + arbret_port: 3100 + arbret_domain: arbret.com + # Admin port reachable from the tailnet only (ACL in headscale's acl.json.j2). + arbret_tailnet_port: 4117 + + tasks: + - name: Publish arbret through Caddy + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: arbret + caddy_site_domain: "{{ arbret_domain }}" + caddy_site_upstream: "localhost:{{ arbret_port }}" + + - name: Redirect www to the apex + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: arbret-www + caddy_site_domain: "www.{{ arbret_domain }}" + caddy_site_body: | + redir https://{{ arbret_domain }}{uri} permanent + + # tailscale0 only. UFW's default-deny applies to the tailnet interface too, + # so the headscale ACL alone does not open it - and this must never be + # reachable on the public interface. + - name: Allow the arbret admin port from the tailnet + community.general.ufw: + rule: allow + interface: tailscale0 + direction: in + port: "{{ arbret_tailnet_port | string }}" + proto: tcp + comment: "arbret admin, tailnet only" + + # The backup contract directory (arbret_handoff.md section 3.3). The arbret + # repo creates it identically; creating it here too means small-backups-box + # can pull - an empty dir - before arbret's first dump, so one missing + # source does not turn the whole pull job red. + - name: Ensure the arbret backup directory exists + ansible.builtin.file: + path: /opt/backups/arbret + state: directory + owner: root + group: backup-pull + mode: "0750" diff --git a/ansible/services/headscale/templates/acl.json.j2 b/ansible/services/headscale/templates/acl.json.j2 index 2362a1f..710c7a8 100644 --- a/ansible/services/headscale/templates/acl.json.j2 +++ b/ansible/services/headscale/templates/acl.json.j2 @@ -26,7 +26,7 @@ "dst": [ "*:22", "tag:arbret-staging-box:4117", - "tag:prd-arbret:4117", + "tag:prd-vipy:4117", "tag:nodito:8006" ], "src": [ diff --git a/ansible/site.yml b/ansible/site.yml index 28e620e..7867397 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -65,6 +65,7 @@ - import_playbook: services/forgejo/deploy_forgejo_playbook.yml - import_playbook: services/lnbits/deploy_lnbits_playbook.yml - import_playbook: services/personal-blog/deploy_personal_blog_playbook.yml +- import_playbook: services/arbret/deploy_arbret_edge_playbook.yml # ── Backups: each source dumps itself, the box pulls ──────────────────────── - import_playbook: services/headscale/setup_backup_headscale.yml diff --git a/arbret_handoff.md b/arbret_handoff.md new file mode 100644 index 0000000..5503a50 --- /dev/null +++ b/arbret_handoff.md @@ -0,0 +1,456 @@ +# arbret on prd-vipy: monitoring and backups + +Handoff for the arbret owner. It covers what changed when prd-arbret was lost, and what +your Ansible repo needs to do so that arbret is **monitored by Gatus** and **backed up to +small-backups-box** from its new home. + +Written 2026-10-17, during the rebuild. + +--- + +## 0. What changed, in one screen + +| | Before | Now | +|---|---|---| +| Host | `prd-arbret`, a droplet of its own | **`prd-vipy`**, shared with the personal_infra services | +| Monitoring | Uptime Kuma (push URLs on `uptime.contrapeso.xyz`) | **Gatus** at `https://status.contrapeso.xyz`, running on `prd-spacey`. Uptime Kuma is gone for good. | +| Backup destination | small-backups-box pulls `arbret@prd-arbret:/opt/arbret/backups/` | small-backups-box pulls **`backup-pull@prd-vipy:/opt/backups/arbret/`** | +| TLS / front door | your own, on prd-arbret | **our Caddy on prd-vipy** (it owns 80/443) | + +Who does what: + +| Area | **You** (arbret repo) | **Us** (personal_infra repo) | +|---|---|---| +| App, workers, Postgres, analytics | deploy and run them | — | +| Public HTTPS for `arbret.com` | tell us your local port | Caddy vhost, certificates | +| Gatus endpoint definitions | — | all of them: Gatus config lives on prd-spacey, which you don't touch | +| Pulled checks (HTTPS, `/api/health`) | keep `/api/health` honest | define and run them | +| systemd unit checks (`arbret-server`, `arbret-worker`, `postgresql`) | keep the unit names stable | add them to vipy's monitored units | +| Host liveness and disk | — | already covered for prd-vipy | +| Analytics timer result | **push it** (§2.4) | register the endpoint | +| Backup dump | **produce it to the contract** (§3) | pull it, retain it 90 days, check freshness | +| Backup dump result | **push it** (§3.5) | register the endpoint | + +Your work comes down to three things: + +1. Live on a shared host without stepping on it (§1). +2. Push two results to Gatus: analytics and backup. +3. Write the dump to a new place, in a fixed shape. + +--- + +## 1. Living on a shared host + +prd-vipy also runs Vaultwarden, Forgejo, LNbits, phoenixd and a reverse proxy for a +dozen other services. Our Ansible converges it regularly. Anything below that your +playbooks change will be reverted by ours, or will break ours. + +### Owned by personal_infra: do not manage these + +- **Ports 80 and 443 and the Caddy install** (`/etc/caddy/**`). Don't install nginx, + certbot or a second Caddy, and don't bind 80/443. Give us a port and we write the vhost + (see below). +- **UFW and fail2ban.** The default policy is deny-incoming, with only SSH and 80/443 + open. Don't disable UFW or reset its rules. If you need a port reachable from outside, + ask us. +- **Users `counterweight` and `backup-pull`**, and their `authorized_keys`. In + particular **don't create, modify or authorise keys for `backup-pull`**: we create it + and it is shared by every backup on this host (§3). +- **`/opt/backups/`** (the parent directory, `root:root 0755`). You own exactly + `/opt/backups/arbret/` and nothing beside it. +- **Tailscale/Headscale membership** of the host, and the tailnet ACLs. +- **Docker.** It's installed for our services. You don't need it (arbret is native + systemd); if you start using it, don't touch `/etc/docker/daemon.json`. +- systemd unit names we use: `caddy`, `forgejo`, `lnbits`, `phoenixd`, + `*-backup.{service,timer}` for vaultwarden/lnbits/forgejo, `liveness-healthcheck`, + `disk-usage-healthcheck`, `service-health-healthcheck`. Prefix all of yours with + `arbret-` and there's no risk of collision. + +### Free for you + +- PostgreSQL. We run no Postgres on this host, so the system `postgresql` package and + cluster are yours. +- `/opt/arbret/`, `/etc/arbret/`, the `arbret` system user, `arbret-*` units and timers. + +### Local ports already taken + +`22, 80, 443, 3000, 7657, 8222, 8333, 8765, 9740, 23334, 50002`. Bind your app to +`127.0.0.1:` on anything else and tell us the port. Our Caddy then serves +`arbret.com` with a Let's Encrypt certificate and reverse-proxies to it, setting +`X-Forwarded-For`, `X-Forwarded-Proto` and `X-Forwarded-Host`. Configure the app to +trust proxy headers from `127.0.0.1` only. + +Port `4117` used to be reachable from one admin laptop over the tailnet. If that's still +needed, say so and we'll add the ACL for prd-vipy. Don't open it in UFW yourself. + +### The host itself + +Debian 13, clock in **UTC**, tailnet name `prd-vipy`. + +--- + +## 2. Monitoring with Gatus + +### 2.1 How it works here + +Gatus runs on prd-spacey and answers at `https://status.contrapeso.xyz`. Alerts go to +Signal. There are two kinds of check: + +- **Pulled.** Gatus makes the request itself (HTTP, TCP, DNS) every N seconds and + evaluates conditions. Nothing is installed on your side. This is the right kind for + anything with a reachable URL. +- **Pushed ("external endpoints").** Something on the host `POST`s a result to Gatus, + and Gatus alarms if a result is `false` **or if nothing arrives within the heartbeat + window**. This is the right kind for things with no URL: timers, jobs, internal state. + The heartbeat is what makes a dead timer visible: silence is a failure. + +Everything about an endpoint (name, group, token, heartbeat, alert rules) is defined in +**our** repo, because Gatus's config files live on prd-spacey. Your side of a push check +is just an HTTP call. + +### 2.2 The push API + +``` +POST https://status.contrapeso.xyz/api/v1/endpoints/{KEY}/external?success={true|false}&error={message} +Authorization: Bearer {TOKEN} +``` + +- **`KEY`** is `sanitize(group) + "_" + sanitize(name)`, where sanitize lowercases and + turns any of `/ _ . , space # + &` into `-`. Use the exact keys in the table below. +- **`success`**: `true` or `false`. +- **`error`**: free text, URL-encoded. It's shown on the dashboard and in the Signal + alert, so on success it's fine to send something useful like `ok 31M`. +- **`TOKEN`**: a bearer token we give you (§4). It's dedicated to arbret: it can only + write arbret's endpoints, and nobody else's token can write yours. + +Responses: + +| HTTP | Meaning | +|---|---| +| `200` | recorded | +| `401` | wrong token for that key | +| `404` | key not registered (yet). We register before you deploy. | + +Test it by hand once you have the token: + +```bash +curl -i -X POST -H "Authorization: Bearer $TOKEN" \ + "https://status.contrapeso.xyz/api/v1/endpoints/probe_arbret-analytics/external?success=true&error=manual%20test" +``` + +### 2.3 Your old Uptime Kuma monitors, mapped + +| Old monitor (prod) | New check | Gatus key | Kind | Who | +|---|---|---|---|---| +| `https`: front door returns 200 | `GET https://arbret.com/` → 200 | `public_arbret` | pulled, 5 min | us | +| `health`: `/api/health` returns ok | `GET https://arbret.com/api/health` → 200 and `[BODY].status == ok` | `public_arbret-health` | pulled, 5 min | us | +| `arbret-server`: unit active | systemd unit check on prd-vipy | `services_vipy-arbret-server` | push, every 5 min, by our host script | us | +| `arbret-worker`: unit active | same | `services_vipy-arbret-worker` | same | us | +| `postgresql`: unit active | same | `services_vipy-postgresql` | same | us | +| `arbret-analytics`: timer ran OK | your analytics run reports its result | `probe_arbret-analytics` | **push, by you**, hourly; heartbeat 130m | **you** (§2.4) | +| (new) beancount: daily job ran OK | your beancount run reports its result | `probe_arbret-beancount` | **push, by you**, daily 03:00; heartbeat 26h | **you** (§2.4, same hook) | +| `arbret-backup`: timer ran OK | your backup run reports its result | `backup-dump_arbret` | **push, by you** | **you** (§3.5) | +| prd-arbret liveness / disk | covered by prd-vipy's own host checks | `liveness_vipy`, `disk_vipy` | push | us | + +We also check: + +- `backup-store_arbret`: the backup **arrived** on small-backups-box, is fresh (<26h, + read from the filename timestamp), non-empty, and not suspiciously smaller than the + previous one. +- `arbret.com` resolves to prd-vipy, checked every 6 hours. +- `arbret.com` registration expiry, alarming 14 days ahead. This already existed. + +**Delete, don't port, every Uptime Kuma push script, timer and URL in your repo.** The +endpoint no longer exists. A leftover timer will just fail silently every two minutes +forever. + +The pulled health check relies on `/api/health` returning HTTP 200 with a JSON body that +has `"status": "ok"` when healthy, and **a non-200 when it isn't**. If the body differs, +tell us the exact shape. A health endpoint that checks the database is worth more than +one that only proves the process answers. + +### 2.4 What you implement: the analytics push + +The cleanest hook is `ExecStopPost=` on the oneshot service the timer runs. systemd runs +it after every run, success or failure, and sets `$SERVICE_RESULT` and `$EXIT_STATUS` +for it. So no wrapper script is needed, and the run itself stays untouched. + +`/etc/arbret/gatus.env` (`root:root 0600`): + +``` +GATUS_PUSH_BASE=https://status.contrapeso.xyz/api/v1/endpoints +GATUS_PUSH_TOKEN= +``` + +`/usr/local/bin/arbret-gatus-push` (`root:root 0755`): + +```bash +#!/usr/bin/env bash +# Usage: arbret-gatus-push +# Meant for ExecStopPost=, where systemd provides SERVICE_RESULT / EXIT_STATUS. +# Can also be called by hand: SERVICE_RESULT=success arbret-gatus-push +set -uo pipefail +KEY="$1" +: "${GATUS_PUSH_BASE:?}" "${GATUS_PUSH_TOKEN:?}" + +if [ "${SERVICE_RESULT:-}" = "success" ]; then + success=true; msg="ok" +else + success=false; msg="${SERVICE_RESULT:-unknown} exit=${EXIT_STATUS:-?}" +fi +enc=$(printf '%s' "$msg" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g; s/=/%3D/g') + +# Never let a monitoring hiccup fail the job itself. +curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${GATUS_PUSH_TOKEN}" \ + "${GATUS_PUSH_BASE}/${KEY}/external?success=${success}&error=${enc}" || true +exit 0 +``` + +In the analytics service unit: + +```ini +[Service] +Type=oneshot +EnvironmentFile=/etc/arbret/gatus.env +ExecStart=...your existing command... +ExecStopPost=/usr/local/bin/arbret-gatus-push probe_arbret-analytics +``` + +Two things to get right: + +- `EnvironmentFile=` puts the token in the unit's environment, **not** in the unit file, + so the unit file can stay world-readable while the env file is `0600`. +- **Tell us the timer's schedule.** We set the heartbeat to the interval plus slack: a + 5-minute job gets an 11-minute window, a daily job gets 26 hours. If the schedule + changes later, tell us, or Gatus will either alarm on a healthy job or sleep through a + dead one. + +As Ansible, this is two templated files, an `EnvironmentFile=` line and an +`ExecStopPost=` line. It's small enough to live in the role that deploys the analytics +timer, rather than becoming a role of its own. + +### 2.5 Staging (arbret-staging-box) + +arbret-staging-box is a VM in our inventory, so its liveness and disk checks already +exist. The same mapping applies with these keys: + +| Check | Key | Who | +|---|---|---| +| front door / health | `public_arbret-staging`, `public_arbret-staging-health` | us | +| units | `services_arbret-staging-local-arbret-server` (and `-worker`, `-postgresql`) | us | +| analytics | `probe_arbret-staging-analytics` | you, with the **staging** token | +| backup dump | `backup-dump_arbret-staging` | you, with the staging token. Only if staging should be backed up; tell us. | + +The staging vhost (`arbretstaging.contrapeso.xyz`) lived on the old prd-vipy and was not +in either repo. We're recreating it on our side. + +--- + +## 3. Backups + +### 3.1 The pipeline + +``` + 02:00 UTC prd-vipy your timer: pg_dump | age -r -> /opt/backups/arbret/arbret_.dump.age + your script: POST backup-dump_arbret (success/failure) + 04:00 UTC small-backups-box rsync backup-pull@prd-vipy:/opt/backups/arbret/ -> ~/backups/arbret/ (keeps 90 days) + 05:30 UTC small-backups-box check-backups.sh -> POST backup-store_arbret (then every 6 h) +``` + +- **The box only pulls.** Nothing on prd-vipy can reach into the backup box. The pull + logs in as `backup-pull`, an unprivileged account with a `restrict`ed key that can + read the backup directories and nothing else. +- **The source encrypts.** Only the age **public** key is on prd-vipy, so the host can't + decrypt its own backups. The private key lives offline with the infra owner. Plaintext + never touches disk or the network. +- **Retention is two-tier.** Keep about 7 days locally (the disk is shared). The box + keeps 90. + +Your existing dump role already produces the right **artefact**: a Postgres custom-format +dump, age-encrypted, named `arbret_YYYYmmdd_HHMMSS.dump.age`. The last pre-incident one +is `arbret_20261016_020001.dump.age`. Only its **location, ownership and reporting** +change. + +### 3.2 What changes from prd-arbret + +| | prd-arbret | prd-vipy | +|---|---|---| +| Directory | `/opt/arbret/backups/` | **`/opt/backups/arbret/`** | +| Account the box pulls as | `arbret` (your user, with our key authorised) | **`backup-pull`** (ours, already exists) | +| SSH key for the pull | you authorised it | **nothing to do.** Remove that task, and don't authorise anything for `arbret` or `backup-pull`. | +| Result reporting | Uptime Kuma push | Gatus push to `backup-dump_arbret` | + +### 3.3 The contract + +The checks on the box enforce these, and some failures are silent, so read every line. + +1. **Directory:** `/opt/backups/arbret/`, owner `root`, group `backup-pull`, mode `0750`. + Don't change `/opt/backups/` itself (`root:root 0755`, ours). The `backup-pull` group + exists before your playbook runs; if it doesn't, fail loudly rather than creating it. +2. **Filename:** `arbret_YYYYmmdd_HHMMSS.dump.age`, timestamp in the host's local time + (UTC). The prefix must be exactly `arbret_`, because the box's prune and freshness + checks glob on `arbret_*`, and it reads the **dump time from the filename**. +3. **Encryption:** `age -r ` in the pipe (§4). Nothing unencrypted is ever + written to disk. +4. **Atomicity:** write `.partial`, set ownership and mode, then `mv` it to the + final name. The pull skips `*.partial`. A half-written file under the final name + would be pulled and kept for 90 days as if it were a backup. +5. **File ownership:** `root:backup-pull`, mode `0640`, on both the `.partial` and the + final file. If the pull account can't read a file, **rsync fails for the whole + directory**, so one bad file blocks every later night. +6. **`set -euo pipefail`.** Without `pipefail`, a failing `pg_dump` piped into a + succeeding `age` exits 0 and produces a tiny, valid-looking, useless artefact. +7. **Timing:** finished before 04:00 UTC. 02:00 is good. +8. **Local prune:** delete `arbret_*.dump.age` older than ~7 days, plus stale + `arbret_*.partial` files at the start of each run. Put nothing else in the directory: + no subdirectories, logs or checksums. +9. **Report** the outcome to Gatus on every exit path (§3.5). +10. **No service stop is needed.** `pg_dump` takes a consistent snapshot online. Don't + stop `arbret-server` for it. + +### 3.4 Reference script + +This is the same shape as the script every other service on the host uses (our +`backup_source` role), adapted to `pg_dump`. Adjust the database name and the +`pg_dump` invocation to match your current role. + +`/usr/local/bin/arbret-backup.sh` (`root:root 0750`): + +```bash +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +BACKUP_DIR=/opt/backups/arbret +RECIPIENT="age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" +RETENTION_DAYS=7 +NAME=arbret +DB=arbret # <- your database name +ARTIFACT="${BACKUP_DIR}/${NAME}_$(date +%Y%m%d_%H%M%S).dump.age" + +PUSH_URL="${GATUS_PUSH_BASE:-}/backup-dump_arbret/external" +PUSH_TOKEN="${GATUS_PUSH_TOKEN:-}" + +report() { # $1=true|false $2=message + [ -n "${GATUS_PUSH_BASE:-}" ] || return 0 + local enc; enc=$(printf '%s' "$2" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + curl -s -o /dev/null --max-time 15 --retry 2 --retry-delay 3 -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" "${PUSH_URL}?success=$1&error=${enc}" || true +} +on_exit() { # the ONE exit trap: bash keeps only one, a second replaces it + local rc=$? + if [ "$rc" -ne 0 ]; then report false "arbret dump exited ${rc}" + elif [ ! -s "$ARTIFACT" ]; then report false "arbret produced no artefact" + else report true "arbret $(du -h "$ARTIFACT" | cut -f1)"; fi +} +trap on_exit EXIT + +command -v age >/dev/null || { echo "age not installed" >&2; exit 1; } +getent group backup-pull >/dev/null || { echo "group backup-pull missing - personal_infra has not provisioned this host" >&2; exit 1; } + +install -d -o root -g backup-pull -m 0750 "$BACKUP_DIR" +rm -f "${BACKUP_DIR}/${NAME}_"*.partial + +runuser -u postgres -- pg_dump -Fc "$DB" | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" +chown root:backup-pull "${ARTIFACT}.partial"; chmod 0640 "${ARTIFACT}.partial" +mv "${ARTIFACT}.partial" "$ARTIFACT" + +find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.dump.age" -mtime +"$RETENTION_DAYS" -delete +echo "wrote $ARTIFACT" +``` + +Units: + +```ini +# /etc/systemd/system/arbret-backup.service (0644; the token is in the env file, not here) +[Unit] +Description=arbret database backup +After=postgresql.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/arbret/gatus.env +ExecStart=/usr/local/bin/arbret-backup.sh + +# /etc/systemd/system/arbret-backup.timer +[Unit] +Description=arbret database backup, nightly + +[Timer] +OnCalendar=*-*-* 02:00:00 +Persistent=true + +[Install] +WantedBy=timers.target +``` + +`age` and `curl` come from Debian (`apt install age curl`). Installing them from your +playbook is fine, since both repos asking for the same package doesn't conflict. + +### 3.5 Reporting the backup + +The script above already reports, on every exit path, to `backup-dump_arbret`, using the +same `/etc/arbret/gatus.env` as the analytics push. Its heartbeat is 26 hours: one +missed night alarms. The box-side check (`backup-store_arbret`) independently notices +within 6 hours of the next pull if the artefact stops arriving or stops being fresh. +So a broken dump and a broken pull show up as **different** red lights. + +### 3.6 Verify after your first deploy + +```bash +sudo systemctl start arbret-backup.service +sudo journalctl -u arbret-backup.service -n 20 --no-pager +ls -l /opt/backups/arbret/ # root backup-pull, -rw-r----- +sudo -u backup-pull head -c 30 /opt/backups/arbret/arbret_*.dump.age # must print "age-encryption.org/v1" +systemctl list-timers arbret-backup.timer +``` + +Then check that `backup-dump_arbret` is green on the status page, and tell us. We'll +trigger a pull rather than wait for 04:00, and confirm `backup-store_arbret`. + +### 3.7 Restoring + +Only the infra owner can decrypt: the age identity is offline. They'll hand you a +decrypted dump over a secure channel. The latest pre-incident one is +`arbret_20261016_020001.dump.age` (31.9 MB, sha256 `2bb771957513654cb907e5d63153e7714d54c7d5695d7caf09a05f948dcf515d`). +Anything written to arbret after 2026-10-16 02:00 UTC is not in any backup. + +For a custom-format dump: + +```bash +sudo -u postgres createdb -O arbret arbret # or let --create do it +sudo -u postgres pg_restore -d arbret --no-owner --role=arbret -j 4 arbret_20261016_020001.dump +``` + +Create the roles the app expects first. `pg_dump -Fc` doesn't include them. Then shred +the plaintext dump: `shred -u arbret_20261016_020001.dump`. + +--- + +## 4. What you'll receive from us + +| Item | Secret | Used for | +|---|---|---| +| Gatus push token, **prod** | yes | `/etc/arbret/gatus.env` on prd-vipy. Keep it in your vault. | +| Gatus push token, **staging** | yes | the same file on arbret-staging-box, if you do the staging pushes | +| age recipient `age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza` | **no**, it's a public key | the backup script. Safe to commit in plain text. | +| SSH access to prd-vipy for your playbooks | access | being arranged separately | +| The decrypted 2026-10-16 dump | yes, it's the full prod DB | restore, handed over once | + +You need **no** SSH key for the backup pull, and nothing from the old Uptime Kuma setup. + +## 5. What we need from you + +1. The local port arbret listens on (for the `arbret.com` vhost), and whether + `www.arbret.com` should be served or redirected. +2. The exact `/api/health` response when healthy, and confirmation that it returns a + non-200 when the DB is down. +3. The analytics timer's schedule (sets the heartbeat). +4. Whether port 4117 over the tailnet is still needed. +5. The final unit names, if not `arbret-server`, `arbret-worker`, `postgresql`. +6. Whether staging should be backed up, and whether staging pushes are in scope. +7. Who besides the infra owner should get Signal alerts for arbret. + +Once we have 1 to 3, we register the endpoints, the vhost and the pull source. Then you +can deploy and see green on the first run.