diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 0be0162..36d35f8 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -7,10 +7,3 @@ root_domain: contrapeso.xyz # playbooks are kept deliberately — the check logic is meant to be rewired to # whatever replaces it. This flag keeps them inert until then. See archive/uptime_kuma/. uptime_kuma_enabled: false - -# age recipient for all backup artefacts -age_backup_recipient: "age192wwdaseqej2ggwyp884gtm05c396anp7chr0vr8m47g50fahpyqr9fsza" - -# Public key small-backups-box pulls with -# Authorised on each source host for an unprivileged, dedicated user only -backup_pull_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOfIixKMhA9z+Nvyx6ToZIniC8aEgyiInRiboaTTemgX offsite-backup-pull" diff --git a/ansible/group_vars/nodito_vms.yml b/ansible/group_vars/nodito_vms.yml deleted file mode 100644 index 1072b51..0000000 --- a/ansible/group_vars/nodito_vms.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Reach the VMs over Tailscale, and fall back to the LAN if the tailnet is down. -# -# ansible_host is a MagicDNS name. If tailscaled is not running on the control -# node that name does not resolve, the probe fails, and the LAN address recorded -# as lan_ip in inventory.ini takes over. -# -# Why the probe-then-connect shape rather than a plain `nc -w5 %h %p`: -# netcat-openbsd's -w is an IDLE timeout as well as a connect timeout, so a -# single `nc -w5` silently tears down the SSH session after five quiet seconds. -# That produced intermittent "Data could not be sent to remote host" failures on -# exactly the long, quiet operations (apt) where a dropped connection costs most. -# `nc -z` probes, then `exec nc` carries the session with no timeout at all. -# -# Safe against the LAN addresses drifting again (which is how fulcrum/mempool -# came to be transposed): known_hosts is keyed to the MagicDNS NAME, so if -# lan_ip ever points at a different machine the host key will not match and ssh -# aborts. Verified by pointing fulcrum-box at mempool-box's address: -# "Host key verification failed." -# -# lan_ip is a convenience, not an identity. If it goes stale the fallback stops -# working; it will never connect you to the wrong box. -ansible_ssh_common_args: >- - -o ProxyCommand="sh -c 'nc -z -w5 %h %p 2>/dev/null && exec nc %h %p || exec nc {{ lan_ip }} %p'" diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml deleted file mode 100644 index 0686ab6..0000000 --- a/ansible/playbooks/backups.yml +++ /dev/null @@ -1,28 +0,0 @@ -- name: Configure the offsite backup pull - hosts: backup_store - gather_facts: yes - - tasks: - - name: Ensure the box pulls every source on a timer - ansible.builtin.include_role: - name: backup_store - vars: - backup_store_sources: - - name: arbret - source: "arbret@prd-arbret:/opt/arbret/backups/" - retention_days: 90 - - name: headscale - source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" - retention_days: 90 - - name: memos - source: "backup-pull@memos-box:/opt/backups/memos/" - retention_days: 90 - - name: vaultwarden - source: "backup-pull@prd-vipy:/opt/backups/vaultwarden/" - retention_days: 90 - - name: lnbits - source: "backup-pull@prd-vipy:/opt/backups/lnbits/" - retention_days: 90 - - name: forgejo - source: "backup-pull@prd-vipy:/opt/backups/forgejo/" - retention_days: 14 diff --git a/ansible/roles/backup_source/README.md b/ansible/roles/backup_source/README.md deleted file mode 100644 index a7794ab..0000000 --- a/ansible/roles/backup_source/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# `backup_source` - -Makes a host back **itself** up: dump to stdout, encrypt with `age`, write to a -local directory, prune, on a systemd timer. `small-backups-box` pulls the -directory later (see `backup_store`). - -Modelled on `prd-arbret`, which has been doing exactly this correctly since -before the rest of the estate was migrated. - -## Usage - -```yaml -- ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: headscale - backup_source_description: "Headscale" - backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" - backup_source_stop_service: headscale - backup_source_retention_days: 7 -``` - -Produces `/opt/backups/headscale/headscale_.tar.gz.age`, -`headscale-backup.{service,timer}`, and `/usr/local/bin/headscale-backup.sh`. - -## Why the source encrypts, not the destination - -`age -r ` is asymmetric and the host holds only the **public** key, so -a compromised host cannot read its own backups — or anyone else's. The scripts -this replaces encrypted with GPG *on the laptop, after the data had already -crossed the network*, which protects the artefact at rest but not in transit. - -The matching identity lives only on lapy and is escrowed. **Lose it and every -artefact everywhere becomes noise**, including arbret's. - -## `backup_source_dump_command` writes to STDOUT - -The role pipes it into `age`, so plaintext never touches the disk. Use `-C /` -with relative paths in `tar` rather than absolute ones: it avoids tar's "removing -leading /" and makes the restore target explicit. - -## Services that are not systemd - -`backup_source_stop_service` runs `systemctl stop/start`. For anything else, -give the pair explicitly — vaultwarden is a docker compose stack, so -`systemctl stop vaultwarden` silently does nothing: - -```yaml -backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" -backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" -``` - -The same EXIT trap wraps both forms. The assert refuses a stop command without a -matching start command, because that combination fails in the one way you would -not notice: the service stops and never comes back. - -## More than one thing to back up - -`tar` takes several paths, so multiple files or directories are normally **one** -artefact — headscale captures `/var/lib/headscale` and `/etc/headscale` together, -lnbits captures its data directory and its `.env`. - -Prefer one artefact. A backup should be a consistent snapshot, and two artefacts -written by two runs can drift — you can end up restoring an `.env` that does not -match the database it configures. Pulling a single file back out needs no -unpacking: - -```bash -age -d -i | tar -xzO opt/lnbits/lnbits/.env -``` - -If you genuinely need separate artefacts, call the role twice with different -`backup_source_name`s rather than extending it — but only one call may set -`backup_source_stop_service`, or the service is stopped twice per night. - -The case this shape cannot express is a **database dump plus a file tree** -(`pg_dump` and a media directory, say): you cannot merge those into one stream -without staging plaintext on disk, which is exactly what this design avoids. -None of the current services need it — all are file trees, all stopped for the -dump. A future one that does should use two role calls. - -## Everything here is sqlite, so everything stops - -All five services are sqlite-backed, several in WAL mode (`-wal`/`-shm` files -present). A live copy of a WAL-mode database can be torn or stale, so each is -stopped for the duration. Measured downtime: under a second for headscale and -memos, ~6 s vaultwarden, ~11 s lnbits, and **2m36s for forgejo** — 2.7 G of repos -and database. That last one is the real cost of a consistent snapshot; if it -becomes unacceptable the answer is `sqlite3 .backup` plus an online repo copy, -not skipping the stop. - -## The trap is the reason this role exists - -When `backup_source_stop_service` is set, the script stops the unit and installs -an EXIT trap that starts it again. Without it, a failed dump leaves the service -down until the next timer fires — **every hand-written script this replaced had -that bug**, and it was only ever masked because their `systemctl stop` failed -first, before anything was stopped. - -Verified on spacey: with the dump forced to fail, the log shows -`Stopping → Writing → Restarting`, the script exits 1 (so systemd marks the unit -failed rather than hiding it), and headscale is `active` afterwards. - -If `systemctl stop` itself fails, `set -e` exits *before* the trap is installed — -which is correct, because nothing was stopped. - -## `.partial` - -The dump writes `.partial` and only `mv`s it into place on success, so -a truncated file is never mistaken for a backup. A failure inside the pipeline -does leave one behind, and the prune glob cannot match it (it ends `.partial`, -not `.tar.gz.age`), so the script clears stale partials at the **start** of each -run. Tested by failing mid-pipeline: 1 partial left, 0 after the next run. - -## `backup_source_stop_service` may be a bare name - -`headscale` and `headscale.service` both work. The unit template normalises it, -because systemd rejects a bare name in `After=` with -`Failed to add dependency ... Invalid argument` — which it logs and then ignores, -so the unit appears to work while carrying no ordering at all. - -## Retention is two-tier - -`backup_source_retention_days` is **local** and short — these hosts are -disk-constrained. The long tail lives on `small-backups-box`, which decides its -own retention per source. Losing the local copy is expected and fine. diff --git a/ansible/roles/backup_source/defaults/main.yml b/ansible/roles/backup_source/defaults/main.yml deleted file mode 100644 index 7de6006..0000000 --- a/ansible/roles/backup_source/defaults/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# Required -backup_source_name: "" # "headscale" -> headscale_.tar.gz.age -backup_source_description: "" # "Headscale" -backup_source_dump_command: "" # must write the payload to STDOUT - -# Placement -backup_source_dir: "/opt/backups/{{ backup_source_name }}" -backup_source_artifact_suffix: "tar.gz.age" - -# Encryption. Asymmetric: the host holds only the public key and cannot decrypt -# what it produces. -backup_source_recipient: "{{ age_backup_recipient }}" - -# The unprivileged account small-backups-box pulls as. It owns the dump -# directory and nothing else; it deliberately has no sudo. -backup_source_pull_user: backup-pull -backup_source_pull_key: "{{ backup_pull_public_key }}" - -# Safety. Give either a systemd unit, or an explicit pair of commands for -# services that are not systemd-managed (vaultwarden is a docker compose stack). -# Whichever is used, a trap guarantees the restart. -backup_source_stop_service: "" # systemd unit stopped for the dump -backup_source_stop_command: "" # overrides stop_service when set -backup_source_start_command: "" # required alongside stop_command - -# Retention here is LOCAL and short; small-backups-box keeps the long tail. -backup_source_retention_days: 7 - -# Schedule. The box pulls at 04:00, so dumps must land before that. -backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/roles/backup_source/handlers/main.yml b/ansible/roles/backup_source/handlers/main.yml deleted file mode 100644 index 37a7f2b..0000000 --- a/ansible/roles/backup_source/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: Reload systemd for backup units - ansible.builtin.systemd: - daemon_reload: yes diff --git a/ansible/roles/backup_source/tasks/main.yml b/ansible/roles/backup_source/tasks/main.yml deleted file mode 100644 index 4283b84..0000000 --- a/ansible/roles/backup_source/tasks/main.yml +++ /dev/null @@ -1,91 +0,0 @@ ---- -- name: Assert backup_source parameters are sane - ansible.builtin.assert: - that: - - backup_source_name | length > 0 - - backup_source_description | length > 0 - - backup_source_dump_command | length > 0 - - backup_source_recipient | length > 0 - - backup_source_recipient is match('^age1[0-9a-z]{58}$') - - not (backup_source_stop_command | length > 0 and backup_source_start_command | length == 0) - fail_msg: >- - backup_source: '{{ backup_source_name | default("") }}' needs a name, - description, dump command and a valid age recipient (age1... 62 chars). - backup_source_stop_command must be paired with backup_source_start_command. - quiet: true - -# Declared here rather than assumed. Stage 1 installed it by hand; this is what -# makes a rebuilt host get it too. -- name: Ensure age is installed - ansible.builtin.apt: - name: age - state: present - update_cache: yes - cache_valid_time: 3600 - -# The pull account: unprivileged, no sudo, exists only so small-backups-box can -# read the dump directory. Trust points one way — the box can read backups, and -# can do nothing else on this host. -- name: "Ensure the {{ backup_source_pull_user }} account exists" - ansible.builtin.user: - name: "{{ backup_source_pull_user }}" - system: yes - shell: /bin/sh # rsync-over-ssh needs a shell; nologin breaks it - home: "/var/lib/{{ backup_source_pull_user }}" - create_home: yes - password: '!' # no password login, ever - when: backup_source_pull_user | length > 0 - -- name: "Authorise the backup box's key for {{ backup_source_pull_user }}" - ansible.posix.authorized_key: - user: "{{ backup_source_pull_user }}" - key: "{{ backup_source_pull_key }}" - key_options: "restrict" # no pty, no forwarding, no user rc - exclusive: yes - state: present - when: backup_source_pull_user | length > 0 - -# The shared container above the per-service directories. It must be traversable -# or the pull account cannot reach its own directory. The script's `mkdir -p` -# runs under `umask 077` and would otherwise create this 0700. -- name: "Ensure {{ backup_source_dir | dirname }} is traversable" - ansible.builtin.file: - path: "{{ backup_source_dir | dirname }}" - state: directory - owner: root - group: root - mode: '0755' - -- name: "Ensure {{ backup_source_dir }} exists" - ansible.builtin.file: - path: "{{ backup_source_dir }}" - state: directory - owner: root - group: "{{ backup_source_pull_user | default('root', true) }}" - mode: '0750' - -- name: "Install the {{ backup_source_name }} backup script" - ansible.builtin.template: - src: backup.sh.j2 - dest: "/usr/local/bin/{{ backup_source_name }}-backup.sh" - owner: root - group: root - mode: '0750' - validate: "bash -n %s" - -- name: "Install the {{ backup_source_name }}-backup systemd units" - ansible.builtin.template: - src: "backup.{{ item }}.j2" - dest: "/etc/systemd/system/{{ backup_source_name }}-backup.{{ item }}" - owner: root - group: root - mode: '0644' - loop: [service, timer] - notify: Reload systemd for backup units - -- name: "Enable the {{ backup_source_name }}-backup timer" - ansible.builtin.systemd: - name: "{{ backup_source_name }}-backup.timer" - enabled: yes - state: started - daemon_reload: yes diff --git a/ansible/roles/backup_source/templates/backup.service.j2 b/ansible/roles/backup_source/templates/backup.service.j2 deleted file mode 100644 index 91ad063..0000000 --- a/ansible/roles/backup_source/templates/backup.service.j2 +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description={{ backup_source_description }} backup -{% if backup_source_stop_service %} -{# systemd rejects a bare name here ("Failed to add dependency ... Invalid - argument"), so normalise to a full unit name. #} -After={{ backup_source_stop_service if '.' in backup_source_stop_service else backup_source_stop_service ~ '.service' }} -{% endif %} - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/{{ backup_source_name }}-backup.sh -StandardOutput=journal -StandardError=journal -SyslogIdentifier={{ backup_source_name }}-backup diff --git a/ansible/roles/backup_source/templates/backup.sh.j2 b/ansible/roles/backup_source/templates/backup.sh.j2 deleted file mode 100644 index 18fabd6..0000000 --- a/ansible/roles/backup_source/templates/backup.sh.j2 +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# {{ backup_source_description }} backup — managed by Ansible (roles/backup_source) -# -# Dumps to stdout, encrypts with age, writes {{ backup_source_dir }}. -# The host holds only the age PUBLIC key, so it cannot read its own backups. -set -euo pipefail -umask 077 - -BACKUP_DIR="{{ backup_source_dir }}" -RETENTION_DAYS={{ backup_source_retention_days }} -RECIPIENT="{{ backup_source_recipient }}" -SUFFIX="{{ backup_source_artifact_suffix }}" -NAME="{{ backup_source_name }}" -{% if backup_source_stop_service or backup_source_stop_command %} -STOP_CMD={{ (backup_source_stop_command or ('systemctl stop ' ~ backup_source_stop_service)) | quote }} -START_CMD={{ (backup_source_start_command or ('systemctl start ' ~ backup_source_stop_service)) | quote }} -SERVICE="{{ backup_source_stop_service or backup_source_description }}" # label for the log only -{% endif %} - -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -ARTIFACT="${BACKUP_DIR}/${NAME}_${TIMESTAMP}.${SUFFIX}" - -die() { echo "FATAL: $*" >&2; exit 1; } -log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } - -# --- Pre-flight --- -[[ -n "$RECIPIENT" ]] || die "no age recipient configured" -command -v age >/dev/null || die "age is not installed" - -# Mode must agree with what the role sets, or each undoes the other every run. -mkdir -p "$BACKUP_DIR" -{% if backup_source_pull_user %} -chown root:{{ backup_source_pull_user }} "$BACKUP_DIR" -chmod 750 "$BACKUP_DIR" -{% else %} -chmod 700 "$BACKUP_DIR" -{% endif %} - -# A run that died mid-dump leaves a .partial. It is not a backup, and the prune -# glob below cannot match it (it ends .partial, not .${SUFFIX}), so clear them -# here or they accumulate forever. -rm -f "${BACKUP_DIR}/${NAME}_"*.partial - -{% if backup_source_stop_service or backup_source_stop_command %} -# --- Stop the service, and guarantee it comes back --- -# The trap is the point: without it a failed dump leaves the service down until -# the next timer fires. Every hand-written script this replaced had that bug. -log "Stopping ${SERVICE}..." -eval "$STOP_CMD" -trap 'log "Restarting ${SERVICE}..."; eval "$START_CMD" || true' EXIT -{% endif %} - -# --- Dump straight into age; plaintext never touches the disk --- -log "Writing ${ARTIFACT}..." -{{ backup_source_dump_command }} | age -r "$RECIPIENT" -o "${ARTIFACT}.partial" -mv "${ARTIFACT}.partial" "$ARTIFACT" -{% if backup_source_pull_user %} -# Readable by the pull account and nobody else. The contents are age-encrypted -# regardless, so this is depth rather than the actual protection. -chown root:{{ backup_source_pull_user }} "$ARTIFACT" -chmod 640 "$ARTIFACT" -{% else %} -chmod 600 "$ARTIFACT" -{% endif %} -log "Wrote ${ARTIFACT} ($(du -h "$ARTIFACT" | cut -f1))" - -# --- Prune --- -log "Pruning local artefacts older than ${RETENTION_DAYS} days..." -find "$BACKUP_DIR" -maxdepth 1 -type f -name "${NAME}_*.${SUFFIX}" -mtime +"${RETENTION_DAYS}" -delete - -log "Done." diff --git a/ansible/roles/backup_source/templates/backup.timer.j2 b/ansible/roles/backup_source/templates/backup.timer.j2 deleted file mode 100644 index d9a7e6a..0000000 --- a/ansible/roles/backup_source/templates/backup.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description={{ backup_source_description }} backup - -[Timer] -OnCalendar={{ backup_source_on_calendar }} -# Persistent: a window missed while the host was down runs on next boot. cron on -# a laptop had no equivalent, which is how two backups went unnoticed for months. -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/backup_store/README.md b/ansible/roles/backup_store/README.md deleted file mode 100644 index 13d9f5a..0000000 --- a/ansible/roles/backup_store/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# `backup_store` - -Pulls already-encrypted backup artefacts from every source host onto -`small-backups-box`, on a timer, and expires them per source. - -Generalises the hand-written `pull-backups.sh` that had one hardcoded source -(`arbret`). That job's behaviour is preserved exactly: same source path, same -90 days, same destination directory. - -## This host holds no key - -Everything pulled here is ciphertext produced by `backup_source` on the source -host. The box cannot read any of it — the age identity lives only on lapy. That -is deliberate: the machine holding every backup should not also be able to open -them. - -## One failing source must not stop the others - -The script is `set -uo pipefail`, **not** `-e`. Each source runs in its own -function, failures are counted, and the script exits non-zero at the end so -systemd marks the unit failed. A dead host costs you that one source, not the -whole run. - -This is the specific failure the whole plan exists to prevent: the laptop jobs -aborted on first error and then silently produced empty directories for nine -months. - -## Trust points one way - -The box authenticates with `~/.ssh/id_pull` to an unprivileged, dedicated -account on each source (`backup-pull`, or `arbret` on prd-arbret), authorised -with `restrict`. That account can read one directory and do nothing else — no -sudo, no pty, no forwarding. A compromised backup box cannot reach into -production. - -## Addressing: names, never IPs - -Sources are addressed by name. The job this replaced hardcoded spacey's IP; the -droplet was later rebuilt, the address was recycled to a stranger, and the -backup failed silently from 2025-12-01 while the directory listing still looked -healthy. - -Two kinds of name are in play: - -- **Tailnet members** (vipy, memos-box, …) → MagicDNS names. These require a - headscale ACL grant from `tag:small-backups-box` to the source's `:22`; without - it the box cannot even resolve the peer, let alone reach it. -- **spacey** is *not* a tailnet member — it is the headscale control server — so - its backup is pulled over the public internet via `headscale.contrapeso.xyz`, - which follows the host if the droplet is rebuilt. - -## Retention here is the long tail - -Sources keep a few days locally; this box keeps 90 (or whatever the source entry -says). Losing the source's local copy is expected. diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml deleted file mode 100644 index c0c128b..0000000 --- a/ansible/roles/backup_store/defaults/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -backup_store_dir: "{{ ansible_env.HOME }}/backups" -backup_store_ssh_key: "{{ ansible_env.HOME }}/.ssh/id_pull" -backup_store_on_calendar: "*-*-* 04:00:00" - -# One entry per source. `retention_days` is the LONG tail; the source keeps its -# own short local retention. -# - name: headscale -# source: "backup-pull@headscale.contrapeso.xyz:/opt/backups/headscale/" -# retention_days: 90 -backup_store_sources: [] diff --git a/ansible/roles/backup_store/handlers/main.yml b/ansible/roles/backup_store/handlers/main.yml deleted file mode 100644 index 632be35..0000000 --- a/ansible/roles/backup_store/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Reload systemd for pull-backups - ansible.builtin.systemd: - daemon_reload: yes - become: yes diff --git a/ansible/roles/backup_store/tasks/main.yml b/ansible/roles/backup_store/tasks/main.yml deleted file mode 100644 index d138830..0000000 --- a/ansible/roles/backup_store/tasks/main.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- name: Assert backup_store sources are sane - ansible.builtin.assert: - that: - - backup_store_sources | length > 0 - - backup_store_sources | map(attribute='name') | list | length == backup_store_sources | length - - backup_store_sources | map(attribute='source') | list | length == backup_store_sources | length - - backup_store_sources | map(attribute='retention_days') | list | length == backup_store_sources | length - fail_msg: "backup_store: every source needs name, source and retention_days" - quiet: true - -- name: Ensure rsync is installed - ansible.builtin.apt: - name: rsync - state: present - update_cache: yes - cache_valid_time: 3600 - become: yes - -- name: Ensure the backup store directory exists - ansible.builtin.file: - path: "{{ backup_store_dir }}" - state: directory - mode: '0700' - -- name: Install the pull-backups script - ansible.builtin.template: - src: pull-backups.sh.j2 - dest: /usr/local/bin/pull-backups.sh - owner: root - group: root - mode: '0755' - validate: "bash -n %s" - become: yes - -- name: Install the pull-backups systemd units - ansible.builtin.template: - src: "pull-backups.{{ item }}.j2" - dest: "/etc/systemd/system/pull-backups.{{ item }}" - owner: root - group: root - mode: '0644' - loop: [service, timer] - become: yes - notify: Reload systemd for pull-backups - -- name: Enable the pull-backups timer - ansible.builtin.systemd: - name: pull-backups.timer - enabled: yes - state: started - daemon_reload: yes - become: yes diff --git a/ansible/roles/backup_store/templates/pull-backups.service.j2 b/ansible/roles/backup_store/templates/pull-backups.service.j2 deleted file mode 100644 index c65516c..0000000 --- a/ansible/roles/backup_store/templates/pull-backups.service.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Pull encrypted backups from production - -[Service] -Type=oneshot -User={{ ansible_user_id }} -ExecStart=/usr/local/bin/pull-backups.sh -StandardOutput=journal -StandardError=journal -SyslogIdentifier=pull-backups diff --git a/ansible/roles/backup_store/templates/pull-backups.sh.j2 b/ansible/roles/backup_store/templates/pull-backups.sh.j2 deleted file mode 100644 index 267900c..0000000 --- a/ansible/roles/backup_store/templates/pull-backups.sh.j2 +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Pull encrypted backups from production — managed by Ansible (roles/backup_store) -# -# Everything here is already ciphertext: this host only moves and expires files, -# and holds no key that can read them. -set -uo pipefail # deliberately NOT -e; see the loop below - -SSH_KEY="{{ backup_store_ssh_key }}" -STORE="{{ backup_store_dir }}" - -log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; } -fail() { echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $*" >&2; failures=$((failures + 1)); } - -failures=0 - -# One source failing must not stop the others. The whole point of this box is -# that a single dead host cannot silently take the rest of the backups with it — -# which is exactly how the laptop-based jobs failed unnoticed for nine months. -{% for src in backup_store_sources %} -# --- {{ src.name }} --- -pull_{{ src.name | replace('-', '_') }}() { - local dir="${STORE}/{{ src.name }}" - mkdir -p "$dir" - log "Pulling {{ src.name }} from {{ src.source }}..." - if rsync -az --timeout=120 \ - -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15" \ - "{{ src.source }}" "$dir/"; then - log " {{ src.name }}: ok ($(find "$dir" -maxdepth 1 -type f | wc -l) artefacts, $(du -sh "$dir" | cut -f1))" - else - fail "{{ src.name }}: rsync failed" - return 1 - fi - log " {{ src.name }}: pruning older than {{ src.retention_days }} days" - find "$dir" -maxdepth 1 -type f -name '{{ src.name }}_*' -mtime +{{ src.retention_days }} -delete -} -pull_{{ src.name | replace('-', '_') }} || true - -{% endfor %} -if [ "$failures" -gt 0 ]; then - log "FAILED: $failures source(s) did not pull" - exit 1 -fi -log "All sources pulled." diff --git a/ansible/roles/backup_store/templates/pull-backups.timer.j2 b/ansible/roles/backup_store/templates/pull-backups.timer.j2 deleted file mode 100644 index 336e1ca..0000000 --- a/ansible/roles/backup_store/templates/pull-backups.timer.j2 +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Daily offsite backup pull - -[Timer] -OnCalendar={{ backup_store_on_calendar }} -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/ansible/services/forgejo/setup_backup_forgejo.yml b/ansible/services/forgejo/setup_backup_forgejo.yml deleted file mode 100644 index d6aafef..0000000 --- a/ansible/services/forgejo/setup_backup_forgejo.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Forgejo backup: dumps locally on vipy, encrypted with age. -# -# The biggest artefact in the estate (~2.7 G) and the reason retention here is -# short: 7 days locally would be 19 G of vipy's 36 G free. The box keeps 14. -# Forgejo is sqlite3 (DB_TYPE in app.ini), so it is stopped for the dump — the -# old job did the same. -- name: Configure the Forgejo backup on the edge host - hosts: edge - become: yes - vars_files: - - ../../group_vars/all/main.yml - - ./forgejo_vars.yml - - tasks: - - name: Ensure Forgejo dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: forgejo - backup_source_description: "Forgejo" - backup_source_dump_command: "tar -czf - -C / var/lib/forgejo etc/forgejo" - backup_source_stop_service: forgejo - backup_source_retention_days: 2 - backup_source_on_calendar: "*-*-* 02:30:00" diff --git a/ansible/services/headscale/setup_backup_headscale.yml b/ansible/services/headscale/setup_backup_headscale.yml deleted file mode 100644 index 9c72d1d..0000000 --- a/ansible/services/headscale/setup_backup_headscale.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: Configure the Headscale backup on the vpn_control host - hosts: vpn_control - become: yes - vars_files: - - ../../group_vars/all/main.yml - - ./headscale_vars.yml - - tasks: - - name: Ensure Headscale dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: headscale - backup_source_description: "Headscale" - # -C / with relative paths: avoids tar's "removing leading /" and makes - # the restore target explicit. - backup_source_dump_command: "tar -czf - -C / var/lib/headscale etc/headscale" - backup_source_stop_service: headscale - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/services/lnbits/setup_backup_lnbits.yml b/ansible/services/lnbits/setup_backup_lnbits.yml deleted file mode 100644 index 46abf31..0000000 --- a/ansible/services/lnbits/setup_backup_lnbits.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# LNBits backup: dumps locally on vipy, encrypted with age. -# The old job produced TWO gpg artefacts (data, then .env separately). They are -# folded into one tar here so the wallet database and the .env that configures -# it are always the same point in time; two artefacts written by two runs can -# drift. Pulling one file back out needs no unpacking: -# age -d -i | tar -xzO opt/lnbits/lnbits/.env -- name: Configure the LNBits backup on the edge host - hosts: edge - become: yes - vars_files: - - ../../group_vars/all/main.yml - - ./lnbits_vars.yml - - tasks: - - name: Ensure LNBits dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: lnbits - backup_source_description: "LNBits" - backup_source_dump_command: "tar -czf - -C / opt/lnbits/data opt/lnbits/lnbits/.env" - backup_source_stop_service: lnbits - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:20:00" diff --git a/ansible/services/memos/setup_backup_memos.yml b/ansible/services/memos/setup_backup_memos.yml deleted file mode 100644 index 2131924..0000000 --- a/ansible/services/memos/setup_backup_memos.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# Memos backup: dumps locally on memos-box, encrypted with age. -# Replaces the lapy pull, which had been writing EMPTY directories since -# 2025-12-27 — its script hardcoded 192.168.1.130, which DHCP later reassigned -# to a different machine that has no rsync. -- name: Configure the Memos backup on its own host - hosts: memos - become: yes - vars_files: - - ../../group_vars/all/main.yml - - ./memos_vars.yml - - tasks: - - name: Ensure Memos dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: memos - backup_source_description: "Memos" - backup_source_dump_command: "tar -czf - -C / var/lib/memos" - # sqlite in WAL mode: stopping checkpoints the WAL, so the artefact is a - # consistent database rather than a torn mid-write copy. The old rsync - # job did not stop it. - backup_source_stop_service: memos - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:00:00" diff --git a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml b/ansible/services/vaultwarden/setup_backup_vaultwarden.yml deleted file mode 100644 index 8430475..0000000 --- a/ansible/services/vaultwarden/setup_backup_vaultwarden.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Vaultwarden backup: dumps locally on vipy, encrypted with age. -# Previously rsynced to lapy in the CLEAR; the artefact now never exists -# unencrypted, on disk or on the wire. -- name: Configure the Vaultwarden backup on the edge host - hosts: edge - become: yes - vars_files: - - ../../group_vars/all/main.yml - - ./vaultwarden_vars.yml - - tasks: - - name: Ensure Vaultwarden dumps itself, encrypted, on a timer - ansible.builtin.include_role: - name: backup_source - vars: - backup_source_name: vaultwarden - backup_source_description: "Vaultwarden" - backup_source_dump_command: "tar -czf - -C / opt/vaultwarden/data" - # Not systemd — a docker compose stack — so stop/start explicitly. - # sqlite in WAL mode, hence stopping at all. - backup_source_stop_command: "docker compose -f /opt/vaultwarden/docker-compose.yml stop" - backup_source_start_command: "docker compose -f /opt/vaultwarden/docker-compose.yml start" - backup_source_retention_days: 7 - backup_source_on_calendar: "*-*-* 02:10:00"