counterweight
c3a7ca0ba7
Jar Lid Archive (Django + Postgres 18 + SeaweedFS) on jar-lid-archive-staging-box (nodito VM 3300, 1 core / 1 GB / 10 GB), public at jars-staging.contrapeso.xyz through vipy's Caddy. New role roles/jar_lid_archive; per-host values in host_vars, so a production host is a new [jar_lid_archive] entry, not a new playbook. ── Containers, following :master ────────────────────────────────────────── The same three services as the app repo's compose.yaml, so staging rehearses what dev and CI run. Every playbook run pulls :master and `up -d --wait`s; compose recreates web only when the digest moved. --wait gates on web's /health/, which checks the database and the bucket, so a green run means the app serves. Upgrading is re-running the playbook. ── web is on the host network ───────────────────────────────────────────── Docker's published ports are DNATed before UFW sees them, so `8000:8000` would answer on the LAN whatever UFW says, and binding to the tailnet IP races tailscaled at boot. On the host network gunicorn's port is an ordinary one: UFW default-deny covers the LAN, tailscaled's ts-input accepts tailscale0 ahead of UFW (checked on mempool-box), and the headscale ACL grants tag:prd-vipy exactly :8000. db and storage publish on 127.0.0.1 only. Verified: LAN times out, vipy gets 200. That matters for HTTPS: the app trusts X-Forwarded-Proto (DJANGO_BEHIND_TLS_PROXY=1), which is only safe while Caddy is the sole client. Login POST through the public URL passes CSRF; cookies are Secure. ── Secrets ──────────────────────────────────────────────────────────────── Generated straight into host_vars/jar_lid_staging_local/vault.yml, never printed: Django key, Postgres and S3 secrets, and seed passwords for the fixture's alice/bob/admin usernames - never the fixture's published passwords, since the site is public. The group vault gains one line, this host's Gatus push token; decrypted, it differs from before by exactly that line. ── Backups, and the outage the first one caused ─────────────────────────── One uncompressed tar (pg_dump -Fc + the SeaweedFS data dir), age-encrypted on the box at 02:20, pulled by small-backups-box; 3 days at both ends. Verified end to end, including decrypting a pulled artefact on lapy. The first test run stopped web AND storage and left the app down 26 minutes: SeaweedFS outlasted its 150s healthcheck budget on this disk, compose marked it unhealthy, and web's depends_on never started it. Now only web is stopped - it is the only writer to both stores, so they are still captured at one instant - and the healthchecks have start_periods sized for the disk. Also inherited from the app's compose.yaml: storage writes /tmp/s3.json and chowns it to seaweed, so on a RESTART of the same container root cannot re-create it (sticky /tmp + protected_regular): "Permission denied". Fixed here with rm -f first; the app repo needs the same line. ── The disk is the problem ──────────────────────────────────────────────── The VM disk is on proxmox-tank-1, the HDD mirror, which knots-box's random reads (~6 MB/s) keep near saturation. Inside this VM: ~96% "full" I/O pressure while idle, ~5 min to start web, sudo occasionally over 12s. Moving it to local-zfs (NVMe) is the fix; noted in the README, not done. ── Also ─────────────────────────────────────────────────────────────────── - lan_ip read from the Proxmox guest agent; host key checked against the one the agent reads off the disk before it went into known_hosts. - Monitoring: liveness + disk (400), probe_jar-lid-archive-staging (403), public /health/ + DNS (402), backup-dump/backup-store. - tofu: the VM definition. - Deployed v2026.10.03.1. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
116 lines
3.5 KiB
HCL
116 lines
3.5 KiB
HCL
# Non-secret nodito infrastructure config. Auto-loaded by tofu (*.auto.tfvars),
|
|
# so no -var-file is needed. Provider credentials live in terraform.tfvars,
|
|
# which is gitignored.
|
|
|
|
proxmox_node = "nodito"
|
|
zfs_storage_name = "proxmox-tank-1"
|
|
template_name = "debian-13-cloud-init"
|
|
cloud_init_user = "counterweight"
|
|
|
|
# Public key injected into every VM via cloud-init.
|
|
ssh_authorized_keys = <<EOKEY
|
|
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOVC68uU7dTpZ8mDleakoBX6crnBdKNuvg6qeD73t+6L counterweightoperator@counterstation
|
|
EOKEY
|
|
|
|
# Note that the resources set here are designed for startup.
|
|
# Some of these services are resource hungry at first, but
|
|
# can be powered down after their bootstrap finishes.
|
|
vms = {
|
|
"small-backups-box" = {
|
|
vmid = 1200
|
|
name = "small-backups-box"
|
|
cores = 1
|
|
memory_mb = 512
|
|
disk_size_gb = 100
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"knots-box" = {
|
|
vmid = 2100
|
|
name = "knots-box"
|
|
cores = 4
|
|
memory_mb = 16384
|
|
disk_size_gb = 10
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"fulcrum-box" = {
|
|
vmid = 2200
|
|
name = "fulcrum-box"
|
|
cores = 1
|
|
memory_mb = 2048
|
|
disk_size_gb = 10
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"mempool-box" = {
|
|
vmid = 2300
|
|
name = "mempool-box"
|
|
cores = 2
|
|
memory_mb = 4096
|
|
disk_size_gb = 30
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
# disk_size_gb is the ROOT disk only. monerod-box also carries a second
|
|
# 500 GB disk (scsi1) for the blockchain, attached in the Proxmox UI and
|
|
# therefore invisible here - the vm resource sets ignore_changes = all, so
|
|
# tofu neither created it nor would notice if it vanished. It is mounted at
|
|
# /var/lib/monero by ansible/roles/monero (tasks/storage.yml), by UUID, and
|
|
# monerod's unit refuses to start without it.
|
|
"monerod-box" = {
|
|
vmid = 2400
|
|
name = "monerod-box"
|
|
cores = 4
|
|
# Raised from 2048 on 2026-09-20: monerod was OOM-killed hourly once the
|
|
# chain passed ~39% (1.74 GB anon-rss against 1973 MB of RAM), and the
|
|
# explorer shares this box. Applied in Proxmox; recorded here because the
|
|
# vm resource sets ignore_changes = all and will not reconcile it.
|
|
memory_mb = 6144
|
|
disk_size_gb = 10
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"memos-box" = {
|
|
vmid = 8100
|
|
name = "memos-box"
|
|
cores = 1
|
|
memory_mb = 1024
|
|
disk_size_gb = 10
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"forgejo-runner-box" = {
|
|
vmid = 3100
|
|
name = "forgejo-runner-box"
|
|
cores = 4
|
|
memory_mb = 4096
|
|
disk_size_gb = 50
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"arbret-staging-box" = {
|
|
vmid = 3200
|
|
name = "arbret-staging-box"
|
|
cores = 2
|
|
memory_mb = 2048
|
|
disk_size_gb = 20
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
"jar-lid-archive-staging-box" = {
|
|
vmid = 3300
|
|
name = "jar-lid-archive-staging-box"
|
|
cores = 1
|
|
memory_mb = 1024
|
|
disk_size_gb = 10
|
|
ipconfig0 = "ip=dhcp"
|
|
},
|
|
# disk_size_gb here is the music library's ceiling, not just the OS: the
|
|
# library lives on the root disk at /srv/music. ~45 GB is usable after
|
|
# Debian, so roughly 300-450 albums in FLAC. Growing it means a Proxmox
|
|
# resize + growpart/resize2fs (03_vm_disk_enlargement.md), because
|
|
# ignore_changes = all means tofu will not do it.
|
|
"music-box" = {
|
|
vmid = 8200
|
|
name = "music-box"
|
|
cores = 1
|
|
memory_mb = 1024
|
|
disk_size_gb = 50
|
|
ipconfig0 = "ip=dhcp"
|
|
}
|
|
}
|
|
|
|
|