From df4f712a93a4daf97cf32a491852622af50d09ee Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 19 Sep 2026 17:49:54 +0200 Subject: [PATCH 1/3] headscale: the ACL was `{}` in code and real on the box. Put it in git `Create ACL policies file` wrote this, verbatim, to /etc/headscale/acl.json: content: | {} An empty policy. The real rules - who may reach which host on which port - were maintained by hand on spacey and existed nowhere else. So the repo and the box disagreed about the single file that governs every machine's access to every other machine, and the repo's version was "nothing is allowed". It had already fired once. /etc/headscale/acl.json.937464.2026-09-12@14:03:08~ is Ansible's own backup of a 1596-byte policy it replaced with {} on 12 Sep; the live file is dated 17 Sep, so somebody rebuilt it by hand afterwards without the playbook ever being corrected. Every subsequent `site.yml` run was one command away from flattening the whole tailnet's access rules again. This commit changes nothing on spacey. The policy now lives in templates/acl.json.j2, captured verbatim from the live file, and the task is a `template:` instead of an inline `{}`. Rendering it produces sha256 1ed14e7247d957db5e9f8d4f4c53846dc646131ad900054c868483a9a84ed8fd, byte for byte the file that was already there - verified before applying, and again after. Three deliberate choices: - A template file, not the inline `content:` used elsewhere in this playbook. The whole failure was three invisible characters passing review; a 2KB policy needs to show up as a diff. - `validate: python3 -m json.tool %s`. headscale refuses to load a malformed policy, and a policy it will not load is the same outage as an empty one. - `"counter-net@"` is now `"{{ headscale_namespace }}@"`, which is already a fact in group_vars/all/main.yml rather than a fourteenth hand-typed copy. The header comment records why a node with no ACL rule is INVISIBLE rather than merely unreachable - it never appears in `tailscale status`, MagicDNS does not resolve it, and the symptom is a downstream "no such host" that looks like a DNS problem. That cost an hour today. Co-Authored-By: Claude Opus 5 (1M context) --- .../headscale/deploy_headscale_playbook.yml | 17 ++- .../services/headscale/templates/acl.json.j2 | 130 ++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 ansible/services/headscale/templates/acl.json.j2 diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index e181f24..fe3a225 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -79,15 +79,24 @@ groups: headscale append: yes - - name: Create ACL policies file + # This used to write `{}` - an empty policy - over the real ACL, which was + # maintained by hand on the box. It had already flattened it once + # (/etc/headscale/acl.json.937464.2026-09-12@14:03:08~ is Ansible's own + # backup of the 1596-byte policy it replaced), and every subsequent + # site.yml run would have taken the whole tailnet's access rules with it. + # + # The policy now lives in templates/acl.json.j2, captured from the live + # file on 2026-09-19. validate first: a malformed policy is one headscale + # refuses to load, which is the same outage as an empty one. + - name: Deploy the Headscale ACL policy become: yes - copy: + ansible.builtin.template: + src: acl.json.j2 dest: /etc/headscale/acl.json - content: | - {} owner: headscale group: headscale mode: '0640' + validate: "python3 -m json.tool %s" notify: Restart headscale - name: Deploy headscale configuration file diff --git a/ansible/services/headscale/templates/acl.json.j2 b/ansible/services/headscale/templates/acl.json.j2 new file mode 100644 index 0000000..98ada80 --- /dev/null +++ b/ansible/services/headscale/templates/acl.json.j2 @@ -0,0 +1,130 @@ +{# + Headscale ACL policy. + + CAPTURED FROM THE LIVE FILE on spacey, 2026-09-19. Until then this playbook + wrote `{}` here - an empty policy - while the real rules were maintained by + hand on the box. Ansible had already flattened them once: + /etc/headscale/acl.json.937464.2026-09-12@14:03:08~ is its own backup of a + 1596-byte policy it replaced with {}. The next full site.yml run would have + done it again, taking the whole tailnet's access rules with it. + + So this is the source of truth now. It is a template and not inline `content:` + (the style used elsewhere in this playbook) precisely because it needs to be + diffable in review - an empty policy is the kind of change that must never + again slip through as three invisible characters. + + ADDING A HOST: it needs a tagOwners entry, and an acls rule naming the exact + ports its peers may reach. A node with no rule is INVISIBLE to the others - + it will not appear in `tailscale status` and MagicDNS will not resolve it, + which surfaces as a Caddy 502 with "no such host" rather than as a refused + connection. +#} +{ + "acls": [ + { + "action": "accept", + "dst": [ + "*:22", + "tag:arbret-staging-box:4117", + "tag:prd-arbret:4117", + "tag:nodito:8006" + ], + "src": [ + "tag:framie" + ] + }, + { + "action": "accept", + "dst": [ + "tag:knots-box:8333", + "tag:knots-box:7152", + "tag:knots-box:23334", + "tag:fulcrum-box:50002", + "tag:mempool-box:8080", + "tag:memos-box:5230", + "tag:arbret-staging-box:80", + "tag:stalwart-box:25", + "tag:stalwart-box:465", + "tag:stalwart-box:993", + "tag:stalwart-box:4190", + "tag:stalwart-box:8080" + ], + "src": [ + "tag:prd-vipy" + ] + }, + { + "action": "accept", + "dst": [ + "tag:knots-box:8332", + "tag:knots-box:28332-28335" + ], + "src": [ + "tag:fulcrum-box" + ] + }, + { + "action": "accept", + "dst": [ + "tag:knots-box:8332", + "tag:fulcrum-box:50001" + ], + "src": [ + "tag:mempool-box" + ] + }, + { + "action": "accept", + "dst": [ + "tag:prd-arbret:22", + "tag:prd-vipy:22", + "tag:memos-box:22", + "tag:stalwart-box:22" + ], + "src": [ + "tag:small-backups-box" + ] + } + ], + "tagOwners": { + "tag:arbret-staging-box": [ + "{{ headscale_namespace }}@" + ], + "tag:forgejo-runner-box": [ + "{{ headscale_namespace }}@" + ], + "tag:framie": [ + "{{ headscale_namespace }}@" + ], + "tag:fulcrum-box": [ + "{{ headscale_namespace }}@" + ], + "tag:knots-box": [ + "{{ headscale_namespace }}@" + ], + "tag:memos-box": [ + "{{ headscale_namespace }}@" + ], + "tag:mempool-box": [ + "{{ headscale_namespace }}@" + ], + "tag:nodito": [ + "{{ headscale_namespace }}@" + ], + "tag:prd-arbret": [ + "{{ headscale_namespace }}@" + ], + "tag:prd-monitoring": [ + "{{ headscale_namespace }}@" + ], + "tag:prd-vipy": [ + "{{ headscale_namespace }}@" + ], + "tag:small-backups-box": [ + "{{ headscale_namespace }}@" + ], + "tag:stalwart-box": [ + "{{ headscale_namespace }}@" + ] + } +} From 3c50cab58d6bbd234f8f24465e9f3f15f6aaa1e8 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 19 Sep 2026 17:50:46 +0200 Subject: [PATCH 2/3] monero: a full node on its own box, reachable from Feather over the WAN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit monerod v0.18.5.1 on monerod-box (VMID 2400), Debian 13, systemd, no Docker. Syncing at 24% as of this commit, 0 restarts, 0 OOM kills. ── The build-from-source requirement could not be met ────────────────────── The intent was to build it ourselves and check the result against the published checksums. That is not achievable for this release, and the role's README says so at length so nobody re-derives it: contrib/guix - the reproducible build that yields a byte-identical tarball - exists ONLY on monero master. At tag v0.18.5.1 contrib/ contains `gitian`, whose README wants an Ubuntu 18.04 host (EOL 2023) and python-cheetah. A plain `make release-static` is not reproducible at all, so it produces a binary whose hash matches nothing - failing both halves of the ask. So we install the published binary and prove its provenance instead: 1. binaryFate's key is VENDORED at files/binaryfate.asc. A trust root fetched over the network at the moment you need it is not a trust root. Its fingerprint is asserted after import regardless. 2. hashes.txt is verified, and the assertion is on gpg's `VALIDSIG ` status line - NOT on the string "Good signature", which is what bitcoin_knots greps for and which passes for any key in the keyring. 3. The hash is looked up in the output of `gpg --decrypt`, not in the raw clearsigned file, so nothing unsigned is ever searched. 4. The lookup requires hash AND filename on one line. Tested with a positive and a negative control before deploying: the real tarball matches, a corrupted hash correctly fails. monero_sha256 in defaults/ is a second in-git assertion so a version bump cannot happen without the checksum moving in the same diff. When v0.19 ships with guix on the release branch this gets easier, and the README records why: the published monero-linux-x64 tarball unpacks to monero-x86_64-linux-gnu-v0.18.5.1/, i.e. it IS the guix DISTNAME renamed, so sha256(guix output) will equal the hashes.txt entry exactly. ── The 500 GB disk is provisioned in code ────────────────────────────────── knots-box mounts /mnt/knots_data (~875 GB of chain) entirely by hand; that mount appears nowhere in Ansible and a rebuild of that host would come up with the blockchain silently missing. tasks/storage.yml does not repeat it. Whole device, no partition table. Everything after mkfs goes by UUID, because /dev/sdX is assigned by the kernel at boot and this estate has been bitten eight times by an identifier written down once whose thing then moved. monero_data_device is used ONLY to create the filesystem. Guards refuse to touch the root disk or to overwrite an unrecognised filesystem, and a final assert confirms the data directory is not still on /. fstab gets `nofail` so a missing disk cannot drop a headless VM into an emergency shell - and the unit gets `RequiresMountsFor=`, without which monerod would then start, find an empty directory, and write a 300 GB chain onto a 10 GB root disk. With it the unit refuses to start and infra/401 says so within five minutes. The mount-point task and the ownership task deliberately share owner and mode. Writing the obvious root:root 0755 on the first made them disagree and every run flipped the directory back and forth; caught by re-running --check. ── Exposure: one HTTPS vhost, zero new open ports ────────────────────────── Two RPC servers, not one: unrestricted on 127.0.0.1:18081 for administration and the health check, restricted on :18089 for everything else. That is `rpc-restricted-bind-port` and deliberately NOT `restricted-rpc=1`, which would restrict the loopback server too. Verified live: the restricted port reports restricted=true and blanks version and start_time. --rpc-login is HTTP Digest (contrib/epee/src/http_auth.cpp) and core_rpc_server::init hands the same login to both servers. Caddy on vipy does TLS and routing only - no caddy_site_basic_auth, because a Basic challenge would collide with the Digest challenge Feather answers. Verified end to end from the WAN: 401 unauthenticated, get_block_count over /json_rpc when authenticated, 401 again with a wrong password. No UFW rule, and that is correct: tailnet traffic bypasses UFW entirely because `-A INPUT -j ts-input` sits ahead of every ufw-* chain. Confirmed on mempool-box before relying on it here. P2P is outbound only. Nothing forwards 18080, so the node dials out, syncs and serves us without relaying for strangers; no-igd follows from that. ── Monitoring: infra, systemd, public. No backups ────────────────────────── Six Gatus endpoints, all green: liveness, disk, services/monerod-box-local/ monerod, probe/monerod, dns/monero.contrapeso.xyz, public/monero. The probe does NOT assert `synchronized`. A full sync takes days and a check requiring it would report DOWN for every one of them - an alarm with the batteries taken out. Height and sync state go in the message instead. The timer carries no `Requires=monerod.service`. Fulcrum's did, and sat active+enabled for seven months without firing once. public/monero asserts 401, not 200, so it goes red if rpc-login ever stops being enforced. No backups, deliberately, following the phoenixd precedent: the chain is re-syncable public data and there is no wallet on this node. ── Known constraint ──────────────────────────────────────────────────────── The VM has 2 GB and no swap; monerod's RSS is ~1.3 GB mid-sync. It is holding (0 OOM kills, ~1 GB available, kernel reclaiming cache) but 8 GB is the recommendation once the initial sync is done. out-peers/in-peers are held at 16/8 partly for this reason. Bumping it is a Proxmox resize plus memory_mb here, since the vm resource sets ignore_changes = all. The 500 GB data disk is likewise invisible to tofu - it was attached in the UI and the vm resource ignores it - so vms.auto.tfvars now carries a comment saying where it is and what mounts it. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/group_vars/all/main.yml | 5 + ansible/group_vars/all/vault.yml | 353 ++++++++++-------- ansible/host_vars/monerod_box_local/main.yml | 27 ++ ansible/infra/402_public_monitoring.yml | 5 + .../infra/403_service_probe_registration.yml | 1 + ansible/inventory.ini | 4 + ansible/roles/monero/README.md | 151 ++++++++ ansible/roles/monero/defaults/main.yml | 96 +++++ ansible/roles/monero/files/binaryfate.asc | 87 +++++ ansible/roles/monero/handlers/main.yml | 17 + ansible/roles/monero/tasks/configure.yml | 55 +++ ansible/roles/monero/tasks/healthcheck.yml | 58 +++ ansible/roles/monero/tasks/install.yml | 232 ++++++++++++ ansible/roles/monero/tasks/main.yml | 15 + ansible/roles/monero/tasks/service.yml | 35 ++ ansible/roles/monero/tasks/storage.yml | 133 +++++++ ansible/roles/monero/tasks/user.yml | 24 ++ .../monero/templates/healthcheck.service.j2 | 19 + .../roles/monero/templates/healthcheck.sh.j2 | 70 ++++ .../monero/templates/healthcheck.timer.j2 | 15 + .../roles/monero/templates/monerod.conf.j2 | 46 +++ .../roles/monero/templates/monerod.service.j2 | 38 ++ .../services/headscale/templates/acl.json.j2 | 4 + .../monero/deploy_monero_playbook.yml | 39 ++ ansible/site.yml | 1 + tofu/nodito/vms.auto.tfvars | 14 + 26 files changed, 1381 insertions(+), 163 deletions(-) create mode 100644 ansible/host_vars/monerod_box_local/main.yml create mode 100644 ansible/roles/monero/README.md create mode 100644 ansible/roles/monero/defaults/main.yml create mode 100644 ansible/roles/monero/files/binaryfate.asc create mode 100644 ansible/roles/monero/handlers/main.yml create mode 100644 ansible/roles/monero/tasks/configure.yml create mode 100644 ansible/roles/monero/tasks/healthcheck.yml create mode 100644 ansible/roles/monero/tasks/install.yml create mode 100644 ansible/roles/monero/tasks/main.yml create mode 100644 ansible/roles/monero/tasks/service.yml create mode 100644 ansible/roles/monero/tasks/storage.yml create mode 100644 ansible/roles/monero/tasks/user.yml create mode 100644 ansible/roles/monero/templates/healthcheck.service.j2 create mode 100644 ansible/roles/monero/templates/healthcheck.sh.j2 create mode 100644 ansible/roles/monero/templates/healthcheck.timer.j2 create mode 100644 ansible/roles/monero/templates/monerod.conf.j2 create mode 100644 ansible/roles/monero/templates/monerod.service.j2 create mode 100644 ansible/services/monero/deploy_monero_playbook.yml diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index f77ae48..1d16bb4 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -48,6 +48,11 @@ subdomains: # DATUM Gateway dashboard (knots-box, proxied via vipy) datum_gateway: datum + # Monero restricted RPC (monerod-box, proxied via vipy). Not a dashboard: + # this is the JSON-RPC endpoint Feather points at. Caddy terminates TLS and + # monerod's own Digest auth does the authentication. + monero: monero + # Read by plays across several groups, so global rather than group_vars/. headscale_namespace: counter-net diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index 76eb7bf..9c3fb3e 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,164 +1,191 @@ $ANSIBLE_VAULT;1.1;AES256 -34383033613438623432656438313763613639313139626432303637383363643339613166626665 -6537663336353661373437303636613364643736663031300a623461353135623537343763383333 -37323731646262613436646231656263396532616132316237353762376363383062646132303435 -3435316666316235660a323163613932616661303935336633656332353766393536393736356539 -61396164386266303464373262373862363361363365363235383363613335663933313062663565 -39613466353161356462656539613335613536363532333431393935303430373435373737363537 -62383266323339373062313134316464303263313830646161333530383736336637333865373764 -31313165396366653763343466643561383236643836616662383439666165343139326232326364 -32633133373863616163343365656231353939366435346534383462396134653064663566343566 -66343464373865383864313264646635316233336133346365396634626261613561373965643339 -30353831393037316334313632376537393361336366326561343832373531303139323937613531 -34386164303430316162363536386137316330336638303365393462386465633862663932376563 -34323862636435306563333065383134343362333733623066656439613339376666353331373665 -32353938663364313036373237323062393430306661633732356434333232376338313835663330 -32306536393666323464663238306562613936376236613935616664653865373330626365626330 -64613066653632646638306338393331376430383165316363323437373362613866313962663432 -62356663636630356239613831353066303530363966396135376134633936313530643262353965 -38383330613962313935366633353635363638303937316362303866343433353437363334396431 -32393134626133353564643233616632376633303934653065613262353436333533376137303862 -35303861383833326537376433336634633730373833313036643365373431666438653033653930 -33346230396661636333613031353133353631306331666362386366633062666534303539316238 -39366634656333646634643032393738333330636230663366623265666533356534623465316665 -35336535326237343231653661613736626239316262386363666264613636326332366232333964 -38353635616662663133376534306366643037613732343233373336653166316330643638353539 -35353532643534613864333839393365313439343663333337373639633832383630316164306132 -63636162373034363839656638353534613733663737626432636436373164303239316436313566 -66623466396663353136323565643262383865323830616232373466373431373261306537646463 -61353966623062356438626635663461396165343366393132393965383734633632666335313030 -37333631373239303231396264393031373162373936303462363736313538343562343737356332 -63663762633032356130303964386664366137373432353533386566613037613163383963623732 -32343765376535343630656633333362393765666131653632613361363163383038323138333537 -37373333316466316562613439356365316163336366623435666166653835616531653563343664 -30303464663838646333373837363961356465613234636138633162366636633030383337643131 -62636236663066316630396132333934353139383465363034363033656161383663656462313939 -62353738396132343734343363636164323163616237613861643862303433633934656165343263 -35313737373333386335663731643664393630653839396464376639306231653934386336353630 -61393735336662663435613431386361373561363531643232303831363163636139653538393931 -66313965646562616237383439343637623835303065333730613865666638383131616261346463 -31396464306537613135653830653138323539393731626264326335336432666333333735646534 -30616439393464326631633635363466303336613135346231626232313361303666323661616638 -64363937343231363536353034363966326132333734386638613737636130646232363666643633 -61353563636466356630346232613761636432303430336461663434333636623962643764336639 -35303565613861323431393036303534303061666437366538306236373930313439353330313632 -61363363303562316664303065613863613339626632643931386438326330373938613762303334 -33316430363931393262373661623137633835656136313235613666613932313236343966306331 -36343031303632323432366337633637336335343638393564313738386162386164613339343662 -62383939663265373265613932633265626263623939303638383838646531343433393864363235 -37313761396436623833376435393537376136373162393465343764326533666533393061643965 -61306365346562333737313962633764313232323161623861336566343735353737633539656162 -34323939306661373662663331653333346264323930663636633134666532623438323537383165 -66343033363962633766303331623866306462623235373838616565653066646132633034363535 -31613462356361613535383963643662373162363363653334393937313039666266366363653537 -64356166396139393262313565303731346534646462333638316661383139346362333364613466 -38303561316263636136303431393366343936653161336238613439366266346136636465303337 -61373634376464333037303862616335623031656133636165636264386265643261373735663266 -39396265366138363430343035356337303438346165366361316230656239326633653537626339 -39613334316563376133343336343563363564653237633764393532336437303334363830396635 -31333963666462313830333163646465356337643263636462356363613266323630393866626535 -35336436663063333364353663616564626535396133386461326433323936323161613633353539 -66323935626433353162666335613839346561343264323763303034613366663233613037343538 -66376236373462383430313030373235373337653338333033306431616562353435656433356365 -37653063363435366532306334373438306632306435653334396630343863623938666335656164 -37386634366430383762316435396237313362313965353634613266616532633465646464313662 -35303433363236393465396161346431306231666432666531626136623966663864376663373562 -37653239633732346536323265643434616136326561666330633435383264333937353366653736 -61353430633363323263323163646639663330663632343038623964396433626337353232373334 -32343938356539303938666137323831646562393233323733336464316165623232326132316362 -61313631626234336164656533616231393663363364623130313439623466396231383537656562 -35393163323131383838356130643863633736613635306635306135666261623731613239303962 -61373432336638353339646665653561653738353731373032366361313562306466373365653333 -35306236336331333961613864346165613763383266613236343066623432303766333434333331 -66376263363132616132366165393235626336323665373135653462653533376138346632393363 -35396165383966326635333134663138366136336430393532373935373264616530613762386430 -37643862316232613261343430616438333831623835663239656531356666613032653837373531 -30386232643537306530323936386561646265616339376265353763663833623539653831656634 -63626564633964343663656435636635656562666236386639353332356566316631643431353330 -32643238623932643732353332633535643738663066383830376432346463393464333236386163 -64376165333266323166613564313061643832656239333165393035343836376434626162643061 -31373861393534636236383765313834356638613332626666653762343537363339383939666263 -61366230623930643534353565316165656132396664393138326139613536343738396330653735 -37623463343834626133313231393130646134363331393238663065643930383237326439343831 -32393536613361616637373761393361303936363863333838666633343063386235316237323531 -62633362316436623761616366383336613765373362393161303765613464643632666462343565 -63616430396164346263666436336463623638313130656231643131336663336565616135373433 -39326533643733333564373434333738343634383033353036646534646632383730613161393130 -62653836376435663435353739373536366466646332653030653332353761626466353631663163 -38613734383634663662376464653733306564313761336566336336656366653536643834653631 -35333963306562393234623039613338643266333762333236366331333431343830393864623830 -34323439386133613035363861623332663933313734313637623739613163663730316134303434 -35363239363836666463366334356435666632396132306163303037663430383535393934323537 -30353266373864346662333035393761646632313032383738346439343232663666343833333565 -35363564623038376666363436663830356165323234653230636264303066666230643632356237 -66633734346633366533623966393462313438333965623033343337323135353730613561363236 -30636333613034626162393137323762306432663761393766303163393536656338366664643432 -34636630323965396166356538623165663335333739323034333237306531393734306138343436 -36613937373932633234303336373530663533646635396664343935326334333239646534343336 -35396666353938326337646431353965653836323731313861613031356561376262363965623934 -65336164623134366365303062393161353965363937646363396564313138323938656330333964 -64663862363766613630333531623463616331363533343962653164376665303463336134346662 -66376238663632663932656230323638323135653666363736643065336330636236323064386361 -37646631303138616565383665663634333338613336346438366333636165366635313337373635 -62656236653961393061373262326534393339363236353336613339656462336661663739396261 -35363636653237316436656662633462306333313336313865333037636661383731336466666536 -36646537653431666261353232653233653930346538383438633466346636656432383963333765 -62346363633265306665366630383831303434336139343837616361666539656262633735363064 -30376138386531336430613563366666333964663231363465326230316233323532366632663930 -63653164373931353861653833326130643365373530663039363166366331303134366336613438 -63633336363539636265396331653630633162316366316130326531363463616464376635376538 -32376638353337343639633434613663666437313564333264383766623561393933333561663235 -37666163626538353736383435373530663438363636643665626266316338336532623235353337 -61653762383630616266353438386137653137616532633066623165383663646437343565366237 -33333463613539346136613135386466656135616136663036313661323738363066633038656139 -38353230313861666463383737303038613731316334333031643438343463666164633536313031 -39613663376436643165333362636462303230656336333234363336633262336363643736336231 -66316630613762383832613836333263303038306235663662323162633932653733626130356665 -65316630346534356261373961353130393334396365376237353439326261336339393336303030 -63323439666439313831626639323735653730356565643938376564316337613333616664643865 -30633530383865663731383635366566613565393632336337306334663937396135303539363536 -65656331386532306438383031313164626237393762303636663831666239336637666465633630 -63383961336333663537666561376532376631343132363638633364616531306363623432396332 -31343864626335613131653762363338393738343531626330663663376531336230656365363661 -64363530333965386439376161313534333435393966656231613161633034303964653564363436 -63396131623562303962383137666439373132633330363237353031306561306362666561376435 -66383535333233663736363762323662363264393365643135323834366565653932313863353437 -38363536323739336165356561613335653464363637636234313637383066633865353239393961 -31613532373361383062393238303430353662383930373764373237383035633364323661623661 -62653661623234366666616532663563316234663137616338393036333937633963323635366165 -61613032363531666338353132346365323661626265626461396362646338373536306362303338 -31646564343434633433663039366533656665643235336136333134366231376336343465336539 -37363139346239666232313366613432646265626564313838346364306133306434313337626562 -32346430316532623037633838303363346130623636386134313566383333613565316135373264 -38393261313435323433326130303032393538333963646430646366653363613830613534353832 -32643434613964613239363966376138376661636335373130356430336631333461623734383535 -35356461326136636234663738343738303064386536336462303632303461303733333666383036 -38383563393739343134363038623638393766316533373163336439336238613734656132623163 -32626564666430663339316163393934323133306238353562323866633738363737643937376435 -66306430623334643766623564363239346361393666663766306637313265313833396435626234 -35366239616132623863343365663833363934316362636638303536656631633364646235366566 -39386637613735373761643339396132323031323438316633363464336636316534643435323861 -62663732366430646538626663313035616235643537643234356434323635373962343633336266 -30323031306438656537626430356231393466313334633934623632663661306361313330633339 -33396234386536336331616161363132343765306562303932313963623037366633383765663134 -62636466636433666530363930633765626531613539363832313361666565323635326334326533 -36336439363836663033643262336139613437623633616138656564393032393263393132303031 -31626336663039633362366663306662333432643261613464303939326562653562346239383261 -37373966383635626433363831653936333964626262363839383936356634633233336365623765 -62313031353538636464303234383865613932323164386336316362323731303263346637346534 -32353734343130343364646161643431353230336364366330363261326334613936633234646264 -39623163346663613431323630303034383761663835373565663166366239303130386139396236 -62393565353761636636366462353038366664663430616332326465373364396264323064666536 -61653035633562316634336263636332363733663666626131383161656535383133613939623034 -66363632613733306336323165336534346337386163656631343332323636353539353737316131 -34616336343865656565646338653037333838663736333330376330663834373138363739633064 -61646439636365323838376131663266636235333062616532613936616339633661303634323531 -63356335386534353639323566376134373565353137333134363761666532366431633634333430 -39326662633765626230326130386333393463323433366162363432613234336634663439313339 -33653737333830336264343839396563316462613032376335356634383436353962613636333866 -32333032623635323430356636623138366635386533646133626164316438393937626462313239 -31636635363064363763396566306234363965346438653738333961623435303233396634643763 -65343763386464366636383734336335646464306262623363303934393734333666636635356365 -65353931303862343765343665373139376530656263356365366333323135383838383030613166 -30303963383833643937636539656164336537383337346436303534303035313935323633663765 -333464336235373230316337333632326138 +61616634393062353739616362333531616534666431633234333539376337313235366337346364 +3135643764366262633636353735613233343361363165390a303638313465643332363839326530 +66623032363563306135643838626435646639306438393638353133393731386139336661383461 +3965353830373536360a653364663734333762616630313563656335333933333566333334393837 +64636438386264316333633839623736333932356365643639376262353564363639303438373534 +37346139393038643332626337646430376165316133343362636235623838343637343062623431 +35333530376430643562656233356165313065633865343439363238663966366332346536363430 +37313261363434376438306333373161303465653132353864366562363235376630373262356633 +64393830643037346565303435633966303234666638653833353138346464383462383136313433 +38356663303231313134623862643762633762303364616433623963326463326232333662643839 +35346435643234663838643934376461393631666363333435623965376239353137613533383033 +34663662666362613661613833383166343965333939336631363633343332653064653761353063 +31356263396434386433393332353532373663396431313539343138623564393434373135616239 +39353163363063323064633935346134663165626265633136633833643538616662333663366664 +61323866313266626637356532336166643261343662636464346566613531626532663961613238 +65353434653466316438343631376332316566376438623835393438313239326339323564623564 +37313938396562623865646338336634333637666538656139646539376531656464653136663462 +30343537303230306331386139376261396162326437623265373336356163636562623932653562 +64353763646636363839613365386233316262356339653332323565383033613561356236666336 +39616636353530636434366430643239383164376139343064316430396431363363383166346130 +61373735313633313639323933646632393231633935326439316466623438393233623139626139 +38353333376164613765656164363436613831663333656239346661623031393238353765356233 +65336632633361356432326339613766633938336537306261633236333835646335356365393330 +38656236363864613862643365383265633134643338383033643434376530303462336564383066 +66663061646432333034316431353236323938623639303263393537313939373162633838343062 +32316230613033363931343162386538376435386139396634333061313930373335306663393938 +30643466303338316663613563636239343432363964653438666662356437623962653266346434 +64616136643835396430306633313736623736316634313466366538663437633437613963386231 +39666565366464326132643836366431653535343162336563356664346166313532393664353139 +39383037323234323038316632383766383266393663646438653038643566376133636237653733 +38623438343432323664303461373930363437356136333235393262336336633266396531663638 +30393335396537356635343730376532363261353039396639653966633033383037656536646161 +39653531303962656234343634346236373362616634313664613764373439643734306639386461 +38663364613365643030616430356139393362386438613838326236643063633538346262626133 +33666537353862353461393939656231666330633434316261656134363062353835663833356435 +62663037383266353037326431376438313165343231656366306265666464303132613133303662 +39646462333439633065356137633932336162333438356337663637353033326362313036303564 +38643662633531383539393064356535363631336266316334386465313965376665306163393664 +64643434326561613333353134633166613464393537323263656466613239343562653636353764 +36343365656636393031626231343033653764393361376532646139613166306561323364356364 +62623838316430616636323966616634343732353031393665356137393532383762363565663038 +35323930633933626464633966343865346466353465626466663435316137366435373734363661 +33663939366165313939643032396562326565343961376366313730333735336333353164623266 +30306261363230306562343433656133303662306563646166386531386439633534623632373932 +30623361346632346463333661366539613233383337353961383564303438643965636235396163 +63363264383762303631613033663039626539336566653062333632383534306230396139613234 +62633862643535383536353537393431376431303262323332613731636534643033303635613030 +61653234313732643563383639303633343266623761623435373734613235656263353864353434 +63353039386665623164626666626330623730616438373832303337633364633735613364623131 +35613433366361616239363464343937363566383934313863636231373434636331383939623566 +34623933373434346561633331653163323664393130336665393465303162396630376236653262 +39393465346632323435636333613361353539643238313239373231376436313631663065386262 +34663637386632396137353261613666316564333636326231636137613235623765313137306537 +38306161633731343838356635613038616363653833623634633061376364376534323532666337 +65366634623331323562323562613735356235393434323063303766653334363236613036353638 +35353438623930366336383062303462646338633939393639643832623730633562306338313939 +35666334336334653534646662316534376235656364356563316632363130356635333336666130 +34616639393834373362323930613863383833663161316339323465616639633531376232303332 +33333333643934623538633738643631386232643932303138383861313261306261323532393233 +61353431373538313762653839343861313337643633643032333765396466393831393162306436 +32393832363638333832666364646266376634663031333932333735643831623361323564386261 +61323937393762613531646564363963626461343262373839383938663661626562613837323133 +39646632316666396637646664356230356131313062363039656163613238386334636636636636 +38363463633733663132366239343766353038383962353639313562343636623261376566653635 +35326338663935386132333035373362326235643864336162303063623539656439306466663834 +38333362353630363835613464303131306333313865623236633362623832633062333565343762 +39336161376661646238313839633165666435346433336337633734353166636566326464666330 +37386536613430343139653134343963626466333734356461323631623761663639633532376133 +61363632303566336161653430613735316562656334323265303163383939323564356465636530 +61626262653461386332646566306666323434613162613033373730303265306231316630346239 +39343735363632623561373838316334393166303365363766663965306161643365656132653531 +32613662663362653366633237386362623962323230303538633732663061376334393666326535 +35653038306233626133363732393964623630383637363937363335663836346564636639383266 +32326433316236376363333163346234303938626635363464636563653032656361323138326530 +36636163356632613738626136376563656233313034666161323362313533366461326132316162 +61646335383662313234666332623932396632333435316466346538313630633038663231373764 +39343933613361336439636536356136383535336262613538323837633563633038306231663632 +33396366343466383538306638656234383930306539633865373031343030333765663735386438 +35363232616533656632396661386566643738353862646535323533323237636137646239353664 +32353261366334353933376436336533306531356462343535346664656435626466356662613032 +32656666336235326165653861326431653335653131663034386166373434303936646634366635 +38356536343664363836616235326465616131636132643331373366656164653638613066333737 +38373864333734306234383130376239373238636262376664613865363265363337393031616566 +35336436373535613535653330306164373462313839316636326534366163326538653633376337 +34643039373461393565653838323061656330346632316133373064613661306365336462646237 +62326134623063346431643335313166306461653239303537663132643239386636653363323662 +39393262613766343563636561356362653665313234366532626638346264316363343264313435 +36386531656534666462383337343835313638376438653165663839393433376332623938363738 +35356538383136333534386531376530383936633463363166326133303264373534613066356538 +34373637303333333836386238303061303834383331653735336233666164656530656666356533 +39343030643236613831663465323735343737623362383734326162356333323864333035366433 +63316264613337363136613062663263313731636335383531383933623339666134643539353330 +61333330336132636666383561356565346563393937653539373431653561653664616137623133 +39336333313330336137346238306130653730326130323163306464376434353736623232656134 +33356435663762666266393461363531396334313733333132666363316133346531626564626334 +66356436393062373364316439646235666336333039383137353764303763636461663638333362 +66306430396530636538353965333862323363323832373033353530306331653935396134323634 +61353964303666643163646464613737393863373236613532353761636230336633373461343266 +34383031353734346465613030653639656366313530333265383632343465393566386161373633 +34313563636265336239323362316432383637303134376331633039333537613461353135303934 +66366334663032336330386632656162643933316666643562616564613633303231666536653536 +61376534663632316338653231653561643338356661353232633431623834396635383034366438 +30376161663339633530646539313132383536303931333366316631356135303933316162373730 +38343139653932666335353437613734366530616261343134386261313330303766666136616532 +38663635393236376661623862646265626239656665323332633532616536643237333137343763 +37663432366337633662303931353939346462633837346638353530636132313436626161633865 +30653065666261363238366534656265663039373962333938633335373163383261363434646437 +35356538623765343866653065306164363363616137323065333062386435393539303930363239 +62653638303766343738346437323536306139303261393863373035336662316566646565316662 +30633464663666383135663365636436363132363437633933643464626339643865383066623039 +66356338633939386337353135613066343236356262363164666466333462383364663134616433 +38626332623530633765656464383136383466623561633666366566316461356437626165316166 +65626339383133623439366662373666346539613231306564643731383935633939343865613534 +66363934373433363134323165383830643535613139313139316137643532303562393432383330 +61376363363939633835383161396464613662383961366634626463346430303631653930613430 +66616132616364623261363731343437633032383361356433333531363964633962336338373338 +61353134616332663135663737613230666534656232623933613862326366393538373737393838 +30343037343432306437663035613831316432336431383938363338316437616164376435376261 +66363664333031653962383630656430646238353337616535343535373935663561306536353165 +65343235396537366635633638306135383164353562336331376336323633323338353361353061 +65623431633731303936303732613964346139306336356461626466343132383162666337333664 +38636631636338353339316666643436386537336231353332663636383337343831323939396636 +30663634653733623530343537636137623332666666343433643830333737336639313064386333 +34656362613863323262326130666361373434336234366337663236663739383366373635343862 +62656365366533633263323963346363396237393462653633313133353935343631393032303566 +66326266353264306266613936333734643631376637663137373232653962646534643565326366 +64646634346534613730646430333432373464653365663666346232373332346231306563636136 +38646565366661613161383530326261613537366633323239656461343337363139353164653935 +66356636653232643035363564306438396233376465646462316463333832636361363137363364 +66653164363836333261386437373864666438383338383963616662386631306530326535303666 +37393037313539393535633333663662363164323635333635646638663264383533303032303561 +34613563353237633761363435636138616462656263353966636363326533363235663761643033 +63613934323330363732353465316261653964656430383532623833653663306632633832653763 +32333932663362663664663535363566656665653839386363373066343336353264663136326136 +66383461626137353430316266356366393937613234656261646632366130613739353266613163 +30383731373137653137353663643866343632383937613136666361626435393331313130386230 +33306439633138306331616165386239643765323765366530336337623264343030333234336234 +64333333613838333734346361636162346564656334396139373535333830313037383864623561 +38336131343532663761326635633531323766363963663763653630643232663631613465333132 +35306635636437323835326633616665313133393162623737386538373364343865383634636463 +33626165623962656432396237653665346464663066633633633564313266343662336333656166 +34646636663864663439356364383031313065663433386530663365353937626366663334346434 +35656365316134306161656133653736306537613133386634386638376534376133346237353765 +64363964303535653037323566653838633231363132666237353765643137383030623930393931 +33313935663366313362656137613865386532303737306437353437336331643435323030663666 +66383139313333633538373836396263353337316264656631376236346466353765653830366366 +66386131336431336466636166366139396337386135636262316237306561353563623061313266 +33343563393031303133393163653638643065636636653766613836386438626332373965383864 +36343936613738336132396265663130643739373264356530636363633239346637653136343761 +64646137383235636231313736343162353237636566663830373739636536393864646639343564 +61323039376537646531633030626665323537366233353939653665326666653263623431356166 +34343538613031336561303165323134303862666139633937323331643930383938666138346230 +33633537666431643234336134333333633664333564396139363136363434376130323632363833 +32363764656133653462633132366630396233613463643833353038316232353962623761373966 +35613132306662663937366464373034336531366262643939333137613031666162383434313664 +65363332383330623863326465353039306338393230303165656366386161656239316161336633 +61616434363637376131316135643336653636313565613035643038386332356263383061643262 +63333439303736623835653636353261363532646266663331313566363562343730303436353636 +63636237343965316366613137353062393832373133316438656433313132643061633130396662 +65346530313962306238373634366266386232613566656337396663626535393964323737323939 +66333165356138353362336166376466303965663439636432653862316433376264376563313137 +30653565386636363730623630646633633832306131626235613862393239386437386561356663 +64303862616530393838383364643266633336326665323961643935356339316564316332633635 +32323239333339633838666233376165303061373465376133383863333232663065626462303362 +37613638303266356537633965323966363331383130613963363933656531663031643030396632 +38343838383731346536616364303531373030643730626165633464646137376439643637633636 +62346664633931376438313639613165306363356532346566336336363662353233613134346132 +30646633363364363234396435323037306663633863383530383034383338636639326362626639 +64313532313939316562353561323534313438346530303536333066393863613136646664353264 +65663530366238343732396430633138613664653962366338643738623636373031363839393534 +38393439393366396262643461303361613666336631313037306231363537346364346432616539 +64616431633730333233616362363638613837356638343131376262363736666532356532653330 +64353431663261623033323337373932396436656633396439323162626563653630633239306233 +37633939363536396439643663623434316161383938393864653437316535313433336461383065 +32396536313835393030313136356530623539646361633935333362373339393662393335353361 +37353366666264613563663836666332353561666464623561653238666537316233343838663363 +38616463643935376233363031323838393163306465333030393331663231623061646530393237 +33356466363261653235336134376166343537643065663235333161383366336132316666383731 +39326662373162313861363132653531323165393939313461663262316337333762393134663030 +35313762356234396336356336393937316262646264306361363235663334333663363136383430 +35366538386335633038326563623833313031383762653266643034373838653863363835383466 +37323138316564383262356633646262623439326464633530313639313637326136656139376362 +35323866663262373938343863653665373139663565623730666366643630636566393266303261 +35666261383464393236353964623234306637643838623535313264356364366661653639303662 +37616531626435323664636538396439373064333563303637303562653763323935666138383338 +32326630303465366332656165343036336630383163643361306136333361373738306238393932 +63363331373434353563353631306361383431636239376464643239326531306130633463653665 +64636536383430613538623663363066646437666537306161613265323936663366353864323230 +32666333326630636433656166343331396166383438313866306566356238653061343461653035 +6236383630643231646264333133343738663663323332336536 diff --git a/ansible/host_vars/monerod_box_local/main.yml b/ansible/host_vars/monerod_box_local/main.yml new file mode 100644 index 0000000..10e28ee --- /dev/null +++ b/ansible/host_vars/monerod_box_local/main.yml @@ -0,0 +1,27 @@ +# monerod-box: the Monero node. +# +# These ports are read twice: by the role that deploys the service here, and by +# the Caddy play that runs on the EDGE host and publishes the restricted RPC. +# A role default is invisible to that second play, which is why they live in +# host_vars - the edge play reads them as +# hostvars['monerod_box_local']., and the role picks them up +# automatically because host_vars outranks role defaults. +monero_p2p_port: 18080 +monero_restricted_rpc_port: 18089 + +# The 500 GB disk added to this VM in Proxmox as scsi1. Named here rather than +# left to the role default so that "which disk holds the blockchain" is a +# property of the machine, recorded next to the machine. The role only ever +# uses it to create the filesystem; the mount itself goes by UUID. +monero_data_device: /dev/sdb + +# Systemd services deployed on this host, monitored every 5 minutes. +# +# The fact lives with the machine rather than in a central map, for the same +# reason the cross-host ports do: "what runs here" is a property of the host, +# and a central list is one more thing to forget to update when a service moves. +# +# Only units WE deploy belong here. Distro units (ssh, cron) have their own +# supervision and would be noise. +monitored_services: + - monerod diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index d924468..a43a8af 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -31,6 +31,7 @@ - {sub: "{{ subdomains.memos }}", host: vipy} - {sub: "{{ subdomains.mempool }}", host: vipy} - {sub: "{{ subdomains.datum_gateway }}", host: vipy} + - {sub: "{{ subdomains.monero }}", host: vipy} # A public resolver on purpose: this must test what the internet sees, not # what a local cache or the tailnet's MagicDNS happens to answer. @@ -50,6 +51,10 @@ - {name: memos, sub: "{{ subdomains.memos }}", path: "/", status: 200} - {name: mempool, sub: "{{ subdomains.mempool }}", path: "/", status: 200} - {name: datum, sub: "{{ subdomains.datum_gateway }}", path: "/", status: 401} + # 401 is CORRECT: monerod answers an unauthenticated request with an HTTP + # Digest challenge. Asserting 200 here would go green precisely when + # rpc-login broke and the node started serving the world for free. + - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401} # Ports published from the edge host by socket_proxy. public_tcp: diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index c92b767..a9c5de5 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -45,6 +45,7 @@ - {name: mempool-mariadb, host: mempool_box_local} - {name: mempool-backend, host: mempool_box_local} - {name: mempool-frontend, host: mempool_box_local} + - {name: monerod, host: monerod_box_local} tasks: - name: Build the probe endpoint list diff --git a/ansible/inventory.ini b/ansible/inventory.ini index c39439d..f14deb4 100644 --- a/ansible/inventory.ini +++ b/ansible/inventory.ini @@ -15,6 +15,7 @@ memos_box_local ansible_host=memos-box lan_ip=192.168.1.145 ansible_user=counter forgejo_runner_local ansible_host=forgejo-runner-box lan_ip=192.168.1.132 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua arbret_staging_local ansible_host=arbret-staging-box lan_ip=192.168.1.147 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua small_backups_local ansible_host=small-backups-box lan_ip=192.168.1.131 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +monerod_box_local ansible_host=monerod-box lan_ip=192.168.1.146 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua # Local connection to laptop: this assumes you're running ansible commands from your personal laptop [lapy] @@ -48,6 +49,9 @@ fulcrum_box_local [mempool] mempool_box_local +[monero] +monerod_box_local + [memos] memos_box_local diff --git a/ansible/roles/monero/README.md b/ansible/roles/monero/README.md new file mode 100644 index 0000000..abbe4a1 --- /dev/null +++ b/ansible/roles/monero/README.md @@ -0,0 +1,151 @@ +# monero + +A Monero full node (`monerod`) on Debian 13, bare metal, systemd, no Docker. +Runs on `monerod-box` (Proxmox VMID 2400), reachable from Feather over the WAN. + +## Why the published binary and not a build from source + +The intent was to build from source and check the result against the published +binary checksums. That is not achievable for the current release, and it is +worth writing down why so nobody re-litigates it: + +- `contrib/guix`, the reproducible-build system that produces a byte-identical + tarball, **exists only on monero `master`**. At tag `v0.18.5.1` `contrib/` + contains `gitian`, not `guix`. +- Reproducing v0.18.5.1 therefore means **Gitian**, whose README requires an + Ubuntu 18.04 host (EOL 2023), LXC or Docker, and `python-cheetah` (Python 2). +- A plain `make release-static` is **not** reproducible. It would give us a + self-built binary whose hash matches nothing - not upstream's, not anyone + else's - so it satisfies neither half of "build it ourselves AND check the + checksum". + +So the role installs the published binary and proves its provenance instead. +That chain is real, and it is the strongest claim available for this release: + +1. binaryFate's public key is **vendored** at `files/binaryfate.asc`. A trust + root fetched over the network at the moment you need it is not a trust root. + Its fingerprint is asserted after import anyway + (`81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92`), so a swapped file is caught. +2. `hashes.txt` is downloaded and its clearsigned signature verified. The + assertion is on gpg's `VALIDSIG ` status line, **not** on the + string "Good signature" - `bitcoin_knots` greps for the latter, which passes + for *any* key in the keyring. +3. The verified plaintext is written out by `gpg --decrypt`, and the tarball's + SHA256 is looked up in *that*, not in the raw clearsigned file. +4. The lookup requires the hash **and** the filename on one line, so a hash + belonging to a different artefact cannot satisfy it. + +`monero_sha256` in `defaults/` is a second, in-git assertion passed to +`get_url`. It is not the security control - step 4 is - but it means a version +bump cannot happen without the checksum moving in the same diff. + +**Bumping the version: change `monero_version` and `monero_sha256` together.** + +### The forward path + +The published `monero-linux-x64-v0.18.5.1.tar.bz2` unpacks to +`monero-x86_64-linux-gnu-v0.18.5.1/` - that is the Guix `DISTNAME`, renamed at +publish time. So once v0.19 ships with Guix on the release branch, +`sha256(guix output) == hashes.txt entry` holds exactly, and a `guix` install +method can be added here without changing anything else. + +## The data disk + +The blockchain is on a **dedicated 500 GB disk** (`scsi1`), not on the 10 GB +root filesystem. `tasks/storage.yml` creates the filesystem and the fstab entry. + +This is done in code on purpose. `knots-box` mounts `/mnt/knots_data` (~875 GB +of chain) entirely by hand, and that mount appears nowhere in Ansible - a +rebuild of that host would come up with the chain silently missing. + +Three things there are load-bearing: + +- **The whole device, no partition table.** One disk, one job; a partition table + is just another layer that can disagree with itself. +- **Everything after mkfs goes by UUID.** `/dev/sdX` ordering is decided by the + kernel at boot. This estate has been bitten eight times by an identifier + written down once whose thing then moved; `monero_data_device` is used *only* + to create the filesystem. +- **`RequiresMountsFor=` in the unit.** `nofail` in fstab means a missing disk + does not drop a headless VM into an emergency shell - but without + `RequiresMountsFor` monerod would then start, find an empty directory, and + begin writing a ~300 GB chain onto a 10 GB root disk. With it, the unit + refuses to start and `infra/401` reports it within five minutes. + +The mount-point task and the post-mount ownership task deliberately use the +**same** owner and mode. Writing the obvious `root:root 0755` on the first made +the two disagree, and every run flipped the directory back and forth. + +## Exposure: one HTTPS vhost, no new open ports + +- **P2P is outbound only.** Nothing forwards 18080 to this box. The node dials + out, syncs fully and serves us; it just does not relay for strangers. + `no-igd=1` follows - with nothing forwarded there is nothing for UPnP to do + except announce us to the router. +- **Two RPC servers, not one.** + - unrestricted on `127.0.0.1:18081` - administration and the health check. + - restricted on `:18089` - the only surface anything off-box touches. + + This is `rpc-restricted-bind-port`, deliberately **not** `restricted-rpc=1`. + That flag restricts the loopback server too, and then we could not ask our own + node anything useful. Verified live: the restricted port reports + `restricted=true` and blanks `version` and `start_time`. +- **`rpc-login` is HTTP Digest** (`contrib/epee/src/http_auth.cpp`), and + `core_rpc_server::init` hands the same login to *both* servers. Both ports + answer an unauthenticated request with 401. +- **Caddy on the edge host terminates TLS and nothing else.** No + `caddy_site_basic_auth`: Caddy's Basic challenge would collide with the Digest + challenge Feather expects. monerod's own auth is the authentication. + +### No UFW rule, and that is correct + +Traffic arriving over the tailnet **bypasses UFW entirely**. Verified on +`mempool-box`: UFW is active with default-deny-incoming and `22/tcp` as its only +allow rule, yet the edge host reaches its frontend port - because +`iptables -S INPUT` puts `-A INPUT -j ts-input` ahead of every `ufw-*` chain. +Adding a UFW allow here would be cargo cult. + +## Monitoring + +Four Gatus endpoints, all verified passing: + +| Endpoint | Source | Answers | +|---|---|---| +| `liveness/monerod_box_local` | `infra/400` | is the host up | +| `disk/monerod_box_local` | `infra/400` | is it running out of space | +| `services/monerod-box-local/monerod` | `infra/401` via `monitored_services` | is the unit active | +| `probe/monerod` | this role + `infra/403` | does monerod actually answer RPC | + +**The probe deliberately does not check `synchronized`.** A full initial sync +takes days. A check requiring `synchronized: true` would report DOWN for every +one of those days, which is not a monitor - it is an alarm with the batteries +taken out. Answering RPC is the health question; height and sync state go in the +message so they are visible without being alertable. + +The timer carries **no `Requires=monerod.service`**. Requires on a timer stops +the timer when the watched unit stops - "if the thing I am watching goes down, +stop watching it" - and leaves nothing to re-arm it. Fulcrum's timer carried it +and sat `active`/`enabled` for seven months without firing once. `bitcoin_knots` +still carries it; do not copy that. + +`Enable and restart the monerod health check timer` reports `changed` on every +run. That is intentional and matches `datum_gateway`: `state: restarted` is how +a timer gets re-armed, and `started` is a no-op on an already-active timer. + +## Backups: none, deliberately + +The blockchain is re-syncable public data and there is no wallet on this node, +so a backup job would move hundreds of GB around to protect nothing. This +follows the `phoenixd` precedent (Plan 5, Model C) of recording an exclusion as +a decision rather than leaving it to look like an oversight. + +## Known constraint: RAM + +The VM has **2 GB and no swap**. Mid-sync, monerod's RSS was already 1,333 MiB +of 1,973 MiB with ~100 MiB free. No OOM kill has happened yet, but the working +set grows with the database and there is no swap to absorb a spike. `out-peers` +and `in-peers` are held down (16/8) partly for this reason. + +**8 GB is the recommendation.** Bumping it means editing `memory_mb` in +`tofu/nodito/vms.auto.tfvars` and resizing the VM in the Proxmox UI - the vm +resource sets `ignore_changes = all`, so tofu will not do it for you. diff --git a/ansible/roles/monero/defaults/main.yml b/ansible/roles/monero/defaults/main.yml new file mode 100644 index 0000000..633f3c0 --- /dev/null +++ b/ansible/roles/monero/defaults/main.yml @@ -0,0 +1,96 @@ +# Monero node (monerod) configuration variables. +# https://docs.getmonero.org/interacting/monerod-reference/ + +# --- Version and artefact --------------------------------------------------- +# Bump monero_version and monero_sha256 TOGETHER. The sha256 here is a second, +# in-git assertion; the one that actually matters is the grep against the +# GPG-signed hashes.txt in install.yml. Having both means a version bump cannot +# be done silently - the checksum has to move in the same diff. +monero_version: "0.18.5.1" +monero_platform: "linux-x64" +monero_tarball: "monero-{{ monero_platform }}-v{{ monero_version }}.tar.bz2" +monero_url: "https://downloads.getmonero.org/cli/{{ monero_tarball }}" +monero_sha256: "22a7dda7b0cb699fdd6b7674c3b4a4465b337cc98a54983523b759e1e7cc9958" + +# The signed list of checksums for every published Monero artefact. +monero_hashes_url: "https://www.getmonero.org/downloads/hashes.txt" + +# binaryFate, the release signing key. The .asc is vendored in files/ rather +# than fetched at deploy time: a trust root downloaded over the network at the +# moment you need it is not a trust root. The fingerprint is asserted after +# import anyway, so a swapped file cannot go unnoticed. +monero_signing_fingerprint: "81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92" + +# --- Service user ----------------------------------------------------------- +monero_user: monero +monero_group: monero + +# --- Directories ------------------------------------------------------------ +monero_conf_dir: /etc/monero +monero_log_dir: /var/log/monero +monero_bin_dir: /usr/local/bin + +# The blockchain. This is the mount point of the dedicated 500 GB disk, NOT a +# directory on the 10 GB root filesystem - see storage.yml. +monero_data_dir: /var/lib/monero + +# --- The data disk ---------------------------------------------------------- +# The whole device, no partition table: this disk has one job and a partition +# table would only add a layer that can disagree with itself. +# +# monero_data_device is used ONLY to create the filesystem, once. Everything +# afterwards - fstab, the mount, the systemd unit - goes by UUID, because +# /dev/sdX ordering is decided by the kernel at boot and is not a stable name +# for anything. This estate has been bitten eight times by an identifier that +# was written down once and then moved. +monero_data_device: /dev/sdb +monero_data_fstype: ext4 +# -m 0: the 5% ext4 reserve exists to stop root-owned daemons deadlocking a full +# root filesystem. On a dedicated data disk it is 25 GB thrown away. +monero_data_mkfs_opts: "-m 0 -L monero-data" + +# --- Network ---------------------------------------------------------------- +# P2P is OUTBOUND ONLY. No inbound port is published from the edge host, so the +# node dials out to peers and never receives. It syncs and serves us completely; +# it just does not relay for strangers. no-igd follows from that: with nothing +# forwarded there is nothing for UPnP to do but tell the router about us. +monero_p2p_port: 18080 + +# Unrestricted RPC, loopback only: administration and the health check. +monero_rpc_bind_ip: "127.0.0.1" +monero_rpc_port: 18081 + +# The RESTRICTED RPC is a second server on its own port, reachable over the +# tailnet and published by Caddy on the edge host. This is deliberately +# rpc-restricted-bind-port and NOT `restricted-rpc=1` - the latter would +# restrict the loopback server too, and then we could not ask our own node +# anything useful. +# +# The authoritative value for the live deployment is in +# host_vars/monerod_box_local/main.yml, which the edge play also reads; the +# value here is the upstream default so the role still works standalone. +monero_restricted_rpc_port: 18089 + +# Peer counts kept modest on purpose: this VM has 2 GB of RAM. See README. +monero_out_peers: 16 +monero_in_peers: 8 + +# --- RPC credentials -------------------------------------------------------- +# monerod's --rpc-login is HTTP Digest (contrib/epee/src/http_auth.cpp) and the +# same login guards BOTH the loopback and the restricted server. Feather sends +# these as the node username/password. +monero_rpc_user: "" +monero_rpc_password: "" + +# monerod.conf holds the RPC login, so --diff is suppressed by default. +# Pass -e monero_reveal_config=true to see what moved. +monero_reveal_config: false + +# --- Health check ----------------------------------------------------------- +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. Any endpoint accepting an HTTP ping +# works; nothing here is specific to a monitoring product. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/monero/files/binaryfate.asc b/ansible/roles/monero/files/binaryfate.asc new file mode 100644 index 0000000..55c5b16 --- /dev/null +++ b/ansible/roles/monero/files/binaryfate.asc @@ -0,0 +1,87 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF3yDrwBEAC1UgCSLILsbdrSk5kfcEYKMvj7lJpIIj9D6LeeyIvovgO7beM0 +63cFCT0v+RH3CVKV8bCCJr8teR3Zgk+IeI6C0CQk+ocqlu0qBAALdZyGyZonozbc +lHGOfQ0rWEy01V/TB36bGhrsE/cM8nhICJ72Pkv3rrukZkprxvEJ+IYCk26Umiue +K1+Pm0sUMrxAQUYlvg8r1swOgLOuo7r8c1gZYvGixdb0t7mBzUgkSdmFUeAa/X8W +WzBPFluWMyetGUKzrV66W1ISHHi/2AyXim235Lqc4MbK2ObKfkZJCjC7y2afs5MS +t+uejz8bchLMM/LvV2TxKIbenho7ZxtGd8blNRAPe6FTOA+yOM50atJah1W3BmLx +sxk4gvII2/zlzaW4RNEy1Ma/47DINPwYB9BA7FqF7BTVt8WxpJ5Y2+8aR58ZtM+q +ObbO5s2O8kuDj94qQKAT8btetbb/pMKhF3XXSARkNZPzNFFtSy9xSin+hLAWpM0b +cftEtJrE7HY7DHOa9J4P/xFqLSQnZGpClg5lyRw34kU1l8sAzovFngN2Zn+RYkiW +y14I+uhVOFzroH2ymIYTwQ0crQJ4OYgtqzv0Rc+U2mTrTE+MRG28COsBqoMvWao9 +K4bhsP6qUmlTtMVFob4AP3eFqF3wg987zqeeCqaidAc7n5fuJHTDGNWFxQARAQAB +tCViaW5hcnlGYXRlIDxiaW5hcnlmYXRlQGdldG1vbmVyby5vcmc+iQJOBBMBCgA4 +FiEEgaxZH+nEtlxYBq/D8K9NRioL35IFAl3yDrwCGy8FCwkIBwIGFQoJCAsCBBYC +AwECHgECF4AACgkQ8K9NRioL35IT2w/+JaVcSVo3vXTpC9TOSxnyjFP1QuPVxn+K +P4la6MzTQlaZOSaoGuXra+82ghWqJGbFhGYmsXWQZtUYVwTMq9N9IUJ55clfkiV3 +HQNIDh/6zz+ibBzxfNLAPTbz0OQ+26dWRmBKkjpIs8KWyVIR/Ma/ax1No20fPKoN +cgRvOct/BKNoX2+PlHh/0tTTIjc0NqNP79ptNyJUUTGL1uZOgfTGa7/FRxil1xiC +JZfbZ2mbdesqjTtF3CI/ahhPKUEKy5Tciy5pnErALPk4rsxdLNdl3NOVrb5tYqGv +GqoQT3KsHkYVHvLCx0Ji2peFa6lF0PyJpudUzlo7L3nn39b8v9ebKd/fk2gthnWr +eoFxEFa6dI0x+HA47FE00I7ze56oH0zFVUhx+3QOHFG3u2mHV+H+GfKaXq9Nd5nU +Wk8AGKJ/xYgX7G0TnUek9aOzSt3dRd+xtSjmgwA3T7vCi6F67SthRpq7F4zuecPl ++Ww1MQhXSdIi6P3ll09gTCOICaqOxuOkMeFFrs2w4XEU517QUzy1VTAcF3gSy9TL +Xpy5jqssjLjvDAHs/Jm+iryQtn/t9YHr7/DP7AAr4+e7aOY3659jP7UiMZ3RKqY5 +/15Ru/usXXFyC56Epsv1spBCthimiK/RdX8XW7mG+BIQJXN337WdicXGWrKVOixj +eJ7ghiSblIeJAjMEEAEKAB0WIQTsEs+GtgjVvRuRCZGpI7HrTHj9WAUCXfOqVwAK +CRCpI7HrTHj9WPSaD/wMa2PWbRj06DUHiKRMrrQF9gkzox2ZOd4om4YIa3opA+GJ +CRl6nMHO7MdGTiun0/jBoT4oU4rqvPYcQlKOXUFYaMz/3vPYyTotC8n1Rvf+DhQZ +CIubdrRW0B5pqa3MiyJ9S/oMf/5otto3eBZSnfegI0J/fNzyDAyS2WIj/iyoAXGQ +yy9lY+KQnjchIHItxFm6RWgZmbO6xbhRq+LUa90jEiSul1PoE+ldpH2wUlRvjP78 +lL4YqWC7U6puhgGPlQV2gvFG4zscXVf8XXV65KysqrB5RI8wx1JxC1n1ocOqwYbw +0P6WIUGSr98cBjzB1l9zKhHumV7AM9FNUf4GD+Mgl8I8i/vOSz8kYRaWCcoRZvWa +pgSEG4uA94MFZUP9b789quoMUv8kTboHU1XFDo/n7K83AVBThcj0BMt7ueZmwjtu +2YVc3vGX7GvsUr8XXYn82hqnHcBoVLI4awGMzzAiOI1KSPtksGr36mYgOuY1jYme +435YlVxH4A/NC1qvOZYLq/6u1pB/BouBtlVxCvyjhlM3/r0PgDB0KveiR72LzVXH +TB0CzmyizZnVYbNR1PvsxFvyPxRLIIfN1nzXFYxwILASlg0zhdkQgFWkZ+WJWA6g +6HrAmrrA8iV7x6ITR6dXrT2fSLdQ+nYTntaXunUw74gGWYaMPe/GTvgcguFoU4kC +MwQQAQoAHRYhBOyzccBByW6YhVJsYUZMOzgUXzoUBQJd8647AAoJEEZMOzgUXzoU +fvgP/jI1w6DxC4qRlEwsIpjvyKQbCWYr9w817gKRPOI0ZHbjoL2zBnBwAWvgEKXV +QTOkldwpWVHvt5qZweO89bJxQFShFdQklEbVXNlf9hMZqbWHt6gyS/65caEUT/nB +pugg6Ug3MIeszin7S2sKeKj3BT2ynJioMeNNUPntogUfWlOuhkVr3JezHV3TtxKf +yrs49AAmoAHK4DH34H7d8HuVjjr0v7gQc2Yu3MeWdR4drGmyU3uNY44fGqU9pP99 +HNW2Ec//HFIz+qgBZ63ps6qyWGOjXyrpn6k9L1lg8zilC1sJxRtYTwp1a/57Kpyu +NkR1265o2zu/cvb5oQxjCWmQPiZWKsoguHxiA93g14QdmnugHPM5L6XCqDl1fkQo +dnCy5uqlqs9Ucz6O4C1GV4Av3YsMepS2Bw5uZFyujqPUdw82oWWOwZfO2xw7kKzN +zd8OnQAKj3piMh1vh375HE+HwdN74txKDcD/HqoTMwuaqX145csUvORoABA/wU4w +yBJiOhJeqvUaWAsG0q6XQMIDYjGQ0rrtY6Ba9E/1UP3D2CpjH5fZChV1fxhCaQOi +nMSCwgA3HRiT4CnTYScsbnk7RcdOlc2Zff+cxyZfMvqcGM4R50NDYu8YnVjnyajn +No5WhvYYPQIoYaDM8b+RHGw8Zw4WpQqAqli0VoNJAux19os9uQINBF3yDrwBEADI +LMleZnQ87iFofqyMm6wd+146dPC0xINz3+ExkCcFJgwiHL4o1sF8LtXjBXVuoc33 +vK7mdU1Fm3N9D4W5tkxEW3NdHICc7r3IHqThFv6lOdckKg1t9HKWzEjWkB4/4Epr +sj7Xc1owuQkOVtkaWiXzv8e/pYM4j+21V71+8b6fuInvA1nNufawzN4m8RDDgbe9 +I9SdndJoO0vnhUrv55APVG6KJBRjT+SGrxkEa3cckhpGUsQkW8KEx1VH0Wc4NBdD +xSRzW+ZUDyMzoUD/b5kxNCqinaM3P95jmNJgqB9l/3ZEPZ0G/hqDg6EhQMe8HrYT +F90Q+QqsX35RbBrdXxjbod+GZPSTg0PExm/hBrt2tTE/t/yovn22PZeJm6Mg9trN +x08b/LLZ1L040s8WqQQUK0btG065PIVRpyIlsUEPt7W+Qdj+eJdZAjIDr04qBxUQ +bwHhnxCYH4PNQq75y/w684ZWoVW3UJE1P8AcXKx/0gp4jjrZCJTAkzfUBEc6vy6H +hzYLpAar52ALBmFGPpkV4PJDdlFX4uijsjVVf23Hi3AQnGWkUw34bftJuHYlluGg +aGXVzISn8MxqcaLUiapxMMg3pZ40m5gCHG7/Gm74xKZCQiNrxCLHn7/rgNV17U6m +aWCNGcaiCXfkbtqUPJrzK6ulDAhg3700Ok5RlVOtAwARAQABiQRsBBgBCgAgFiEE +gaxZH+nEtlxYBq/D8K9NRioL35IFAl3yDrwCGy4CQAkQ8K9NRioL35LBdCAEGQEK +AB0WIQStVkzajxZlrOeLXf0lk4OOq7H2VQUCXfIOvAAKCRAlk4OOq7H2VfMbD/oC +NxmlGhAJ2okXzcpFURNyIXTCt/Dlys+lwH2mwCjcniUcA5KsT1mV1Sb4SxsBsX18 +Fhol2wUUI/B3ELz15rOYTJq7u8TPpPnGsHtALgNwN1o90EYMRpMl7jiPJLKSWVA1 +spp7FBginaP/jkQwtcy+sHiJvmEiY2kmeB27aAA5bM8bafjzpqQ6Io/iguRVYex1 +ypH5D1Dq5bkrk8Fd+3bs2dcuB9jYy0lQlaYl8bJ4I4ZOgdO4nTKq0U0RTfYAR+wH +vpfzKrYQD7ZnX6mllXyvC+L4CujngND3GiavoOPX9JHJURrYC1WiMyKPb8nk1CrF +cIfLxFYqy535suen35T23G8B0ZY/i48ccRjI7ZrickIBSjJZmJLYevsejz+T1vqf +348WCAYisi44NL6vwAoBkvYLRw45YhzAZDZoKCv+ke7zjZAO3lnh+rR3j+QCe9gH +22gqJbhGpU54EpYYVx3GxIfaqQeHa5H02rSW8AUGyGjgdTo7x5NCAJjgtYyR08k1 +19aghaA8yi1wsbmiuz9viH2xJ1vPTgLLQE5qv0ed0uOGEeDtRZSfaWeYwk+qhier +bY3R3dzVOyan6W8uWaVgE8Ch9LttYAKa9qLoLDQE9tQLU6t1gklFfQiaLBzyPIgT +Ct6Aezwlcp2cDhxYazXjyxRTYtfshVPTXJMBF+0o+W78D/9c3f9Yv61Z1MktrHIt +5REIj5YGT3bAeDnUoGE+k+QzNx8856Hu5j4prQGav8F8+V69iqwVciK0sBoFUgI5 +JJizfVbEfmbZgl5c4S8OhApNv0Gbhj9U/XftM2Pz/m/QfsB5Az1jQ8jQGHULqEWB +d4Z+Cnt0LVmGxJkz0prlaUxrGp8NyO3A1RxWJbP6P9fPIJFFwDSer2xeq79dZGFQ +CLFxA3d8kjr9nioGZFaGK/LfbVcZBBM6VI0ziKRnWPeS3Zi50oy7+/gk7HgUvUmT +8uenHLE/kmfHg1lAycQ/TDY7/sP1Wtnbr72HEFOKDIvFIF8zaWwpaYSWrj6NRTJE +b9wa2Y5TervcvOcgzhleJP3PfDrahlfgUFtD/919cuQNG416tedumAaAlLpdBEUH +9F9FQ2Hzp8Pxm3N6LzbtgDoc6NXY2C08NPI+IwHysNIXzaH3jJ53TV8pCNvvd4Ok +Sk2fOVG/fDuZibwthAFjR3NIVJIP19xyxkalbeuLT+IUSzgN2ndojtK9eh7awMii +S/AkjI5G33OKYP0WZx+6o0i21rWXIepWgm74wMa9t9NCnNY50NIvesnG1AaXNKJ5 +D2o1uLXbwis6Fm2sMNutCkQFYsk/IFWC/Y8DsJzwkOty8gl3Q0NQi64IXpFGzUuG +lmE0kDaYQqoBKXNXcybElBID8Q== +=tkJZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/monero/handlers/main.yml b/ansible/roles/monero/handlers/main.yml new file mode 100644 index 0000000..15f7e63 --- /dev/null +++ b/ansible/roles/monero/handlers/main.yml @@ -0,0 +1,17 @@ +--- +# Ungated, and that is the whole point. Across this estate the restart handlers +# were written with `when: uptime_kuma_enabled | default(false)`, which made +# them inert: Fulcrum's config changes landed on disk and silently never took +# effect for months. Three tasks notify this handler - the config, the systemd +# unit and the binary itself - and all three need it to actually fire. +- name: Restart monerod + systemd: + name: monerod + state: restarted + daemon_reload: yes + +- name: Restart monerod health check timer + systemd: + name: monerod-healthcheck.timer + state: restarted + daemon_reload: yes diff --git a/ansible/roles/monero/tasks/configure.yml b/ansible/roles/monero/tasks/configure.yml new file mode 100644 index 0000000..b8af394 --- /dev/null +++ b/ansible/roles/monero/tasks/configure.yml @@ -0,0 +1,55 @@ +--- +# Directory layout follows upstream's systemd guide: +# https://docs.getmonero.org/running-node/monerod-systemd/ +# The data directory is not here - it is the mount point, handled in storage.yml. + +- name: Create the Monero config directory + ansible.builtin.file: + path: "{{ monero_conf_dir }}" + state: directory + owner: root + group: "{{ monero_group }}" + mode: '0750' + +- name: Create the Monero log directory + ansible.builtin.file: + path: "{{ monero_log_dir }}" + state: directory + owner: "{{ monero_user }}" + group: "{{ monero_group }}" + mode: '0750' + +- name: Assert the RPC credentials are set + ansible.builtin.assert: + that: + - monero_rpc_user | length > 0 + - monero_rpc_password | length > 0 + quiet: true + fail_msg: >- + monero_rpc_user and monero_rpc_password must be set. They are the only + authentication in front of the restricted RPC that Caddy publishes to the + internet, and monerod started without them would serve it to anyone. + +- name: Create monerod.conf + ansible.builtin.template: + src: monerod.conf.j2 + dest: "{{ monero_conf_dir }}/monerod.conf" + owner: root + group: "{{ monero_group }}" + mode: '0640' + # monerod.conf carries the RPC login, which is the only authentication in + # front of the endpoint Caddy publishes. `--diff` prints rendered content, so + # running with --diff would put it on the terminal and into any log capturing + # it. Suppressed by default; pass -e monero_reveal_config=true when you + # genuinely need the diff. This is a task keyword, not a module parameter. + diff: "{{ monero_reveal_config | default(false) | bool }}" + notify: Restart monerod + +- name: Create the monerod systemd unit + ansible.builtin.template: + src: monerod.service.j2 + dest: /etc/systemd/system/monerod.service + owner: root + group: root + mode: '0644' + notify: Restart monerod diff --git a/ansible/roles/monero/tasks/healthcheck.yml b/ansible/roles/monero/tasks/healthcheck.yml new file mode 100644 index 0000000..794ccba --- /dev/null +++ b/ansible/roles/monero/tasks/healthcheck.yml @@ -0,0 +1,58 @@ +--- +# "Is monerod actually answering", as opposed to infra/401's "is the unit +# active". A daemon can be perfectly `active` and wedged. + +- name: Install the health check's dependencies + apt: + name: + - curl + - jq + state: present + +- name: Create the monerod health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/monerod-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +# 0600: this unit carries both the Gatus bearer token and the RPC password. +# They live here rather than in the script so the script itself stays readable +# and credential-free. +- name: Create the monerod health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/monerod-healthcheck.service + owner: root + group: root + mode: '0600' + notify: Restart monerod health check timer + +- name: Create the monerod health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/monerod-healthcheck.timer + owner: root + group: root + mode: '0644' + notify: Restart monerod health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated: enabling a timer is deployment, not monitoring. +- name: Enable and restart the monerod health check timer + systemd: + name: monerod-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +# Arms the timer and smoke-tests the check. Restarting the timer alone is not +# enough with OnBootSec + OnUnitActiveSec - see roles/bitcoin_knots/README.md. +- name: Run the monerod health check once to arm the timer + ansible.builtin.command: systemctl start monerod-healthcheck.service + changed_when: false diff --git a/ansible/roles/monero/tasks/install.yml b/ansible/roles/monero/tasks/install.yml new file mode 100644 index 0000000..0175680 --- /dev/null +++ b/ansible/roles/monero/tasks/install.yml @@ -0,0 +1,232 @@ +--- +# Install monerod from the published release, and prove the bytes are the ones +# binaryFate signed before they are allowed anywhere near /usr/local/bin. +# +# These tasks are the security control of this role. Do not "simplify" them. +# +# Why the published binary and not a build from source: the reproducible-build +# system that yields a byte-identical tarball (contrib/guix) exists only on +# monero master. The v0.18.x release branch still uses Gitian, which wants an +# Ubuntu 18.04 host and Python 2. A plain `make release-static` is not +# reproducible, so it would give us a self-built binary whose hash matches +# nothing - neither upstream's nor anyone else's. The signature chain below is +# the strongest claim actually available for this release. See README.md. + +- name: Install the tools the verification chain needs + apt: + name: + - gnupg + - curl + - bzip2 + state: present + update_cache: yes + +- name: Check whether monerod is already installed + ansible.builtin.stat: + path: "{{ monero_bin_dir }}/monerod" + register: monero_binary + +- name: Read the installed monerod version + ansible.builtin.command: "{{ monero_bin_dir }}/monerod --version" + register: monero_installed_version + changed_when: false + failed_when: false + check_mode: false + when: monero_binary.stat.exists + +- name: Decide whether monerod needs installing + ansible.builtin.set_fact: + monero_needs_install: >- + {{ not monero_binary.stat.exists + or monero_version not in (monero_installed_version.stdout | default('')) }} + +# Everything below runs only for a fresh install or a version bump. + +# A throwaway keyring. The verification must not depend on, or add to, whatever +# happens to be in root's keyring already. +- name: Create a temporary GnuPG home for the verification + ansible.builtin.tempfile: + state: directory + suffix: monero-gpg + register: monero_gnupghome + when: monero_needs_install | bool + +- name: Copy binaryFate's signing key to the host + ansible.builtin.copy: + src: binaryfate.asc + dest: "{{ monero_gnupghome.path }}/binaryfate.asc" + mode: '0644' + when: monero_needs_install | bool + +- name: Import binaryFate's signing key + ansible.builtin.command: "gpg --batch --import {{ monero_gnupghome.path }}/binaryfate.asc" + environment: + GNUPGHOME: "{{ monero_gnupghome.path }}" + register: monero_key_import + changed_when: "'imported' in monero_key_import.stderr" + when: monero_needs_install | bool + +- name: Read the fingerprints of the imported key + ansible.builtin.shell: >- + set -o pipefail && + gpg --batch --with-colons --fingerprint | awk -F: '/^fpr:/{print $10}' + args: + executable: /bin/bash + environment: + GNUPGHOME: "{{ monero_gnupghome.path }}" + register: monero_key_fingerprints + changed_when: false + when: monero_needs_install | bool + +# The vendored file could be swapped in a bad pull request. The fingerprint is +# the thing being trusted, so it is checked, not assumed. +- name: Assert the imported key is binaryFate's + ansible.builtin.assert: + that: + - monero_signing_fingerprint in monero_key_fingerprints.stdout_lines + quiet: true + fail_msg: >- + files/binaryfate.asc does not contain the expected key + {{ monero_signing_fingerprint }}. Refusing to go on. + when: monero_needs_install | bool + +- name: Download the signed hashes.txt + ansible.builtin.get_url: + url: "{{ monero_hashes_url }}" + dest: "{{ monero_gnupghome.path }}/hashes.txt" + mode: '0644' + force: yes + when: monero_needs_install | bool + +# One command that both verifies the signature and writes out ONLY the verified +# plaintext. Grepping the clearsigned file directly would search text that has +# not been through the signature check. +- name: Verify hashes.txt and extract the verified plaintext + ansible.builtin.command: >- + gpg --batch --status-fd 2 + --output {{ monero_gnupghome.path }}/hashes-verified.txt + --decrypt {{ monero_gnupghome.path }}/hashes.txt + environment: + GNUPGHOME: "{{ monero_gnupghome.path }}" + register: monero_hashes_verify + changed_when: false + failed_when: false + when: monero_needs_install | bool + +# Greping for "Good signature" - which is what the bitcoin_knots role does - +# passes for ANY key in the keyring. VALIDSIG names the key, and the line +# carries both the signing subkey and the primary key fingerprint, so checking +# that our fingerprint appears on it survives a subkey rotation. +- name: Assert hashes.txt carries a valid signature from binaryFate + ansible.builtin.assert: + that: + - monero_hashes_verify.rc == 0 + - monero_hashes_verify.stderr is search('VALIDSIG[^\n]*' ~ monero_signing_fingerprint) + quiet: true + fail_msg: >- + hashes.txt did not verify against {{ monero_signing_fingerprint }}. + This is the whole security control of this role - aborting. + gpg said: {{ monero_hashes_verify.stderr | default('') }} + when: monero_needs_install | bool + +- name: Download the monerod release tarball + ansible.builtin.get_url: + url: "{{ monero_url }}" + dest: "/tmp/{{ monero_tarball }}" + mode: '0644' + checksum: "sha256:{{ monero_sha256 }}" + force: yes + when: monero_needs_install | bool + +- name: Compute the SHA256 of the downloaded tarball + ansible.builtin.stat: + path: "/tmp/{{ monero_tarball }}" + checksum_algorithm: sha256 + get_checksum: yes + register: monero_tarball_stat + when: monero_needs_install | bool + +# The check that actually ties these bytes to binaryFate's signature. The +# get_url checksum above only ties them to a string in our own repo; this ties +# them to the signed list. Both the hash AND the filename must be on one line, +# so a hash belonging to a different artefact cannot satisfy it. +- name: Look up the downloaded tarball's checksum in the signed hashes + ansible.builtin.command: + argv: + - grep + - -F + - -x + - "{{ monero_tarball_stat.stat.checksum }} {{ monero_tarball }}" + - "{{ monero_gnupghome.path }}/hashes-verified.txt" + register: monero_hash_match + changed_when: false + failed_when: false + when: monero_needs_install | bool + +- name: Assert the tarball is the one binaryFate signed + ansible.builtin.assert: + that: + - monero_hash_match.rc == 0 + quiet: true + fail_msg: >- + {{ monero_tarball }} hashes to {{ monero_tarball_stat.stat.checksum }}, + which is not the value signed for that filename in hashes.txt. + Do not install this file. + when: monero_needs_install | bool + +- name: Create a temporary extraction directory + ansible.builtin.tempfile: + state: directory + suffix: monero-extract + register: monero_extract_dir + when: monero_needs_install | bool + +- name: Extract the monerod tarball + ansible.builtin.unarchive: + src: "/tmp/{{ monero_tarball }}" + dest: "{{ monero_extract_dir.path }}" + remote_src: yes + when: monero_needs_install | bool + +# By find rather than by a hardcoded path: the tarball unpacks to +# monero-x86_64-linux-gnu-v/, which is not the name of the tarball. +- name: Locate the extracted monerod binary + ansible.builtin.find: + paths: "{{ monero_extract_dir.path }}" + patterns: monerod + recurse: yes + file_type: file + register: monero_extracted + when: monero_needs_install | bool + +- name: Assert the archive contained monerod + ansible.builtin.assert: + that: + - monero_extracted.files | length > 0 + quiet: true + fail_msg: "No monerod binary in {{ monero_tarball }}" + when: monero_needs_install | bool + +# Only monerod. The wallet and blockchain utilities are in the same tarball if +# they are ever wanted; a node does not need them and every binary installed is +# surface that has to be kept current. +- name: Install monerod + ansible.builtin.copy: + src: "{{ monero_extracted.files[0].path }}" + dest: "{{ monero_bin_dir }}/monerod" + remote_src: yes + owner: root + group: root + mode: '0755' + when: monero_needs_install | bool + notify: Restart monerod + +- name: Clean up the download and verification artefacts + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "/tmp/{{ monero_tarball }}" + - "{{ monero_extract_dir.path | default('/nonexistent') }}" + - "{{ monero_gnupghome.path | default('/nonexistent') }}" + when: monero_needs_install | bool diff --git a/ansible/roles/monero/tasks/main.yml b/ansible/roles/monero/tasks/main.yml new file mode 100644 index 0000000..699cf3b --- /dev/null +++ b/ansible/roles/monero/tasks/main.yml @@ -0,0 +1,15 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is the main verification tool in this repo. +# +# The order is load-bearing: +# user must exist before anything chowns to it +# storage the data directory IS the mount point of the 500 GB disk, so if the +# disk is not there the daemon must not start and quietly begin +# filling a 10 GB root filesystem with a 300 GB blockchain +- ansible.builtin.import_tasks: user.yml +- ansible.builtin.import_tasks: storage.yml +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: configure.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/monero/tasks/service.yml b/ansible/roles/monero/tasks/service.yml new file mode 100644 index 0000000..ea11f62 --- /dev/null +++ b/ansible/roles/monero/tasks/service.yml @@ -0,0 +1,35 @@ +--- +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start monerod + systemd: + name: monerod + enabled: yes + state: started + +- name: Flush handlers so monerod is running before we ask it anything + meta: flush_handlers + +# Generous, and non-fatal. On a first start monerod initialises LMDB on a fresh +# 500 GB disk before it opens the RPC port; on a later start it is quick. A +# timeout here is not a deployment failure, and the health check is the thing +# that decides whether the node is actually well. +- name: Wait for the monerod RPC port + ansible.builtin.wait_for: + port: "{{ monero_rpc_port }}" + host: "{{ monero_rpc_bind_ip }}" + delay: 5 + timeout: 180 + ignore_errors: yes + +- name: Check the monerod service status + systemd: + name: monerod + register: monero_service_status + changed_when: false + +- name: Display the monerod service status + ansible.builtin.debug: + msg: "monerod is {{ 'running' if monero_service_status.status.ActiveState == 'active' else 'NOT running' }}" diff --git a/ansible/roles/monero/tasks/storage.yml b/ansible/roles/monero/tasks/storage.yml new file mode 100644 index 0000000..beef59c --- /dev/null +++ b/ansible/roles/monero/tasks/storage.yml @@ -0,0 +1,133 @@ +--- +# The 500 GB data disk. +# +# knots-box does this by hand: /mnt/knots_data holds ~875 GB of blockchain and +# appears nowhere in Ansible, so a rebuild of that host would come up with the +# chain silently missing. This role does it in code instead. + +- name: Check the Monero data device + ansible.builtin.stat: + path: "{{ monero_data_device }}" + register: monero_device + +- name: Find the device backing the root filesystem + ansible.builtin.command: findmnt -no SOURCE / + register: monero_root_source + changed_when: false + check_mode: false + +# A guard, not ceremony. mkfs on the wrong device is unrecoverable, and +# /dev/sdX names are assigned by the kernel at boot - the whole reason +# everything below this point works by UUID. +- name: Assert the data device is a real block device and is not the root disk + ansible.builtin.assert: + that: + - monero_device.stat.exists + - monero_device.stat.isblk | default(false) + - not monero_root_source.stdout.startswith(monero_data_device) + quiet: true + fail_msg: >- + {{ monero_data_device }} is not a usable data device, or it is the disk + the root filesystem lives on ({{ monero_root_source.stdout }}). + Refusing to touch it. Check `lsblk` on the host. + +- name: Read any existing filesystem on the data device + ansible.builtin.command: "blkid -o value -s TYPE {{ monero_data_device }}" + register: monero_existing_fs + changed_when: false + failed_when: false + check_mode: false + +# community.general.filesystem is idempotent and will not rebuild a filesystem +# that is already the requested type, but it would happily create one on a disk +# holding something it does not recognise. Say so out loud instead. +- name: Refuse to reformat a disk that already carries a different filesystem + ansible.builtin.assert: + that: + - monero_existing_fs.stdout | trim in ['', monero_data_fstype] + quiet: true + fail_msg: >- + {{ monero_data_device }} already holds a '{{ monero_existing_fs.stdout | trim }}' + filesystem, not '{{ monero_data_fstype }}'. Refusing to overwrite it. + +- name: Create the filesystem on the data device + community.general.filesystem: + fstype: "{{ monero_data_fstype }}" + dev: "{{ monero_data_device }}" + opts: "{{ monero_data_mkfs_opts }}" + +- name: Read the filesystem UUID + ansible.builtin.command: "blkid -o value -s UUID {{ monero_data_device }}" + register: monero_fs_uuid + changed_when: false + failed_when: false + check_mode: false + +# Everything past this point mounts by UUID, so there is nothing sensible to do +# without one. On a --check run against a disk that has no filesystem yet this +# is expected and unavoidable: check mode did not create the filesystem, so +# there is no UUID to read. Say that, rather than leaving a bare blkid rc=2. +- name: Assert the UUID looks like a UUID + ansible.builtin.assert: + that: + - monero_fs_uuid.stdout | trim | length == 36 + quiet: true + fail_msg: >- + No filesystem UUID on {{ monero_data_device }}. + {% if ansible_check_mode %}This is expected under --check when the disk + has not been formatted yet - check mode cannot create the filesystem, so + it cannot dry-run the mount either. Run without --check once, then + --check is meaningful from then on.{% else %}mkfs appears to have failed; + check `lsblk` and `blkid` on the host.{% endif %} + +# Same owner and mode as the "Own the mounted data directory" task below, and +# that is deliberate rather than duplication. Before the disk is mounted this +# creates the mount point; afterwards it targets the mounted filesystem and is a +# no-op. Giving it root:root 0755 here - the obvious thing to write - made the +# two tasks disagree, and each run flipped the directory back and forth. +- name: Create the Monero data directory as a mount point + ansible.builtin.file: + path: "{{ monero_data_dir }}" + state: directory + owner: "{{ monero_user }}" + group: "{{ monero_group }}" + mode: '0750' + +# By UUID, never by /dev/sdb. nofail so a missing disk does not drop a headless +# VM into an emergency shell - monerod is stopped from starting by +# RequiresMountsFor= in its unit instead, which the systemd monitor then reports. +# noatime because LMDB touches a lot of pages and none of it wants an atime write. +- name: Mount the data disk by UUID + ansible.posix.mount: + path: "{{ monero_data_dir }}" + src: "UUID={{ monero_fs_uuid.stdout | trim }}" + fstype: "{{ monero_data_fstype }}" + opts: defaults,noatime,nofail + state: mounted + +# AFTER the mount, never before: chowning the mount point first sets permissions +# on the directory the mount then hides, and the blockchain lands on a root-owned +# filesystem that monerod cannot write to. +- name: Own the mounted data directory + ansible.builtin.file: + path: "{{ monero_data_dir }}" + state: directory + owner: "{{ monero_user }}" + group: "{{ monero_group }}" + mode: '0750' + +- name: Confirm the data directory is a mount point and not the root filesystem + ansible.builtin.command: "findmnt -no SOURCE --target {{ monero_data_dir }}" + register: monero_mounted_source + changed_when: false + check_mode: false + +- name: Assert the blockchain will land on the data disk + ansible.builtin.assert: + that: + - monero_mounted_source.stdout | trim != monero_root_source.stdout | trim + quiet: true + fail_msg: >- + {{ monero_data_dir }} resolves to {{ monero_mounted_source.stdout | trim }}, + which is the root filesystem. The 500 GB disk is not mounted and a full + sync would fill the 10 GB root disk. diff --git a/ansible/roles/monero/tasks/user.yml b/ansible/roles/monero/tasks/user.yml new file mode 100644 index 0000000..269ed53 --- /dev/null +++ b/ansible/roles/monero/tasks/user.yml @@ -0,0 +1,24 @@ +--- +# The service account, created before anything that chowns to it. +# +# This is its own file because two later steps both depend on it and neither +# owns it: storage.yml chowns the mount point to this user, and install.yml +# installs a binary that runs as it. Creating it inside either one makes the +# other's ordering an accident. + +- name: Create the monero system group + group: + name: "{{ monero_group }}" + system: yes + state: present + +- name: Create the monero system user + user: + name: "{{ monero_user }}" + group: "{{ monero_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ monero_data_dir }}" + create_home: no + comment: "Monero node daemon" + state: present diff --git a/ansible/roles/monero/templates/healthcheck.service.j2 b/ansible/roles/monero/templates/healthcheck.service.j2 new file mode 100644 index 0000000..7619581 --- /dev/null +++ b/ansible/roles/monero/templates/healthcheck.service.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=Monero Node Health Check +After=network.target monerod.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/monerod-healthcheck-push.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +Environment=MONERO_RPC_HOST={{ monero_rpc_bind_ip }} +Environment=MONERO_RPC_PORT={{ monero_rpc_port }} +Environment=MONERO_RPC_USER={{ monero_rpc_user }} +Environment=MONERO_RPC_PASSWORD={{ monero_rpc_password }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/monero/templates/healthcheck.sh.j2 b/ansible/roles/monero/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..7aebd80 --- /dev/null +++ b/ansible/roles/monero/templates/healthcheck.sh.j2 @@ -0,0 +1,70 @@ +#!/bin/bash +# Monero node health check — managed by Ansible (roles/monero) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed monerod-healthcheck.service +# Reporting anywhere else is optional and generic. +# +# WHAT THIS DELIBERATELY DOES NOT CHECK: whether the node is synchronised. +# A full initial sync takes days. A check that required `synchronized: true` +# would report DOWN for every one of those days, which is not a monitor, it is +# an alarm with the batteries taken out. Answering RPC is the health question; +# sync progress is reported in the message so it is visible without being +# alertable. + +RPC_HOST="${MONERO_RPC_HOST:-127.0.0.1}" +RPC_PORT="${MONERO_RPC_PORT:-18081}" +RPC_USER="${MONERO_RPC_USER:-}" +RPC_PASSWORD="${MONERO_RPC_PASSWORD:-}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +report() { + local status=$1 + local msg=$2 + + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded_msg="${msg// /%20}" + + # Gatus external endpoint: a POST with a bearer token. + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${encoded_msg}" || true +} + +# --digest, not --user: monerod's --rpc-login is HTTP Digest auth, so curl has +# to answer the 401 challenge rather than send credentials up front. +response=$(curl -s --max-time 30 --digest -u "${RPC_USER}:${RPC_PASSWORD}" \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","id":"healthcheck","method":"get_info"}' \ + "http://${RPC_HOST}:${RPC_PORT}/json_rpc" 2>/dev/null) + +if [ -z "$response" ]; then + report "down" "monerod RPC not responding" + exit 1 +fi + +status=$(echo "$response" | jq -r '.result.status // empty' 2>/dev/null) + +if [ "$status" != "OK" ]; then + report "down" "monerod RPC status: ${status:-unparseable}" + exit 1 +fi + +height=$(echo "$response" | jq -r '.result.height // 0' 2>/dev/null) +target=$(echo "$response" | jq -r '.result.target_height // 0' 2>/dev/null) +synced=$(echo "$response" | jq -r '.result.synchronized // false' 2>/dev/null) + +# monerod reports target_height=0 once it has caught up, so show the height we +# actually reached rather than a confusing "of 0". +if [ "$target" = "0" ]; then + report "up" "OK height=${height} synced=${synced}" +else + report "up" "OK height=${height}/${target} synced=${synced}" +fi +exit 0 diff --git a/ansible/roles/monero/templates/healthcheck.timer.j2 b/ansible/roles/monero/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..8c850a9 --- /dev/null +++ b/ansible/roles/monero/templates/healthcheck.timer.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Monero Node Health Check Timer +# Deliberately NO `Requires=monerod.service`. Requires on a timer stops the +# timer when the required unit stops - "if the thing I am watching goes down, +# stop watching it" - and leaves nothing to re-arm when the service returns. +# Fulcrum's timer carried it and sat active+enabled for seven months without +# firing once. bitcoin_knots' timer still carries it; do not copy that. + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/monero/templates/monerod.conf.j2 b/ansible/roles/monero/templates/monerod.conf.j2 new file mode 100644 index 0000000..9043ca8 --- /dev/null +++ b/ansible/roles/monero/templates/monerod.conf.j2 @@ -0,0 +1,46 @@ +# monerod configuration - managed by Ansible (roles/monero) +# +# Config-file syntax is the command-line options with the leading -- removed; +# valueless flags take =1. https://docs.getmonero.org/interacting/monero-config-file/ + +data-dir={{ monero_data_dir }} +log-file={{ monero_log_dir }}/monerod.log +log-level=0 +max-log-file-size=104850000 + +# --- P2P: outbound only ----------------------------------------------------- +# Nothing forwards {{ monero_p2p_port }} to this box, so the node dials out to +# peers and never receives. It syncs and serves us completely; it simply does +# not relay for strangers. no-igd follows: with nothing forwarded there is +# nothing for UPnP to do except announce us to the router. +p2p-bind-ip=0.0.0.0 +p2p-bind-port={{ monero_p2p_port }} +no-igd=1 +out-peers={{ monero_out_peers }} +in-peers={{ monero_in_peers }} + +# --- Unrestricted RPC: loopback only ---------------------------------------- +# Administration and the health check. Never leaves the box. +rpc-bind-ip={{ monero_rpc_bind_ip }} +rpc-bind-port={{ monero_rpc_port }} + +# --- Restricted RPC: the only surface anything off-box touches -------------- +# A SECOND server, view-only, reached over the tailnet by Caddy on the edge +# host. This is rpc-restricted-bind-port and deliberately NOT `restricted-rpc=1`: +# that flag restricts the loopback server too, and then we could not ask our own +# node anything useful. +# +# Bound to 0.0.0.0 and reachable only over the tailnet - Tailscale's ts-input +# chain sits ahead of UFW, so nothing on the LAN or the internet reaches this +# port without going through Caddy. confirm-external-bind is monerod's +# "yes, I meant to do that" for a non-loopback bind. +rpc-restricted-bind-ip=0.0.0.0 +rpc-restricted-bind-port={{ monero_restricted_rpc_port }} +confirm-external-bind=1 + +# HTTP Digest, and it guards BOTH servers above - core_rpc_server::init passes +# the same login to each. This is what Feather sends as the node credentials. +rpc-login={{ monero_rpc_user }}:{{ monero_rpc_password }} + +# Nothing subscribes to ZMQ here, so it is one less listener. +no-zmq=1 diff --git a/ansible/roles/monero/templates/monerod.service.j2 b/ansible/roles/monero/templates/monerod.service.j2 new file mode 100644 index 0000000..929d2ca --- /dev/null +++ b/ansible/roles/monero/templates/monerod.service.j2 @@ -0,0 +1,38 @@ +[Unit] +Description=Monero Daemon +Documentation=https://docs.getmonero.org/running-node/monerod-systemd/ +After=network-online.target +Wants=network-online.target +# The blockchain lives on a separate 500 GB disk mounted at {{ monero_data_dir }}. +# Without this line a failed mount would not stop monerod - it would start, find +# an empty directory, and begin writing a ~300 GB chain onto the 10 GB root +# filesystem. This turns that silent disaster into a unit that refuses to start, +# which the systemd monitor reports within five minutes. +RequiresMountsFor={{ monero_data_dir }} + +[Service] +Type=simple +User={{ monero_user }} +Group={{ monero_group }} +ExecStart={{ monero_bin_dir }}/monerod --config-file {{ monero_conf_dir }}/monerod.conf --non-interactive + +Restart=always +RestartSec=30 +# monerod flushes LMDB on shutdown and that is not instant on a full chain. +# Killing it mid-flush is how the database gets corrupted and resynced from zero. +TimeoutStopSec=300 + +StandardOutput=journal +StandardError=journal + +# Hardening: the daemon only ever writes to its own data and log directories. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={{ monero_data_dir }} {{ monero_log_dir }} + +LimitNOFILE=65535 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/services/headscale/templates/acl.json.j2 b/ansible/services/headscale/templates/acl.json.j2 index 98ada80..6ab6fa5 100644 --- a/ansible/services/headscale/templates/acl.json.j2 +++ b/ansible/services/headscale/templates/acl.json.j2 @@ -42,6 +42,7 @@ "tag:fulcrum-box:50002", "tag:mempool-box:8080", "tag:memos-box:5230", + "tag:monerod-box:18089", "tag:arbret-staging-box:80", "tag:stalwart-box:25", "tag:stalwart-box:465", @@ -108,6 +109,9 @@ "tag:mempool-box": [ "{{ headscale_namespace }}@" ], + "tag:monerod-box": [ + "{{ headscale_namespace }}@" + ], "tag:nodito": [ "{{ headscale_namespace }}@" ], diff --git a/ansible/services/monero/deploy_monero_playbook.yml b/ansible/services/monero/deploy_monero_playbook.yml new file mode 100644 index 0000000..ca90693 --- /dev/null +++ b/ansible/services/monero/deploy_monero_playbook.yml @@ -0,0 +1,39 @@ +--- +# Monero: a full node, reachable from Feather over the WAN. +# +# The chain lives on a dedicated 500 GB disk mounted at /var/lib/monero; the +# role creates the filesystem and the fstab entry, and monerod's unit carries +# RequiresMountsFor so it cannot start without it. An initial sync takes days +# and nothing here touches the data directory afterwards. +# +# Exposure is one HTTPS vhost and no new open ports. See roles/monero/README.md. +- name: Deploy the Monero node + hosts: monero + become: yes + vars: + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_monerod/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - monero + +- name: Publish the restricted Monero RPC from the edge host + hosts: edge + become: yes + tasks: + # No basic auth here on purpose. monerod's own --rpc-login is HTTP Digest + # and guards this endpoint; Caddy's basic_auth would put a second, Basic + # challenge in front of it that Feather would answer with the wrong scheme. + # Caddy's job is TLS and routing, nothing else. + - name: Expose the restricted Monero RPC over HTTPS + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: monero + caddy_site_domain: "{{ subdomains.monero }}.{{ root_domain }}" + caddy_site_upstream: "{{ hostvars['monerod_box_local'].ansible_host }}:{{ hostvars['monerod_box_local'].monero_restricted_rpc_port }}" + caddy_site_resolvers: "100.100.100.100" + + # The per-site validate cannot catch a conflict between sites. + - name: Validate the whole Caddyfile + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + changed_when: false diff --git a/ansible/site.yml b/ansible/site.yml index 35f5867..c777577 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -53,6 +53,7 @@ - import_playbook: services/fulcrum/deploy_fulcrum_playbook.yml - import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml - import_playbook: services/mempool/deploy_mempool_playbook.yml +- import_playbook: services/monero/deploy_monero_playbook.yml - import_playbook: services/memos/deploy_memos_playbook.yml - import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml - import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml diff --git a/tofu/nodito/vms.auto.tfvars b/tofu/nodito/vms.auto.tfvars index a9ea0a5..8e50853 100644 --- a/tofu/nodito/vms.auto.tfvars +++ b/tofu/nodito/vms.auto.tfvars @@ -48,6 +48,20 @@ vms = { disk_size_gb = 30 ipconfig0 = "ip=dhcp" }, + # disk_size_gb is the ROOT disk only. monerod-box also carries a second + # 500 GB disk (scsi1) for the blockchain, attached in the Proxmox UI and + # therefore invisible here - the vm resource sets ignore_changes = all, so + # tofu neither created it nor would notice if it vanished. It is mounted at + # /var/lib/monero by ansible/roles/monero (tasks/storage.yml), by UUID, and + # monerod's unit refuses to start without it. + "monerod-box" = { + vmid = 2400 + name = "monerod-box" + cores = 4 + memory_mb = 2048 + disk_size_gb = 10 + ipconfig0 = "ip=dhcp" + }, "memos-box" = { vmid = 8100 name = "memos-box" From e791ffb18313d68c00b06176a4c8c68058f01588 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 20 Sep 2026 17:06:30 +0200 Subject: [PATCH 3/3] xmrblocks: the Monero explorer, built off-box because 2 GB cannot compile it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Onion Monero Blockchain Explorer on monerod-box, published at exmrplore.contrapeso.xyz. Ten Gatus endpoints green across the node and the explorer. ── It needs a compiled monero source tree, not a running monerod ─────────── xmrblocks is not an RPC client. cmake/FindMonero.cmake imports 21 STATIC libraries out of a monero build tree and includes its headers, because it opens the LMDB directly. The release tarball we install for the daemon ships only executables - no .a files, no headers - so there is nothing on the box to link against, and building it means compiling monero first. Measured on Debian 13, building only the libraries it links: peak RSS, one cc1plus ~3.2 GB peak RSS at -j4 ~5.5 GB wall time at -j4 167 s source + build tree 195 MB toolchain 1.6 GB That rules the box out at ANY parallelism: 768 MiB and 1800 MiB were both OOM-killed at -j1 and -j2, because a single translation unit over 3 GB is not something lower -j works around. My first estimate of "~10 GB of disk" was wrong by two orders of magnitude - that figure describes upstream's full `make`, which also builds monerod, both wallets and the test suite, none of it linked here. Disk was never the constraint. So scripts/build-xmrblocks.sh does the build in a debian:13 container. An artefact only one machine can produce is one nobody can audit; this way any host with Docker and ~6 GB reproduces it. It verifies the monero source against binaryFate's signature exactly as roles/monero does for the daemon, and the role asserts the resulting sha256 ON THE CONTROL NODE before copying a single byte. ── Three upstream problems, none documented anywhere ─────────────────────── Found by building, not by reading: 1. libasio-dev is absent from upstream's dependency list. That list is Ubuntu-based; Crow wants standalone asio.hpp, not boost/asio.hpp, and on Debian 13 the build dies outright. 2. The explorer does not compile against monero v0.18.5.1. v0.18.5.0 changed get_pruned_transaction_hash from returning the hash to returning bool with an out-param, and src/page.h still calls the old form. Upstream's Dockerfile pins v0.18.4.0, which is why nobody has hit it. Patched from an exact-line anchor, so if upstream fixes it the build STOPS rather than quietly doing something else. We patch instead of building against 0.18.4.0 because the signed hashes.txt only covers the current release - using the older source would mean dropping the provenance chain. LMDB is VERSION 5 in both, so either reads the other's database. 3. FindMonero.cmake hunts for miniupnpc inside the monero build tree with NO_DEFAULT_PATH, but monero stopped vendoring miniupnp as a submodule. Fixed with a cache override rather than patching their cmake. USE_DEVICE_TREZOR=OFF as well - an explorer has no use for hardware wallets and it drops the generated protobuf units. Verified that xmrblocks still links. ── Two things in the unit that look wrong and are not ────────────────────── WorkingDirectory is load-bearing: page.h hardcodes TMPL_DIR as "./templates", resolved relative to cwd at REQUEST time. Get it wrong and the service starts, binds 8081, and 500s every page. That is also why the health check fetches a real page and greps it instead of probing the port. ReadWritePaths must include the blockchain directory even though the database is opened read-only: monero sets mdb_flags = MDB_RDONLY WITHOUT MDB_NOLOCK, so LMDB still registers this reader in lock.mdb. With ProtectSystem=strict and nothing writable the open fails. "It only reads, so grant it nothing" is wrong. ── Timeouts are measured, not guessed ────────────────────────────────────── proxmox-tank-1 is a spinning-disk mirror and xmrblocks says so on startup. First start took 98s to open the LMDB before binding (wait_for now allows 300s), and a COLD index page took 68 SECONDS where the next two took 0.87s and 0.012s (the check now allows 90s, its unit 180s). The first pass used 90s and 25s and produced exactly the false failures you would expect. A stat() bug was caught the same way: `stat` computes a sha1 by default, so checking that data.mdb exists was hashing the entire 40 GB blockchain. --check found it before a real run did. ── Exposure ──────────────────────────────────────────────────────────────── Caddy on vipy terminates TLS; the upstream is reached over the tailnet, which needed a headscale ACL entry for tag:monerod-box:8081 - without it the node is invisible, not merely unreachable, and the symptom is a 502 "no such host". No authentication, deliberately: it is a block explorer over public chain data. That includes --enable-pusher, so a visitor can relay a raw transaction through our node. A choice, not an oversight; flip xmrblocks_enable_pusher to close it. ── Also ──────────────────────────────────────────────────────────────────── monerod-box raised 2048 -> 6144 MiB. Not for the explorer: monerod alone had outgrown 2 GB past ~39% of the chain and was OOM-killed twice in a morning at 1.74 GB anon-rss. Recorded in vms.auto.tfvars because ignore_changes = all means tofu will not reconcile it. The box now idles at ~1.2 GB of 5.9 GB with both services up. No backups, deliberately - binary, templates and chain are all reproducible. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/group_vars/all/main.yml | 4 + ansible/host_vars/monerod_box_local/main.yml | 4 + ansible/infra/402_public_monitoring.yml | 2 + .../infra/403_service_probe_registration.yml | 1 + ansible/roles/xmrblocks/README.md | 176 +++++++++++++++++ ansible/roles/xmrblocks/defaults/main.yml | 75 ++++++++ ansible/roles/xmrblocks/handlers/main.yml | 12 ++ ansible/roles/xmrblocks/tasks/healthcheck.yml | 54 ++++++ ansible/roles/xmrblocks/tasks/install.yml | 129 +++++++++++++ ansible/roles/xmrblocks/tasks/main.yml | 6 + ansible/roles/xmrblocks/tasks/service.yml | 57 ++++++ .../templates/healthcheck.service.j2 | 20 ++ .../xmrblocks/templates/healthcheck.sh.j2 | 59 ++++++ .../xmrblocks/templates/healthcheck.timer.j2 | 14 ++ .../xmrblocks/templates/xmrblocks.service.j2 | 58 ++++++ .../services/headscale/templates/acl.json.j2 | 1 + .../xmrblocks/deploy_xmrblocks_playbook.yml | 39 ++++ ansible/site.yml | 1 + scripts/build-xmrblocks.sh | 180 ++++++++++++++++++ tofu/nodito/vms.auto.tfvars | 6 +- 20 files changed, 897 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/xmrblocks/README.md create mode 100644 ansible/roles/xmrblocks/defaults/main.yml create mode 100644 ansible/roles/xmrblocks/handlers/main.yml create mode 100644 ansible/roles/xmrblocks/tasks/healthcheck.yml create mode 100644 ansible/roles/xmrblocks/tasks/install.yml create mode 100644 ansible/roles/xmrblocks/tasks/main.yml create mode 100644 ansible/roles/xmrblocks/tasks/service.yml create mode 100644 ansible/roles/xmrblocks/templates/healthcheck.service.j2 create mode 100644 ansible/roles/xmrblocks/templates/healthcheck.sh.j2 create mode 100644 ansible/roles/xmrblocks/templates/healthcheck.timer.j2 create mode 100644 ansible/roles/xmrblocks/templates/xmrblocks.service.j2 create mode 100644 ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml create mode 100755 scripts/build-xmrblocks.sh diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 1d16bb4..3f79c44 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -53,6 +53,10 @@ subdomains: # monerod's own Digest auth does the authentication. monero: monero + # Monero block explorer (monerod-box, proxied via vipy). Fully public and + # unauthenticated, like any block explorer - the data is the public chain. + monero_explorer: exmrplore + # Read by plays across several groups, so global rather than group_vars/. headscale_namespace: counter-net diff --git a/ansible/host_vars/monerod_box_local/main.yml b/ansible/host_vars/monerod_box_local/main.yml index 10e28ee..181beb1 100644 --- a/ansible/host_vars/monerod_box_local/main.yml +++ b/ansible/host_vars/monerod_box_local/main.yml @@ -9,6 +9,9 @@ monero_p2p_port: 18080 monero_restricted_rpc_port: 18089 +# The explorer's HTTP port, read by the Caddy play on the edge host. +xmrblocks_port: 8081 + # The 500 GB disk added to this VM in Proxmox as scsi1. Named here rather than # left to the role default so that "which disk holds the blockchain" is a # property of the machine, recorded next to the machine. The role only ever @@ -25,3 +28,4 @@ monero_data_device: /dev/sdb # supervision and would be noise. monitored_services: - monerod + - xmrblocks diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index a43a8af..c24df46 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -32,6 +32,7 @@ - {sub: "{{ subdomains.mempool }}", host: vipy} - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - {sub: "{{ subdomains.monero }}", host: vipy} + - {sub: "{{ subdomains.monero_explorer }}", host: vipy} # A public resolver on purpose: this must test what the internet sees, not # what a local cache or the tailnet's MagicDNS happens to answer. @@ -55,6 +56,7 @@ # Digest challenge. Asserting 200 here would go green precisely when # rpc-login broke and the node started serving the world for free. - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401} + - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200} # Ports published from the edge host by socket_proxy. public_tcp: diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index a9c5de5..f6e6bf4 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -46,6 +46,7 @@ - {name: mempool-backend, host: mempool_box_local} - {name: mempool-frontend, host: mempool_box_local} - {name: monerod, host: monerod_box_local} + - {name: xmrblocks, host: monerod_box_local} tasks: - name: Build the probe endpoint list diff --git a/ansible/roles/xmrblocks/README.md b/ansible/roles/xmrblocks/README.md new file mode 100644 index 0000000..a7207dd --- /dev/null +++ b/ansible/roles/xmrblocks/README.md @@ -0,0 +1,176 @@ +# xmrblocks + +The [Onion Monero Blockchain Explorer](https://github.com/moneroexamples/onion-monero-blockchain-explorer), +on `monerod-box` beside the node, published at `exmrplore.contrapeso.xyz`. + +## Why this role installs a binary instead of building one + +Every other compiled service in this estate builds on its own host — +`bitcoin_knots` and `datum_gateway` both do. This one cannot, and the reason is +worth recording so nobody "fixes" it later. + +`xmrblocks` is not a client of monerod. Its `cmake/FindMonero.cmake` imports 21 +**static libraries** out of a compiled monero source tree and its headers, because +it opens the LMDB directly rather than going through RPC. Building it therefore +means compiling monero first — and the release tarball we install for the daemon +contains only executables, no `.a` files and no headers, so there is nothing on +the box to link against. + +Measured on Debian 13, building only the libraries `xmrblocks` needs: + +| | | +|---|---| +| peak RSS, one `cc1plus` | **~3.2 GB** | +| peak RSS, `-j4` concurrent | **~5.5 GB** | +| wall time, `-j4` | 167 s | +| build tree | 125 MB (source 70 MB) | +| toolchain | 1.6 GB | + +`monerod-box` had **2 GB** when this was measured (it has since been raised to +6 GB, for monerod's own sake — see below). It was tried at 768 MiB and 1800 MiB, +at `-j1` and `-j2`, and every combination was OOM-killed — a single translation unit +exceeding 3 GB is not something lower parallelism can work around. Disk was +never the constraint; the often-quoted "~10 GB" for a monero build describes +upstream's full `make`, which also produces monerod, both wallets and the test +suite, none of which are linked here. + +So the binary is built elsewhere by **`scripts/build-xmrblocks.sh`**, which runs +the whole thing in a `debian:13` container. That matters: an artifact only one +machine can produce is an artifact nobody can audit. Any host with Docker and +~6 GB of RAM reproduces it, and the script verifies the monero source against +binaryFate's GPG signature exactly as `roles/monero` does for the daemon. + +The role refuses to ship anything it has not checked: `xmrblocks_sha256` is +asserted **on the control node, before the copy**, against the value the build +script prints. + +## Three upstream problems, and the fixes + +None of these are documented anywhere. All three were found by building. + +1. **`libasio-dev` is missing from upstream's dependency list.** The Dockerfile + is Ubuntu-based and installs `libboost-all-dev`; Crow wants standalone + `asio.hpp`, not `boost/asio.hpp`. On Debian 13 the build dies with + `asio.hpp: No such file or directory`. + +2. **The explorer does not compile against monero v0.18.5.1.** v0.18.5.0 changed + + ```cpp + crypto::hash get_pruned_transaction_hash(const transaction&, const crypto::hash&); + bool get_pruned_transaction_hash(const transaction&, const crypto::hash&, crypto::hash& res); + ``` + + and `src/page.h` still calls the old form. Upstream's Dockerfile pins + `MONERO_BRANCH=v0.18.4.0`, which is why nobody has hit it. The build script + applies a one-line patch, **anchored on the exact source line** so that if + upstream ever fixes it the build stops loudly instead of silently doing + something else. + + We patch rather than build against v0.18.4.0 deliberately: the current signed + `hashes.txt` only covers the current release, so using the older source would + mean giving up the provenance chain. The LMDB format is `VERSION 5` in both, + so either version reads the other's database safely. + +3. **`FindMonero.cmake` looks for miniupnpc inside the monero build tree** with + `NO_DEFAULT_PATH`, but monero stopped vendoring miniupnp as a submodule. Fixed + with a `-D Xmr_MINIUPNPC_LIBRARY=` cache override pointing at the system + library, rather than patching their cmake. + +`USE_DEVICE_TREZOR=OFF` is also passed: an explorer has no use for hardware +wallet support, and it drops the generated protobuf units, which are among the +heaviest to compile. `xmrblocks` was verified to link against the resulting +libraries. + +## Two things the unit gets right that look wrong + +**`WorkingDirectory` is load-bearing.** `src/page.h` hardcodes +`#define TMPL_DIR "./templates"` — the HTML is resolved relative to the process +working directory, at request time. Get it wrong and the service starts happily, +listens on 8081, and returns 500 for every page. That is also why the health +check fetches a real page and greps it rather than just probing the port. + +**`ReadWritePaths` must include the blockchain directory**, even though the +database is opened read-only. `MicroCore::init` sets `DBF_RDONLY`, and monero's +`db_lmdb.cpp` turns that into `mdb_flags = MDB_RDONLY` **without `MDB_NOLOCK`** — +so LMDB still registers this process in the reader table inside `lock.mdb`. With +`ProtectSystem=strict` and nothing writable, the open fails outright. The +intuitive "it only reads, grant it nothing" is wrong. `data.mdb` itself remains +read-only to the process, and writing to monerod's lock table is what any second +reader does. + +## Exposure + +Published by `caddy_site` on the edge host at `exmrplore.contrapeso.xyz`, with +TLS terminated there and the upstream reached over the tailnet via MagicDNS. + +The explorer listens on `0.0.0.0:8081`. Nothing on the LAN or the internet +reaches that directly: Tailscale's `ts-input` chain sits ahead of UFW, so only +tailnet peers can, and the Headscale ACL grants `tag:prd-vipy` exactly +`tag:monerod-box:8081`. A node with no ACL rule is *invisible* rather than +merely unreachable — it never appears in `tailscale status`, MagicDNS does not +resolve it, and the symptom is a Caddy 502 saying "no such host". + +**There is no authentication, by choice.** A block explorer serves public chain +data. But note what that includes: `--enable-pusher` is on (upstream's default), +so any visitor can submit a raw transaction and have *our* node relay it. That is +a deliberate decision, not an oversight — drop `xmrblocks_enable_pusher` to +`false` to close it. + +`--enable-json-api` and `--enable-autorefresh-option` are likewise upstream's +defaults. + +## Monitoring + +| Endpoint | Source | Answers | +|---|---|---| +| `services/monerod-box-local/xmrblocks` | `infra/401` via `monitored_services` | is the unit active | +| `probe/xmrblocks` | this role + `infra/403` | does it serve a real page | +| `dns/exmrplore.contrapeso.xyz`, `public/exmrplore` | `infra/402` | can the internet reach it | + +`public/exmrplore` asserts **200**, unlike the Monero RPC endpoint next door +which asserts 401 — this one genuinely has no auth in front of it. + +The timer carries no `Requires=xmrblocks.service`; see the comment in +`healthcheck.timer.j2` for why that would be backwards. + +## Backups: none, deliberately + +Everything here is derived or reproducible: the binary comes from +`scripts/build-xmrblocks.sh`, the templates ship with it, and the data it serves +is monerod's blockchain, which is itself excluded for the same reason. There is +no state on this service worth a backup job. Recorded as a decision so it does +not read as an oversight — the same treatment `phoenixd` and `monero` get. + +## Upgrading + +1. Edit the pins at the top of `scripts/build-xmrblocks.sh` (`EXPLORER_COMMIT`, + and `MONERO_VERSION` + `MONERO_SRC_SHA256` together). +2. Run it. It prints the new artifact sha256. +3. Put that in `xmrblocks_sha256` and `xmrblocks_explorer_commit` here. +4. Converge. The role asserts the checksum before copying anything. + +If the patch anchor no longer matches, the script stops — read the diff before +forcing it. + +## Runtime cost, and the RAM history + +On **2026-09-20 the VM was raised from 2 GB to 6 GB**, and that was not for the +explorer. monerod alone had outgrown 2 GB once the chain passed ~39%: it was +OOM-killed twice in one morning at 1.74 GB anon-rss, restarting every hour. With +6 GB the box sits at ~1.2 GB used of 5.9 GB with both services running +(monerod ~1.0 GB, xmrblocks ~96 MB). + +**The disk is the real performance limit, not memory.** `proxmox-tank-1` is a +spinning-disk mirror, and xmrblocks says so on startup: *"The blockchain is on a +rotating drive: this will be very slow, use an SSD if possible"*. Measured +consequences, both of which the role accounts for: + +- first start takes ~98 seconds to open the LMDB before it binds, so + `wait_for` allows 300s +- a COLD index page took **68 seconds**, while the two requests after it took + 0.87s and 0.012s — so the health check allows 90s and its unit 180s. A short + timeout there does not detect a slow explorer, it manufactures false DOWNs. + +Footprint note: xmrblocks mmaps the LMDB, so most of its apparent size is +reclaimable page cache rather than anonymous memory, and `--concurrency` is held +at 2 for the same reason. diff --git a/ansible/roles/xmrblocks/defaults/main.yml b/ansible/roles/xmrblocks/defaults/main.yml new file mode 100644 index 0000000..d90cab8 --- /dev/null +++ b/ansible/roles/xmrblocks/defaults/main.yml @@ -0,0 +1,75 @@ +# xmrblocks - the Onion Monero Blockchain Explorer. +# https://github.com/moneroexamples/onion-monero-blockchain-explorer +# +# This role INSTALLS a prebuilt binary; it does not compile one. xmrblocks links +# 21 static libraries out of a compiled monero source tree, and building that +# needs ~3.2 GB of RAM for a single compiler (~5.5 GB at -j4). monerod-box has +# 2 GB, so it cannot build this at any -j. See README.md. +# +# The artifact is produced by scripts/build-xmrblocks.sh, which runs the whole +# build in a debian:13 container - so it is reproducible on any machine with +# Docker, not just the one that happened to build it. + +# --- The artifact ------------------------------------------------------------ +# A directory on the CONTROL NODE holding `xmrblocks`, `templates/` and MANIFEST. +xmrblocks_artifact_dir: "{{ lookup('env', 'HOME') }}/personal_infra_artifacts/xmrblocks" + +# Pinned checksum of the binary we intend to ship. Asserted on the control node +# BEFORE anything is copied: shipping an unverified binary to a host is the one +# thing this role must never do. Update it together with the build pins in +# scripts/build-xmrblocks.sh. +xmrblocks_sha256: "a40c495e73a44fbb3b666fcd8a01466a2aef12ed1906558f3d5b7e458c256af6" + +# Recorded so the deployed host can be matched back to a build. +xmrblocks_explorer_commit: "29593b709edeb23daba035009ccd134a5862543a" +xmrblocks_monero_version: "0.18.5.1" + +# --- Install layout ---------------------------------------------------------- +# The binary hardcodes TMPL_DIR as "./templates", relative to the working +# directory - so templates MUST sit beside the binary and the unit MUST set +# WorkingDirectory. Get that wrong and it starts fine, then fails per-request. +xmrblocks_dir: /opt/xmrblocks + +# Runs as the monero user: the explorer opens monerod's LMDB directly, and +# /var/lib/monero is 0750 monero:monero. It opens it READ-ONLY (DBF_RDONLY in +# MicroCore::init), so it is safe alongside a running monerod. +xmrblocks_user: monero +xmrblocks_group: monero + +# --- Runtime ----------------------------------------------------------------- +# The LMDB directory, not the data directory: monerod's data-dir plus /lmdb. +xmrblocks_bc_path: /var/lib/monero/lmdb + +# 0.0.0.0 so the edge host can reach it over the tailnet. Nothing on the LAN or +# the internet can: Tailscale's ts-input chain sits ahead of UFW, and the only +# public route is the Caddy vhost on vipy. +xmrblocks_bind: "0.0.0.0" +xmrblocks_port: 8081 + +# The unrestricted RPC on loopback. Used for the mempool view and tx pushing. +xmrblocks_daemon_url: "127.0.0.1:18081" +# monerod's --rpc-login guards BOTH its RPC servers, so the explorer needs it. +xmrblocks_rpc_user: "{{ monero_rpc_user | default('') }}" +xmrblocks_rpc_password: "{{ monero_rpc_password | default('') }}" + +# Upstream's own defaults (the Dockerfile CMD). +# NOTE: enable-pusher lets a visitor submit a raw transaction through YOUR node. +# It is a deliberate choice here, not an oversight. +xmrblocks_enable_json_api: true +xmrblocks_enable_autorefresh: true +xmrblocks_enable_pusher: true + +# Crow's thread count. Held low on purpose: this box has 2 GB and monerod +# already uses ~1.3 GB of it. +xmrblocks_concurrency: 2 + +# The unit file carries the RPC login, so --diff is suppressed by default. +xmrblocks_reveal_config: false + +# --- Health check ------------------------------------------------------------ +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/xmrblocks/handlers/main.yml b/ansible/roles/xmrblocks/handlers/main.yml new file mode 100644 index 0000000..7be239a --- /dev/null +++ b/ansible/roles/xmrblocks/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: Restart xmrblocks + systemd: + name: xmrblocks + state: restarted + daemon_reload: yes + +- name: Restart xmrblocks health check timer + systemd: + name: xmrblocks-healthcheck.timer + state: restarted + daemon_reload: yes diff --git a/ansible/roles/xmrblocks/tasks/healthcheck.yml b/ansible/roles/xmrblocks/tasks/healthcheck.yml new file mode 100644 index 0000000..7a3a3a1 --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/healthcheck.yml @@ -0,0 +1,54 @@ +--- +# "Does the explorer actually serve a page", as opposed to infra/401's "is the +# unit active". xmrblocks can be perfectly `active` and still fail every +# request - a missing templates directory does exactly that. + +- name: Install the health check's dependencies + apt: + name: + - curl + state: present + +- name: Create the xmrblocks health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/xmrblocks-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +# 0600: carries the Gatus bearer token. +- name: Create the xmrblocks health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/xmrblocks-healthcheck.service + owner: root + group: root + mode: '0600' + notify: Restart xmrblocks health check timer + +- name: Create the xmrblocks health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/xmrblocks-healthcheck.timer + owner: root + group: root + mode: '0644' + notify: Restart xmrblocks health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated: enabling a timer is deployment, not monitoring. +- name: Enable and restart the xmrblocks health check timer + systemd: + name: xmrblocks-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +- name: Run the xmrblocks health check once to arm the timer + ansible.builtin.command: systemctl start xmrblocks-healthcheck.service + changed_when: false diff --git a/ansible/roles/xmrblocks/tasks/install.yml b/ansible/roles/xmrblocks/tasks/install.yml new file mode 100644 index 0000000..688e5bc --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/install.yml @@ -0,0 +1,129 @@ +--- +# Install the prebuilt xmrblocks artifact produced by scripts/build-xmrblocks.sh. +# +# The verification happens on the CONTROL NODE, before a single byte is copied. +# Shipping a binary to a host and checking it afterwards is not verification, +# it is an apology. + +- name: Assert the artifact checksum is pinned + ansible.builtin.assert: + that: + - xmrblocks_sha256 | length == 64 + quiet: true + fail_msg: >- + xmrblocks_sha256 is not set. Build the artifact with + scripts/build-xmrblocks.sh and pin the sha it prints, so this role cannot + ship a binary nobody has checked. + +- name: Check the artifact on the control node + ansible.builtin.stat: + path: "{{ xmrblocks_artifact_dir }}/xmrblocks" + checksum_algorithm: sha256 + get_checksum: yes + delegate_to: localhost + become: no + register: xmrblocks_local + changed_when: false + +- name: Assert the artifact exists and matches the pinned checksum + ansible.builtin.assert: + that: + - xmrblocks_local.stat.exists + - xmrblocks_local.stat.checksum == xmrblocks_sha256 + quiet: true + fail_msg: >- + {{ xmrblocks_artifact_dir }}/xmrblocks is + {{ 'missing - run scripts/build-xmrblocks.sh' + if not xmrblocks_local.stat.exists + else 'sha256 ' ~ xmrblocks_local.stat.checksum ~ ', expected ' ~ xmrblocks_sha256 }} + +- name: Check the templates directory on the control node + ansible.builtin.stat: + path: "{{ xmrblocks_artifact_dir }}/templates/index.html" + delegate_to: localhost + become: no + register: xmrblocks_local_templates + changed_when: false + +# The binary resolves TMPL_DIR as "./templates" at request time, so a missing +# templates directory is not a startup error - it is a 500 on the first page +# view. Catch it here instead. +- name: Assert the templates came with the artifact + ansible.builtin.assert: + that: + - xmrblocks_local_templates.stat.exists + quiet: true + fail_msg: "{{ xmrblocks_artifact_dir }}/templates/index.html is missing - the artifact is incomplete" + +- name: Install the runtime shared libraries xmrblocks needs + apt: + name: + # From `objdump -p xmrblocks | grep NEEDED`. Boost is version-locked to + # the build image's Debian release; monerod-box is Debian 13 and the + # artifact is built in debian:13, so 1.83 matches on both sides. + - libboost-chrono1.83.0 + - libboost-filesystem1.83.0 + - libboost-program-options1.83.0 + - libboost-serialization1.83.0 + - libboost-thread1.83.0 + - libhidapi-libusb0 + - libunbound8 + - libunwind8 + state: present + update_cache: yes + +- name: Create the xmrblocks directory + ansible.builtin.file: + path: "{{ xmrblocks_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Install the xmrblocks binary + ansible.builtin.copy: + src: "{{ xmrblocks_artifact_dir }}/xmrblocks" + dest: "{{ xmrblocks_dir }}/xmrblocks" + owner: root + group: root + mode: '0755' + notify: Restart xmrblocks + +- name: Install the xmrblocks templates + ansible.builtin.copy: + src: "{{ xmrblocks_artifact_dir }}/templates/" + dest: "{{ xmrblocks_dir }}/templates/" + owner: root + group: root + mode: '0644' + directory_mode: '0755' + notify: Restart xmrblocks + +- name: Record which build is deployed + ansible.builtin.copy: + src: "{{ xmrblocks_artifact_dir }}/MANIFEST" + dest: "{{ xmrblocks_dir }}/MANIFEST" + owner: root + group: root + mode: '0644' + +- name: Assert the blockchain database is where we think it is + ansible.builtin.stat: + path: "{{ xmrblocks_bc_path }}/data.mdb" + # data.mdb is the whole blockchain - tens of GB and growing. stat computes a + # sha1 by DEFAULT, so without these it hashes the entire chain just to answer + # "does this file exist", and the play hangs for many minutes. + get_checksum: no + get_mime: no + get_attributes: no + register: xmrblocks_lmdb + +- name: Fail if the LMDB is missing + ansible.builtin.assert: + that: + - xmrblocks_lmdb.stat.exists + quiet: true + fail_msg: >- + No LMDB at {{ xmrblocks_bc_path }}/data.mdb. xmrblocks reads monerod's + database directly, so it has nothing to serve. Check that monerod is + deployed and its data disk is mounted. diff --git a/ansible/roles/xmrblocks/tasks/main.yml b/ansible/roles/xmrblocks/tasks/main.yml new file mode 100644 index 0000000..8a024a2 --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is the main verification tool in this repo. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/xmrblocks/tasks/service.yml b/ansible/roles/xmrblocks/tasks/service.yml new file mode 100644 index 0000000..38fecd3 --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/service.yml @@ -0,0 +1,57 @@ +--- +- name: Assert the daemon RPC credentials are available + ansible.builtin.assert: + that: + - xmrblocks_rpc_user | length > 0 + - xmrblocks_rpc_password | length > 0 + quiet: true + fail_msg: >- + xmrblocks needs monerod's RPC login: --rpc-login guards both of monerod's + RPC servers, so without it the mempool view and the pusher get 401. + +- name: Create the xmrblocks systemd unit + ansible.builtin.template: + src: xmrblocks.service.j2 + dest: /etc/systemd/system/xmrblocks.service + owner: root + group: root + mode: '0644' + # The unit embeds --daemon-login in ExecStart, so --diff would print the + # RPC password. -e xmrblocks_reveal_config=true when you need to see it. + diff: "{{ xmrblocks_reveal_config | default(false) | bool }}" + notify: Restart xmrblocks + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start xmrblocks + systemd: + name: xmrblocks + enabled: yes + state: started + +- name: Flush handlers so xmrblocks is running before we probe it + meta: flush_handlers + +# Generous on purpose. xmrblocks opens the whole LMDB before it binds, and this +# pool is a spinning-disk mirror - it logs "The blockchain is on a rotating +# drive: this will be very slow" and took 98 SECONDS to come up on first start. +# 90s was not enough and produced a scary failure on a service that was fine. +- name: Wait for the xmrblocks HTTP port + ansible.builtin.wait_for: + port: "{{ xmrblocks_port }}" + host: 127.0.0.1 + delay: 5 + timeout: 300 + ignore_errors: yes + +- name: Check the xmrblocks service status + systemd: + name: xmrblocks + register: xmrblocks_status + changed_when: false + +- name: Display the xmrblocks service status + ansible.builtin.debug: + msg: "xmrblocks is {{ 'running' if xmrblocks_status.status.ActiveState == 'active' else 'NOT running' }}" diff --git a/ansible/roles/xmrblocks/templates/healthcheck.service.j2 b/ansible/roles/xmrblocks/templates/healthcheck.service.j2 new file mode 100644 index 0000000..ffae4da --- /dev/null +++ b/ansible/roles/xmrblocks/templates/healthcheck.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=Onion Monero Blockchain Explorer Health Check +After=network.target xmrblocks.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/xmrblocks-healthcheck-push.sh +# Longer than the script's 90s curl plus its push retries. A oneshot defaults to +# TimeoutStartSec=90, which would kill the check mid-request on a cold cache and +# report a failure that says nothing about the service. +TimeoutStartSec=180 +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +Environment=XMRBLOCKS_URL=http://127.0.0.1:{{ xmrblocks_port }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/xmrblocks/templates/healthcheck.sh.j2 b/ansible/roles/xmrblocks/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..2effe2a --- /dev/null +++ b/ansible/roles/xmrblocks/templates/healthcheck.sh.j2 @@ -0,0 +1,59 @@ +#!/bin/bash +# xmrblocks health check — managed by Ansible (roles/xmrblocks) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed xmrblocks-healthcheck.service +# Reporting anywhere else is optional and generic. +# +# This fetches a real page rather than just opening the port. The binary +# resolves its HTML as "./templates" relative to the working directory, so a +# deploy that got WorkingDirectory wrong listens happily on 8081 and fails +# every request. A TCP check would call that healthy. + +URL="${XMRBLOCKS_URL:-http://127.0.0.1:8081}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +report() { + local status=$1 + local msg=$2 + + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded_msg="${msg// /%20}" + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${encoded_msg}" || true +} + +# 90s, not the obvious 10 or 25. The index page renders ~46 KB from the LMDB, +# and this pool is a spinning-disk mirror shared with a syncing monerod: a COLD +# request measured 68 SECONDS, while the two immediately after it took 0.87s and +# 0.012s. A short timeout here does not detect a slow explorer, it manufactures +# a false DOWN every time the page cache has gone cold between runs. +body=$(curl -s --max-time 90 -w '\n%{http_code}' "${URL}/" 2>/dev/null) +code=$(printf '%s' "$body" | tail -1) + +if [ -z "$code" ] || [ "$code" = "000" ]; then + report "down" "xmrblocks not responding" + exit 1 +fi + +if [ "$code" != "200" ]; then + report "down" "xmrblocks HTTP ${code}" + exit 1 +fi + +# A 200 that is not the explorer is still a failure. The index page always +# carries the project name in its title. +if ! printf '%s' "$body" | grep -qi "monero"; then + report "down" "xmrblocks served 200 but not an explorer page (templates missing?)" + exit 1 +fi + +report "up" "OK" +exit 0 diff --git a/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..48713a4 --- /dev/null +++ b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Onion Monero Blockchain Explorer Health Check Timer +# Deliberately NO `Requires=xmrblocks.service`. Requires on a timer stops the +# timer when the watched unit stops - "if the thing I am watching goes down, +# stop watching it" - leaving nothing to re-arm when it returns. Fulcrum's +# timer carried it and sat active+enabled for seven months without firing. + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/xmrblocks/templates/xmrblocks.service.j2 b/ansible/roles/xmrblocks/templates/xmrblocks.service.j2 new file mode 100644 index 0000000..64c58af --- /dev/null +++ b/ansible/roles/xmrblocks/templates/xmrblocks.service.j2 @@ -0,0 +1,58 @@ +[Unit] +Description=Onion Monero Blockchain Explorer (xmrblocks) +Documentation=https://github.com/moneroexamples/onion-monero-blockchain-explorer +After=network-online.target monerod.service +Wants=network-online.target +# The explorer reads monerod's LMDB directly. If the blockchain disk is not +# mounted there is nothing to serve, and without this the unit would start and +# fail on every request instead of refusing to start at all. +RequiresMountsFor={{ xmrblocks_bc_path }} + +[Service] +Type=simple +User={{ xmrblocks_user }} +Group={{ xmrblocks_group }} + +# NOT decoration: the binary hardcodes TMPL_DIR as "./templates", so it resolves +# its HTML relative to the working directory. Without this it starts happily and +# then 500s on the first page view. +WorkingDirectory={{ xmrblocks_dir }} + +ExecStart={{ xmrblocks_dir }}/xmrblocks \ + --bc-path {{ xmrblocks_bc_path }} \ + --daemon-url {{ xmrblocks_daemon_url }} \ + --daemon-login {{ xmrblocks_rpc_user }}:{{ xmrblocks_rpc_password }} \ + --bindaddr {{ xmrblocks_bind }} \ + --port {{ xmrblocks_port }} \ + --concurrency {{ xmrblocks_concurrency }}{% if xmrblocks_enable_json_api %} \ + --enable-json-api{% endif %}{% if xmrblocks_enable_autorefresh %} \ + --enable-autorefresh-option{% endif %}{% if xmrblocks_enable_pusher %} \ + --enable-pusher{% endif %} + + +Restart=always +RestartSec=20 +TimeoutStopSec=30 + +StandardOutput=journal +StandardError=journal + +# Hardening. The explorer serves static templates and reads the chain. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only + +# The LMDB directory must be WRITABLE even though the database is opened +# read-only. monero sets `mdb_flags = MDB_RDONLY` without MDB_NOLOCK +# (db_lmdb.cpp), so LMDB still registers this process in the reader table +# inside lock.mdb. With ProtectSystem=strict and no ReadWritePaths the open +# fails outright - the obvious "it only reads, so grant nothing" is wrong here. +# Writing to monerod's own lock table is normal and is what any second reader +# does; data.mdb itself stays read-only to this process. +ReadWritePaths={{ xmrblocks_bc_path }} + +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/services/headscale/templates/acl.json.j2 b/ansible/services/headscale/templates/acl.json.j2 index 6ab6fa5..5860544 100644 --- a/ansible/services/headscale/templates/acl.json.j2 +++ b/ansible/services/headscale/templates/acl.json.j2 @@ -43,6 +43,7 @@ "tag:mempool-box:8080", "tag:memos-box:5230", "tag:monerod-box:18089", + "tag:monerod-box:8081", "tag:arbret-staging-box:80", "tag:stalwart-box:25", "tag:stalwart-box:465", diff --git a/ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml b/ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml new file mode 100644 index 0000000..c3dcc29 --- /dev/null +++ b/ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml @@ -0,0 +1,39 @@ +--- +# The Monero block explorer, on the same box as the node. +# +# xmrblocks reads monerod's LMDB directly rather than going through RPC, which +# is why it lives here and not on its own VM. It is a 26 MB binary and 132 KB +# of templates; the expensive part is building it, and that happens elsewhere +# (scripts/build-xmrblocks.sh) because this box has 2 GB and a single monero +# translation unit needs ~3.2 GB. See roles/xmrblocks/README.md. +- name: Deploy the Monero blockchain explorer + hosts: monero + become: yes + vars: + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_xmrblocks/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - xmrblocks + +- name: Publish the explorer from the edge host + hosts: edge + become: yes + tasks: + # No basic auth: this is a public block explorer over public chain data, + # which is what a block explorer is for. Caddy terminates TLS and proxies. + # Note the explorer has NO authentication of its own, so everything it can + # do is world-reachable - including the tx pusher, which relays a raw + # transaction through our node. That is deliberate; see the role README. + - name: Expose the Monero explorer over HTTPS + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: monero-explorer + caddy_site_domain: "{{ subdomains.monero_explorer }}.{{ root_domain }}" + caddy_site_upstream: "{{ hostvars['monerod_box_local'].ansible_host }}:{{ hostvars['monerod_box_local'].xmrblocks_port }}" + caddy_site_resolvers: "100.100.100.100" + + # The per-site validate cannot catch a conflict between sites. + - name: Validate the whole Caddyfile + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + changed_when: false diff --git a/ansible/site.yml b/ansible/site.yml index c777577..284633c 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -54,6 +54,7 @@ - import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml - import_playbook: services/mempool/deploy_mempool_playbook.yml - import_playbook: services/monero/deploy_monero_playbook.yml +- import_playbook: services/xmrblocks/deploy_xmrblocks_playbook.yml - import_playbook: services/memos/deploy_memos_playbook.yml - import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml - import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml diff --git a/scripts/build-xmrblocks.sh b/scripts/build-xmrblocks.sh new file mode 100755 index 0000000..44e3ede --- /dev/null +++ b/scripts/build-xmrblocks.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# +# Build xmrblocks (the Onion Monero Blockchain Explorer) reproducibly. +# +# WHY THIS SCRIPT EXISTS +# ---------------------- +# xmrblocks is not a client of monerod. It links 21 static libraries out of a +# compiled monero source tree and opens the LMDB directly, so building it means +# first compiling monero itself. Measured on Debian 13: a single cc1plus peaks +# at ~3.1 GB and a -j4 build peaks at ~5.5 GB. monerod-box has 2 GB, so it +# cannot build this at any level of parallelism - the binary has to be built +# elsewhere and shipped. +# +# Shipping a binary that only one machine can produce is how estates end up +# with artefacts nobody can rebuild. This script is the answer to that: it runs +# the whole build inside a debian:13 container, so any machine with Docker and +# ~6 GB of RAM produces the same binary. Nothing is installed on the host. +# +# WHAT IT GUARANTEES +# - the monero source is the GPG-signed release tarball, verified against +# binaryFate's key (the same chain roles/monero uses for the daemon) +# - the explorer is pinned to an exact commit +# - the one upstream patch is applied from an anchor, and the script FAILS +# if the anchor is missing rather than silently skipping it +# +# WHAT IT IS NOT +# Not bit-for-bit reproducible. Debian's toolchain, timestamps and build +# paths all leak in. It is repeatable, not deterministic; the MANIFEST +# records what went in so two builds can be compared. +# +# Usage: scripts/build-xmrblocks.sh [output-dir] +# Needs: docker, ~6 GB RAM available to it, ~2 GB disk +set -euo pipefail + +# --- Pins. Change these together and rebuild. -------------------------------- +MONERO_VERSION="0.18.5.1" +MONERO_SRC_TARBALL="monero-source-v${MONERO_VERSION}.tar.bz2" +MONERO_SRC_URL="https://downloads.getmonero.org/cli/${MONERO_SRC_TARBALL}" +# From the GPG-signed hashes.txt. Verified by the container, not trusted from here. +MONERO_SRC_SHA256="9ec6ed0fd37db9d81cf7738a5f0536cf9aec6ed8ef8fd48649a59a6aaf20de3d" +BINARYFATE_FPR="81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92" + +EXPLORER_REPO="https://github.com/moneroexamples/onion-monero-blockchain-explorer.git" +EXPLORER_COMMIT="29593b709edeb23daba035009ccd134a5862543a" + +BUILD_IMAGE="debian:13" # must match the TARGET host's Debian release +JOBS="${JOBS:-4}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SIGNING_KEY="${REPO_ROOT}/ansible/roles/monero/files/binaryfate.asc" +OUT_DIR="${1:-$HOME/personal_infra_artifacts/xmrblocks}" + +command -v docker >/dev/null || { echo "docker is required" >&2; exit 1; } +[ -f "$SIGNING_KEY" ] || { echo "missing signing key: $SIGNING_KEY" >&2; exit 1; } + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +cp "$SIGNING_KEY" "$WORK/binaryfate.asc" +mkdir -p "$OUT_DIR" + +echo "==> building xmrblocks" +echo " monero ${MONERO_VERSION} (signed source tarball)" +echo " explorer ${EXPLORER_COMMIT}" +echo " image ${BUILD_IMAGE}, -j${JOBS}" +echo " output ${OUT_DIR}" + +cat > "$WORK/build.sh" </dev/null + +cd /build + +echo "--> verifying the monero source tarball" +export GNUPGHOME=/build/gnupg; mkdir -p \$GNUPGHOME; chmod 700 \$GNUPGHOME +gpg --batch --import /build/binaryfate.asc 2>/dev/null +gpg --batch --with-colons --fingerprint | awk -F: '/^fpr:/{print \$10}' \\ + | grep -qx "${BINARYFATE_FPR}" || { echo "signing key is not binaryFate's"; exit 1; } +curl -sSL -o hashes.txt https://www.getmonero.org/downloads/hashes.txt +gpg --batch --status-fd 2 --output hashes-verified.txt --decrypt hashes.txt 2>gpg.err +grep -q "VALIDSIG.*${BINARYFATE_FPR}" gpg.err || { echo "hashes.txt did not verify"; exit 1; } +curl -sSL -o "${MONERO_SRC_TARBALL}" "${MONERO_SRC_URL}" +ACTUAL=\$(sha256sum "${MONERO_SRC_TARBALL}" | cut -d' ' -f1) +[ "\$ACTUAL" = "${MONERO_SRC_SHA256}" ] || { echo "tarball sha mismatch"; exit 1; } +grep -Fxq "\$ACTUAL ${MONERO_SRC_TARBALL}" hashes-verified.txt \\ + || { echo "tarball is not the one binaryFate signed"; exit 1; } +echo " source verified against the signed hashes" + +tar xjf "${MONERO_SRC_TARBALL}" +SRC=/build/monero-source-v${MONERO_VERSION} + +echo "--> building the monero libraries xmrblocks links against" +# Only the libraries. Upstream's Dockerfile runs a full \`make\`, which also +# builds monerod, both wallets and the test suite - none of which are linked +# into xmrblocks. USE_DEVICE_TREZOR=OFF drops the generated protobuf units too; +# an explorer has no use for hardware wallets. +mkdir -p \$SRC/build/release && cd \$SRC/build/release +cmake -D CMAKE_BUILD_TYPE=Release -D BUILD_TESTS=OFF -D USE_DEVICE_TREZOR=OFF ../.. >/dev/null +cmake --build . -j${JOBS} --target \\ + common blocks cryptonote_basic cryptonote_core multisig cryptonote_protocol \\ + daemonizer mnemonics epee lmdb device wallet-crypto blockchain_db ringct \\ + wallet cncrypto easylogging version checkpoints randomx hardforks >/dev/null + +echo "--> fetching the explorer at ${EXPLORER_COMMIT}" +cd /build +git clone -q "${EXPLORER_REPO}" explorer +cd explorer && git checkout -q "${EXPLORER_COMMIT}" + +# monero v0.18.5.0 changed get_pruned_transaction_hash from returning the hash +# to returning bool with an out-param. The explorer still calls the old form, +# so it does not compile against the daemon we actually run. Anchored on the +# exact line: if upstream fixes this, the anchor vanishes and we stop loudly +# rather than building something unexpected. +OLD=' txd.hash = get_pruned_transaction_hash(tx, tx.prunable_hash);' +NEW=' get_pruned_transaction_hash(tx, tx.prunable_hash, txd.hash);' +if grep -qF "\$OLD" src/page.h; then + sed -i "s|\$OLD|\$NEW|" src/page.h + echo " applied get_pruned_transaction_hash patch" +elif grep -qF "\$NEW" src/page.h; then + echo " upstream already uses the new signature - patch not needed" +else + echo "PATCH ANCHOR NOT FOUND in src/page.h - upstream changed; review before building" >&2 + exit 1 +fi + +echo "--> building xmrblocks" +mkdir -p build && cd build +# Xmr_MINIUPNPC_LIBRARY: FindMonero.cmake searches for miniupnpc *inside* the +# monero build tree with NO_DEFAULT_PATH, but monero no longer vendors it. +# Point it at the system library instead of patching their cmake. +cmake -D MONERO_DIR=\$SRC \\ + -D Xmr_MINIUPNPC_LIBRARY=/usr/lib/x86_64-linux-gnu/libminiupnpc.a .. >/dev/null +cmake --build . -j${JOBS} >/dev/null +[ -x xmrblocks ] || { echo "no xmrblocks produced"; exit 1; } +./xmrblocks --help >/dev/null 2>&1 || true + +echo "--> collecting" +install -m 0755 xmrblocks /out/xmrblocks +rm -rf /out/templates && cp -r templates /out/templates +chmod -R a+rX /out/templates + +{ + echo "artifact: xmrblocks" + echo "built: \$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "build-image: ${BUILD_IMAGE}" + echo "monero-version: ${MONERO_VERSION}" + echo "monero-src-sha: ${MONERO_SRC_SHA256}" + echo "explorer-commit: ${EXPLORER_COMMIT}" + echo "cmake-flags: USE_DEVICE_TREZOR=OFF BUILD_TESTS=OFF CMAKE_BUILD_TYPE=Release" + echo "patches: get_pruned_transaction_hash -> out-param form (monero >= 0.18.5.0)" + echo "xmrblocks-sha256: \$(sha256sum /out/xmrblocks | cut -d' ' -f1)" + echo "templates-sha256: \$(find /out/templates -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" + echo "needed-libs:" + objdump -p /out/xmrblocks 2>/dev/null | awk '/NEEDED/{print " " \$2}' | sort +} > /out/MANIFEST +INNER +chmod +x "$WORK/build.sh" + +docker run --rm \ + -v "$WORK":/build \ + -v "$OUT_DIR":/out \ + "$BUILD_IMAGE" bash /build/build.sh + +echo +echo "==> done" +sed -n '1,12p' "$OUT_DIR/MANIFEST" +echo +echo " pin xmrblocks_sha256 in ansible/roles/xmrblocks/defaults/main.yml to:" +grep '^xmrblocks-sha256:' "$OUT_DIR/MANIFEST" | awk '{print " " $2}' diff --git a/tofu/nodito/vms.auto.tfvars b/tofu/nodito/vms.auto.tfvars index 8e50853..40877bd 100644 --- a/tofu/nodito/vms.auto.tfvars +++ b/tofu/nodito/vms.auto.tfvars @@ -58,7 +58,11 @@ vms = { vmid = 2400 name = "monerod-box" cores = 4 - memory_mb = 2048 + # Raised from 2048 on 2026-09-20: monerod was OOM-killed hourly once the + # chain passed ~39% (1.74 GB anon-rss against 1973 MB of RAM), and the + # explorer shares this box. Applied in Proxmox; recorded here because the + # vm resource sets ignore_changes = all and will not reconcile it. + memory_mb = 6144 disk_size_gb = 10 ipconfig0 = "ip=dhcp" },