diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml index 1d16bb4..3f79c44 100644 --- a/ansible/group_vars/all/main.yml +++ b/ansible/group_vars/all/main.yml @@ -53,6 +53,10 @@ subdomains: # monerod's own Digest auth does the authentication. monero: monero + # Monero block explorer (monerod-box, proxied via vipy). Fully public and + # unauthenticated, like any block explorer - the data is the public chain. + monero_explorer: exmrplore + # Read by plays across several groups, so global rather than group_vars/. headscale_namespace: counter-net diff --git a/ansible/host_vars/monerod_box_local/main.yml b/ansible/host_vars/monerod_box_local/main.yml index 10e28ee..181beb1 100644 --- a/ansible/host_vars/monerod_box_local/main.yml +++ b/ansible/host_vars/monerod_box_local/main.yml @@ -9,6 +9,9 @@ monero_p2p_port: 18080 monero_restricted_rpc_port: 18089 +# The explorer's HTTP port, read by the Caddy play on the edge host. +xmrblocks_port: 8081 + # The 500 GB disk added to this VM in Proxmox as scsi1. Named here rather than # left to the role default so that "which disk holds the blockchain" is a # property of the machine, recorded next to the machine. The role only ever @@ -25,3 +28,4 @@ monero_data_device: /dev/sdb # supervision and would be noise. monitored_services: - monerod + - xmrblocks diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index a43a8af..c24df46 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -32,6 +32,7 @@ - {sub: "{{ subdomains.mempool }}", host: vipy} - {sub: "{{ subdomains.datum_gateway }}", host: vipy} - {sub: "{{ subdomains.monero }}", host: vipy} + - {sub: "{{ subdomains.monero_explorer }}", host: vipy} # A public resolver on purpose: this must test what the internet sees, not # what a local cache or the tailnet's MagicDNS happens to answer. @@ -55,6 +56,7 @@ # Digest challenge. Asserting 200 here would go green precisely when # rpc-login broke and the node started serving the world for free. - {name: monero, sub: "{{ subdomains.monero }}", path: "/get_info", status: 401} + - {name: exmrplore, sub: "{{ subdomains.monero_explorer }}", path: "/", status: 200} # Ports published from the edge host by socket_proxy. public_tcp: diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index a9c5de5..f6e6bf4 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -46,6 +46,7 @@ - {name: mempool-backend, host: mempool_box_local} - {name: mempool-frontend, host: mempool_box_local} - {name: monerod, host: monerod_box_local} + - {name: xmrblocks, host: monerod_box_local} tasks: - name: Build the probe endpoint list diff --git a/ansible/roles/xmrblocks/README.md b/ansible/roles/xmrblocks/README.md new file mode 100644 index 0000000..a7207dd --- /dev/null +++ b/ansible/roles/xmrblocks/README.md @@ -0,0 +1,176 @@ +# xmrblocks + +The [Onion Monero Blockchain Explorer](https://github.com/moneroexamples/onion-monero-blockchain-explorer), +on `monerod-box` beside the node, published at `exmrplore.contrapeso.xyz`. + +## Why this role installs a binary instead of building one + +Every other compiled service in this estate builds on its own host — +`bitcoin_knots` and `datum_gateway` both do. This one cannot, and the reason is +worth recording so nobody "fixes" it later. + +`xmrblocks` is not a client of monerod. Its `cmake/FindMonero.cmake` imports 21 +**static libraries** out of a compiled monero source tree and its headers, because +it opens the LMDB directly rather than going through RPC. Building it therefore +means compiling monero first — and the release tarball we install for the daemon +contains only executables, no `.a` files and no headers, so there is nothing on +the box to link against. + +Measured on Debian 13, building only the libraries `xmrblocks` needs: + +| | | +|---|---| +| peak RSS, one `cc1plus` | **~3.2 GB** | +| peak RSS, `-j4` concurrent | **~5.5 GB** | +| wall time, `-j4` | 167 s | +| build tree | 125 MB (source 70 MB) | +| toolchain | 1.6 GB | + +`monerod-box` had **2 GB** when this was measured (it has since been raised to +6 GB, for monerod's own sake — see below). It was tried at 768 MiB and 1800 MiB, +at `-j1` and `-j2`, and every combination was OOM-killed — a single translation unit +exceeding 3 GB is not something lower parallelism can work around. Disk was +never the constraint; the often-quoted "~10 GB" for a monero build describes +upstream's full `make`, which also produces monerod, both wallets and the test +suite, none of which are linked here. + +So the binary is built elsewhere by **`scripts/build-xmrblocks.sh`**, which runs +the whole thing in a `debian:13` container. That matters: an artifact only one +machine can produce is an artifact nobody can audit. Any host with Docker and +~6 GB of RAM reproduces it, and the script verifies the monero source against +binaryFate's GPG signature exactly as `roles/monero` does for the daemon. + +The role refuses to ship anything it has not checked: `xmrblocks_sha256` is +asserted **on the control node, before the copy**, against the value the build +script prints. + +## Three upstream problems, and the fixes + +None of these are documented anywhere. All three were found by building. + +1. **`libasio-dev` is missing from upstream's dependency list.** The Dockerfile + is Ubuntu-based and installs `libboost-all-dev`; Crow wants standalone + `asio.hpp`, not `boost/asio.hpp`. On Debian 13 the build dies with + `asio.hpp: No such file or directory`. + +2. **The explorer does not compile against monero v0.18.5.1.** v0.18.5.0 changed + + ```cpp + crypto::hash get_pruned_transaction_hash(const transaction&, const crypto::hash&); + bool get_pruned_transaction_hash(const transaction&, const crypto::hash&, crypto::hash& res); + ``` + + and `src/page.h` still calls the old form. Upstream's Dockerfile pins + `MONERO_BRANCH=v0.18.4.0`, which is why nobody has hit it. The build script + applies a one-line patch, **anchored on the exact source line** so that if + upstream ever fixes it the build stops loudly instead of silently doing + something else. + + We patch rather than build against v0.18.4.0 deliberately: the current signed + `hashes.txt` only covers the current release, so using the older source would + mean giving up the provenance chain. The LMDB format is `VERSION 5` in both, + so either version reads the other's database safely. + +3. **`FindMonero.cmake` looks for miniupnpc inside the monero build tree** with + `NO_DEFAULT_PATH`, but monero stopped vendoring miniupnp as a submodule. Fixed + with a `-D Xmr_MINIUPNPC_LIBRARY=` cache override pointing at the system + library, rather than patching their cmake. + +`USE_DEVICE_TREZOR=OFF` is also passed: an explorer has no use for hardware +wallet support, and it drops the generated protobuf units, which are among the +heaviest to compile. `xmrblocks` was verified to link against the resulting +libraries. + +## Two things the unit gets right that look wrong + +**`WorkingDirectory` is load-bearing.** `src/page.h` hardcodes +`#define TMPL_DIR "./templates"` — the HTML is resolved relative to the process +working directory, at request time. Get it wrong and the service starts happily, +listens on 8081, and returns 500 for every page. That is also why the health +check fetches a real page and greps it rather than just probing the port. + +**`ReadWritePaths` must include the blockchain directory**, even though the +database is opened read-only. `MicroCore::init` sets `DBF_RDONLY`, and monero's +`db_lmdb.cpp` turns that into `mdb_flags = MDB_RDONLY` **without `MDB_NOLOCK`** — +so LMDB still registers this process in the reader table inside `lock.mdb`. With +`ProtectSystem=strict` and nothing writable, the open fails outright. The +intuitive "it only reads, grant it nothing" is wrong. `data.mdb` itself remains +read-only to the process, and writing to monerod's lock table is what any second +reader does. + +## Exposure + +Published by `caddy_site` on the edge host at `exmrplore.contrapeso.xyz`, with +TLS terminated there and the upstream reached over the tailnet via MagicDNS. + +The explorer listens on `0.0.0.0:8081`. Nothing on the LAN or the internet +reaches that directly: Tailscale's `ts-input` chain sits ahead of UFW, so only +tailnet peers can, and the Headscale ACL grants `tag:prd-vipy` exactly +`tag:monerod-box:8081`. A node with no ACL rule is *invisible* rather than +merely unreachable — it never appears in `tailscale status`, MagicDNS does not +resolve it, and the symptom is a Caddy 502 saying "no such host". + +**There is no authentication, by choice.** A block explorer serves public chain +data. But note what that includes: `--enable-pusher` is on (upstream's default), +so any visitor can submit a raw transaction and have *our* node relay it. That is +a deliberate decision, not an oversight — drop `xmrblocks_enable_pusher` to +`false` to close it. + +`--enable-json-api` and `--enable-autorefresh-option` are likewise upstream's +defaults. + +## Monitoring + +| Endpoint | Source | Answers | +|---|---|---| +| `services/monerod-box-local/xmrblocks` | `infra/401` via `monitored_services` | is the unit active | +| `probe/xmrblocks` | this role + `infra/403` | does it serve a real page | +| `dns/exmrplore.contrapeso.xyz`, `public/exmrplore` | `infra/402` | can the internet reach it | + +`public/exmrplore` asserts **200**, unlike the Monero RPC endpoint next door +which asserts 401 — this one genuinely has no auth in front of it. + +The timer carries no `Requires=xmrblocks.service`; see the comment in +`healthcheck.timer.j2` for why that would be backwards. + +## Backups: none, deliberately + +Everything here is derived or reproducible: the binary comes from +`scripts/build-xmrblocks.sh`, the templates ship with it, and the data it serves +is monerod's blockchain, which is itself excluded for the same reason. There is +no state on this service worth a backup job. Recorded as a decision so it does +not read as an oversight — the same treatment `phoenixd` and `monero` get. + +## Upgrading + +1. Edit the pins at the top of `scripts/build-xmrblocks.sh` (`EXPLORER_COMMIT`, + and `MONERO_VERSION` + `MONERO_SRC_SHA256` together). +2. Run it. It prints the new artifact sha256. +3. Put that in `xmrblocks_sha256` and `xmrblocks_explorer_commit` here. +4. Converge. The role asserts the checksum before copying anything. + +If the patch anchor no longer matches, the script stops — read the diff before +forcing it. + +## Runtime cost, and the RAM history + +On **2026-09-20 the VM was raised from 2 GB to 6 GB**, and that was not for the +explorer. monerod alone had outgrown 2 GB once the chain passed ~39%: it was +OOM-killed twice in one morning at 1.74 GB anon-rss, restarting every hour. With +6 GB the box sits at ~1.2 GB used of 5.9 GB with both services running +(monerod ~1.0 GB, xmrblocks ~96 MB). + +**The disk is the real performance limit, not memory.** `proxmox-tank-1` is a +spinning-disk mirror, and xmrblocks says so on startup: *"The blockchain is on a +rotating drive: this will be very slow, use an SSD if possible"*. Measured +consequences, both of which the role accounts for: + +- first start takes ~98 seconds to open the LMDB before it binds, so + `wait_for` allows 300s +- a COLD index page took **68 seconds**, while the two requests after it took + 0.87s and 0.012s — so the health check allows 90s and its unit 180s. A short + timeout there does not detect a slow explorer, it manufactures false DOWNs. + +Footprint note: xmrblocks mmaps the LMDB, so most of its apparent size is +reclaimable page cache rather than anonymous memory, and `--concurrency` is held +at 2 for the same reason. diff --git a/ansible/roles/xmrblocks/defaults/main.yml b/ansible/roles/xmrblocks/defaults/main.yml new file mode 100644 index 0000000..d90cab8 --- /dev/null +++ b/ansible/roles/xmrblocks/defaults/main.yml @@ -0,0 +1,75 @@ +# xmrblocks - the Onion Monero Blockchain Explorer. +# https://github.com/moneroexamples/onion-monero-blockchain-explorer +# +# This role INSTALLS a prebuilt binary; it does not compile one. xmrblocks links +# 21 static libraries out of a compiled monero source tree, and building that +# needs ~3.2 GB of RAM for a single compiler (~5.5 GB at -j4). monerod-box has +# 2 GB, so it cannot build this at any -j. See README.md. +# +# The artifact is produced by scripts/build-xmrblocks.sh, which runs the whole +# build in a debian:13 container - so it is reproducible on any machine with +# Docker, not just the one that happened to build it. + +# --- The artifact ------------------------------------------------------------ +# A directory on the CONTROL NODE holding `xmrblocks`, `templates/` and MANIFEST. +xmrblocks_artifact_dir: "{{ lookup('env', 'HOME') }}/personal_infra_artifacts/xmrblocks" + +# Pinned checksum of the binary we intend to ship. Asserted on the control node +# BEFORE anything is copied: shipping an unverified binary to a host is the one +# thing this role must never do. Update it together with the build pins in +# scripts/build-xmrblocks.sh. +xmrblocks_sha256: "a40c495e73a44fbb3b666fcd8a01466a2aef12ed1906558f3d5b7e458c256af6" + +# Recorded so the deployed host can be matched back to a build. +xmrblocks_explorer_commit: "29593b709edeb23daba035009ccd134a5862543a" +xmrblocks_monero_version: "0.18.5.1" + +# --- Install layout ---------------------------------------------------------- +# The binary hardcodes TMPL_DIR as "./templates", relative to the working +# directory - so templates MUST sit beside the binary and the unit MUST set +# WorkingDirectory. Get that wrong and it starts fine, then fails per-request. +xmrblocks_dir: /opt/xmrblocks + +# Runs as the monero user: the explorer opens monerod's LMDB directly, and +# /var/lib/monero is 0750 monero:monero. It opens it READ-ONLY (DBF_RDONLY in +# MicroCore::init), so it is safe alongside a running monerod. +xmrblocks_user: monero +xmrblocks_group: monero + +# --- Runtime ----------------------------------------------------------------- +# The LMDB directory, not the data directory: monerod's data-dir plus /lmdb. +xmrblocks_bc_path: /var/lib/monero/lmdb + +# 0.0.0.0 so the edge host can reach it over the tailnet. Nothing on the LAN or +# the internet can: Tailscale's ts-input chain sits ahead of UFW, and the only +# public route is the Caddy vhost on vipy. +xmrblocks_bind: "0.0.0.0" +xmrblocks_port: 8081 + +# The unrestricted RPC on loopback. Used for the mempool view and tx pushing. +xmrblocks_daemon_url: "127.0.0.1:18081" +# monerod's --rpc-login guards BOTH its RPC servers, so the explorer needs it. +xmrblocks_rpc_user: "{{ monero_rpc_user | default('') }}" +xmrblocks_rpc_password: "{{ monero_rpc_password | default('') }}" + +# Upstream's own defaults (the Dockerfile CMD). +# NOTE: enable-pusher lets a visitor submit a raw transaction through YOUR node. +# It is a deliberate choice here, not an oversight. +xmrblocks_enable_json_api: true +xmrblocks_enable_autorefresh: true +xmrblocks_enable_pusher: true + +# Crow's thread count. Held low on purpose: this box has 2 GB and monerod +# already uses ~1.3 GB of it. +xmrblocks_concurrency: 2 + +# The unit file carries the RPC login, so --diff is suppressed by default. +xmrblocks_reveal_config: false + +# --- Health check ------------------------------------------------------------ +# WHERE TO REPORT HEALTH — the one place to plug in monitoring. Empty means +# check, exit honestly, report nowhere. +healthcheck_push_url: "" +# Bearer token for the Gatus external endpoint. Required whenever a push URL +# is set: Gatus rejects an unauthenticated push with 401. +healthcheck_push_token: "" diff --git a/ansible/roles/xmrblocks/handlers/main.yml b/ansible/roles/xmrblocks/handlers/main.yml new file mode 100644 index 0000000..7be239a --- /dev/null +++ b/ansible/roles/xmrblocks/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: Restart xmrblocks + systemd: + name: xmrblocks + state: restarted + daemon_reload: yes + +- name: Restart xmrblocks health check timer + systemd: + name: xmrblocks-healthcheck.timer + state: restarted + daemon_reload: yes diff --git a/ansible/roles/xmrblocks/tasks/healthcheck.yml b/ansible/roles/xmrblocks/tasks/healthcheck.yml new file mode 100644 index 0000000..7a3a3a1 --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/healthcheck.yml @@ -0,0 +1,54 @@ +--- +# "Does the explorer actually serve a page", as opposed to infra/401's "is the +# unit active". xmrblocks can be perfectly `active` and still fail every +# request - a missing templates directory does exactly that. + +- name: Install the health check's dependencies + apt: + name: + - curl + state: present + +- name: Create the xmrblocks health check script + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: /usr/local/bin/xmrblocks-healthcheck-push.sh + owner: root + group: root + mode: '0755' + validate: "bash -n %s" + +# 0600: carries the Gatus bearer token. +- name: Create the xmrblocks health check systemd service + ansible.builtin.template: + src: healthcheck.service.j2 + dest: /etc/systemd/system/xmrblocks-healthcheck.service + owner: root + group: root + mode: '0600' + notify: Restart xmrblocks health check timer + +- name: Create the xmrblocks health check systemd timer + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: /etc/systemd/system/xmrblocks-healthcheck.timer + owner: root + group: root + mode: '0644' + notify: Restart xmrblocks health check timer + +- name: Reload systemd daemon after health check units + systemd: + daemon_reload: yes + +# Ungated: enabling a timer is deployment, not monitoring. +- name: Enable and restart the xmrblocks health check timer + systemd: + name: xmrblocks-healthcheck.timer + enabled: yes + state: restarted + daemon_reload: yes + +- name: Run the xmrblocks health check once to arm the timer + ansible.builtin.command: systemctl start xmrblocks-healthcheck.service + changed_when: false diff --git a/ansible/roles/xmrblocks/tasks/install.yml b/ansible/roles/xmrblocks/tasks/install.yml new file mode 100644 index 0000000..688e5bc --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/install.yml @@ -0,0 +1,129 @@ +--- +# Install the prebuilt xmrblocks artifact produced by scripts/build-xmrblocks.sh. +# +# The verification happens on the CONTROL NODE, before a single byte is copied. +# Shipping a binary to a host and checking it afterwards is not verification, +# it is an apology. + +- name: Assert the artifact checksum is pinned + ansible.builtin.assert: + that: + - xmrblocks_sha256 | length == 64 + quiet: true + fail_msg: >- + xmrblocks_sha256 is not set. Build the artifact with + scripts/build-xmrblocks.sh and pin the sha it prints, so this role cannot + ship a binary nobody has checked. + +- name: Check the artifact on the control node + ansible.builtin.stat: + path: "{{ xmrblocks_artifact_dir }}/xmrblocks" + checksum_algorithm: sha256 + get_checksum: yes + delegate_to: localhost + become: no + register: xmrblocks_local + changed_when: false + +- name: Assert the artifact exists and matches the pinned checksum + ansible.builtin.assert: + that: + - xmrblocks_local.stat.exists + - xmrblocks_local.stat.checksum == xmrblocks_sha256 + quiet: true + fail_msg: >- + {{ xmrblocks_artifact_dir }}/xmrblocks is + {{ 'missing - run scripts/build-xmrblocks.sh' + if not xmrblocks_local.stat.exists + else 'sha256 ' ~ xmrblocks_local.stat.checksum ~ ', expected ' ~ xmrblocks_sha256 }} + +- name: Check the templates directory on the control node + ansible.builtin.stat: + path: "{{ xmrblocks_artifact_dir }}/templates/index.html" + delegate_to: localhost + become: no + register: xmrblocks_local_templates + changed_when: false + +# The binary resolves TMPL_DIR as "./templates" at request time, so a missing +# templates directory is not a startup error - it is a 500 on the first page +# view. Catch it here instead. +- name: Assert the templates came with the artifact + ansible.builtin.assert: + that: + - xmrblocks_local_templates.stat.exists + quiet: true + fail_msg: "{{ xmrblocks_artifact_dir }}/templates/index.html is missing - the artifact is incomplete" + +- name: Install the runtime shared libraries xmrblocks needs + apt: + name: + # From `objdump -p xmrblocks | grep NEEDED`. Boost is version-locked to + # the build image's Debian release; monerod-box is Debian 13 and the + # artifact is built in debian:13, so 1.83 matches on both sides. + - libboost-chrono1.83.0 + - libboost-filesystem1.83.0 + - libboost-program-options1.83.0 + - libboost-serialization1.83.0 + - libboost-thread1.83.0 + - libhidapi-libusb0 + - libunbound8 + - libunwind8 + state: present + update_cache: yes + +- name: Create the xmrblocks directory + ansible.builtin.file: + path: "{{ xmrblocks_dir }}" + state: directory + owner: root + group: root + mode: '0755' + +- name: Install the xmrblocks binary + ansible.builtin.copy: + src: "{{ xmrblocks_artifact_dir }}/xmrblocks" + dest: "{{ xmrblocks_dir }}/xmrblocks" + owner: root + group: root + mode: '0755' + notify: Restart xmrblocks + +- name: Install the xmrblocks templates + ansible.builtin.copy: + src: "{{ xmrblocks_artifact_dir }}/templates/" + dest: "{{ xmrblocks_dir }}/templates/" + owner: root + group: root + mode: '0644' + directory_mode: '0755' + notify: Restart xmrblocks + +- name: Record which build is deployed + ansible.builtin.copy: + src: "{{ xmrblocks_artifact_dir }}/MANIFEST" + dest: "{{ xmrblocks_dir }}/MANIFEST" + owner: root + group: root + mode: '0644' + +- name: Assert the blockchain database is where we think it is + ansible.builtin.stat: + path: "{{ xmrblocks_bc_path }}/data.mdb" + # data.mdb is the whole blockchain - tens of GB and growing. stat computes a + # sha1 by DEFAULT, so without these it hashes the entire chain just to answer + # "does this file exist", and the play hangs for many minutes. + get_checksum: no + get_mime: no + get_attributes: no + register: xmrblocks_lmdb + +- name: Fail if the LMDB is missing + ansible.builtin.assert: + that: + - xmrblocks_lmdb.stat.exists + quiet: true + fail_msg: >- + No LMDB at {{ xmrblocks_bc_path }}/data.mdb. xmrblocks reads monerod's + database directly, so it has nothing to serve. Check that monerod is + deployed and its data disk is mounted. diff --git a/ansible/roles/xmrblocks/tasks/main.yml b/ansible/roles/xmrblocks/tasks/main.yml new file mode 100644 index 0000000..8a024a2 --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# import_tasks, not include_tasks: static imports stay visible to --list-tasks, +# which is the main verification tool in this repo. +- ansible.builtin.import_tasks: install.yml +- ansible.builtin.import_tasks: service.yml +- ansible.builtin.import_tasks: healthcheck.yml diff --git a/ansible/roles/xmrblocks/tasks/service.yml b/ansible/roles/xmrblocks/tasks/service.yml new file mode 100644 index 0000000..38fecd3 --- /dev/null +++ b/ansible/roles/xmrblocks/tasks/service.yml @@ -0,0 +1,57 @@ +--- +- name: Assert the daemon RPC credentials are available + ansible.builtin.assert: + that: + - xmrblocks_rpc_user | length > 0 + - xmrblocks_rpc_password | length > 0 + quiet: true + fail_msg: >- + xmrblocks needs monerod's RPC login: --rpc-login guards both of monerod's + RPC servers, so without it the mempool view and the pusher get 401. + +- name: Create the xmrblocks systemd unit + ansible.builtin.template: + src: xmrblocks.service.j2 + dest: /etc/systemd/system/xmrblocks.service + owner: root + group: root + mode: '0644' + # The unit embeds --daemon-login in ExecStart, so --diff would print the + # RPC password. -e xmrblocks_reveal_config=true when you need to see it. + diff: "{{ xmrblocks_reveal_config | default(false) | bool }}" + notify: Restart xmrblocks + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + +- name: Enable and start xmrblocks + systemd: + name: xmrblocks + enabled: yes + state: started + +- name: Flush handlers so xmrblocks is running before we probe it + meta: flush_handlers + +# Generous on purpose. xmrblocks opens the whole LMDB before it binds, and this +# pool is a spinning-disk mirror - it logs "The blockchain is on a rotating +# drive: this will be very slow" and took 98 SECONDS to come up on first start. +# 90s was not enough and produced a scary failure on a service that was fine. +- name: Wait for the xmrblocks HTTP port + ansible.builtin.wait_for: + port: "{{ xmrblocks_port }}" + host: 127.0.0.1 + delay: 5 + timeout: 300 + ignore_errors: yes + +- name: Check the xmrblocks service status + systemd: + name: xmrblocks + register: xmrblocks_status + changed_when: false + +- name: Display the xmrblocks service status + ansible.builtin.debug: + msg: "xmrblocks is {{ 'running' if xmrblocks_status.status.ActiveState == 'active' else 'NOT running' }}" diff --git a/ansible/roles/xmrblocks/templates/healthcheck.service.j2 b/ansible/roles/xmrblocks/templates/healthcheck.service.j2 new file mode 100644 index 0000000..ffae4da --- /dev/null +++ b/ansible/roles/xmrblocks/templates/healthcheck.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=Onion Monero Blockchain Explorer Health Check +After=network.target xmrblocks.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/xmrblocks-healthcheck-push.sh +# Longer than the script's 90s curl plus its push retries. A oneshot defaults to +# TimeoutStartSec=90, which would kill the check mid-request on a cold cache and +# report a failure that says nothing about the service. +TimeoutStartSec=180 +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +Environment=XMRBLOCKS_URL=http://127.0.0.1:{{ xmrblocks_port }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/xmrblocks/templates/healthcheck.sh.j2 b/ansible/roles/xmrblocks/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..2effe2a --- /dev/null +++ b/ansible/roles/xmrblocks/templates/healthcheck.sh.j2 @@ -0,0 +1,59 @@ +#!/bin/bash +# xmrblocks health check — managed by Ansible (roles/xmrblocks) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed xmrblocks-healthcheck.service +# Reporting anywhere else is optional and generic. +# +# This fetches a real page rather than just opening the port. The binary +# resolves its HTML as "./templates" relative to the working directory, so a +# deploy that got WorkingDirectory wrong listens happily on 8081 and fails +# every request. A TCP check would call that healthy. + +URL="${XMRBLOCKS_URL:-http://127.0.0.1:8081}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +report() { + local status=$1 + local msg=$2 + + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded_msg="${msg// /%20}" + local _ok=false + [ "${status}" = "up" ] && _ok=true + curl -s --max-time 15 --retry 2 -o /dev/null -X POST \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${_ok}&error=${encoded_msg}" || true +} + +# 90s, not the obvious 10 or 25. The index page renders ~46 KB from the LMDB, +# and this pool is a spinning-disk mirror shared with a syncing monerod: a COLD +# request measured 68 SECONDS, while the two immediately after it took 0.87s and +# 0.012s. A short timeout here does not detect a slow explorer, it manufactures +# a false DOWN every time the page cache has gone cold between runs. +body=$(curl -s --max-time 90 -w '\n%{http_code}' "${URL}/" 2>/dev/null) +code=$(printf '%s' "$body" | tail -1) + +if [ -z "$code" ] || [ "$code" = "000" ]; then + report "down" "xmrblocks not responding" + exit 1 +fi + +if [ "$code" != "200" ]; then + report "down" "xmrblocks HTTP ${code}" + exit 1 +fi + +# A 200 that is not the explorer is still a failure. The index page always +# carries the project name in its title. +if ! printf '%s' "$body" | grep -qi "monero"; then + report "down" "xmrblocks served 200 but not an explorer page (templates missing?)" + exit 1 +fi + +report "up" "OK" +exit 0 diff --git a/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..48713a4 --- /dev/null +++ b/ansible/roles/xmrblocks/templates/healthcheck.timer.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Onion Monero Blockchain Explorer Health Check Timer +# Deliberately NO `Requires=xmrblocks.service`. Requires on a timer stops the +# timer when the watched unit stops - "if the thing I am watching goes down, +# stop watching it" - leaving nothing to re-arm when it returns. Fulcrum's +# timer carried it and sat active+enabled for seven months without firing. + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/xmrblocks/templates/xmrblocks.service.j2 b/ansible/roles/xmrblocks/templates/xmrblocks.service.j2 new file mode 100644 index 0000000..64c58af --- /dev/null +++ b/ansible/roles/xmrblocks/templates/xmrblocks.service.j2 @@ -0,0 +1,58 @@ +[Unit] +Description=Onion Monero Blockchain Explorer (xmrblocks) +Documentation=https://github.com/moneroexamples/onion-monero-blockchain-explorer +After=network-online.target monerod.service +Wants=network-online.target +# The explorer reads monerod's LMDB directly. If the blockchain disk is not +# mounted there is nothing to serve, and without this the unit would start and +# fail on every request instead of refusing to start at all. +RequiresMountsFor={{ xmrblocks_bc_path }} + +[Service] +Type=simple +User={{ xmrblocks_user }} +Group={{ xmrblocks_group }} + +# NOT decoration: the binary hardcodes TMPL_DIR as "./templates", so it resolves +# its HTML relative to the working directory. Without this it starts happily and +# then 500s on the first page view. +WorkingDirectory={{ xmrblocks_dir }} + +ExecStart={{ xmrblocks_dir }}/xmrblocks \ + --bc-path {{ xmrblocks_bc_path }} \ + --daemon-url {{ xmrblocks_daemon_url }} \ + --daemon-login {{ xmrblocks_rpc_user }}:{{ xmrblocks_rpc_password }} \ + --bindaddr {{ xmrblocks_bind }} \ + --port {{ xmrblocks_port }} \ + --concurrency {{ xmrblocks_concurrency }}{% if xmrblocks_enable_json_api %} \ + --enable-json-api{% endif %}{% if xmrblocks_enable_autorefresh %} \ + --enable-autorefresh-option{% endif %}{% if xmrblocks_enable_pusher %} \ + --enable-pusher{% endif %} + + +Restart=always +RestartSec=20 +TimeoutStopSec=30 + +StandardOutput=journal +StandardError=journal + +# Hardening. The explorer serves static templates and reads the chain. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only + +# The LMDB directory must be WRITABLE even though the database is opened +# read-only. monero sets `mdb_flags = MDB_RDONLY` without MDB_NOLOCK +# (db_lmdb.cpp), so LMDB still registers this process in the reader table +# inside lock.mdb. With ProtectSystem=strict and no ReadWritePaths the open +# fails outright - the obvious "it only reads, so grant nothing" is wrong here. +# Writing to monerod's own lock table is normal and is what any second reader +# does; data.mdb itself stays read-only to this process. +ReadWritePaths={{ xmrblocks_bc_path }} + +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/services/headscale/templates/acl.json.j2 b/ansible/services/headscale/templates/acl.json.j2 index 6ab6fa5..5860544 100644 --- a/ansible/services/headscale/templates/acl.json.j2 +++ b/ansible/services/headscale/templates/acl.json.j2 @@ -43,6 +43,7 @@ "tag:mempool-box:8080", "tag:memos-box:5230", "tag:monerod-box:18089", + "tag:monerod-box:8081", "tag:arbret-staging-box:80", "tag:stalwart-box:25", "tag:stalwart-box:465", diff --git a/ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml b/ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml new file mode 100644 index 0000000..c3dcc29 --- /dev/null +++ b/ansible/services/xmrblocks/deploy_xmrblocks_playbook.yml @@ -0,0 +1,39 @@ +--- +# The Monero block explorer, on the same box as the node. +# +# xmrblocks reads monerod's LMDB directly rather than going through RPC, which +# is why it lives here and not on its own VM. It is a 26 MB binary and 132 KB +# of templates; the expensive part is building it, and that happens elsewhere +# (scripts/build-xmrblocks.sh) because this box has 2 GB and a single monero +# translation unit needs ~3.2 GB. See roles/xmrblocks/README.md. +- name: Deploy the Monero blockchain explorer + hosts: monero + become: yes + vars: + healthcheck_push_url: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints/probe_xmrblocks/external" + healthcheck_push_token: "{{ gatus_push_tokens[inventory_hostname] }}" + roles: + - xmrblocks + +- name: Publish the explorer from the edge host + hosts: edge + become: yes + tasks: + # No basic auth: this is a public block explorer over public chain data, + # which is what a block explorer is for. Caddy terminates TLS and proxies. + # Note the explorer has NO authentication of its own, so everything it can + # do is world-reachable - including the tx pusher, which relays a raw + # transaction through our node. That is deliberate; see the role README. + - name: Expose the Monero explorer over HTTPS + ansible.builtin.include_role: + name: caddy_site + vars: + caddy_site_name: monero-explorer + caddy_site_domain: "{{ subdomains.monero_explorer }}.{{ root_domain }}" + caddy_site_upstream: "{{ hostvars['monerod_box_local'].ansible_host }}:{{ hostvars['monerod_box_local'].xmrblocks_port }}" + caddy_site_resolvers: "100.100.100.100" + + # The per-site validate cannot catch a conflict between sites. + - name: Validate the whole Caddyfile + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + changed_when: false diff --git a/ansible/site.yml b/ansible/site.yml index c777577..284633c 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -54,6 +54,7 @@ - import_playbook: services/datum-gateway/deploy_datum_gateway_playbook.yml - import_playbook: services/mempool/deploy_mempool_playbook.yml - import_playbook: services/monero/deploy_monero_playbook.yml +- import_playbook: services/xmrblocks/deploy_xmrblocks_playbook.yml - import_playbook: services/memos/deploy_memos_playbook.yml - import_playbook: services/forgejo-runner/deploy_forgejo_runner_playbook.yml - import_playbook: services/phoenixd/deploy_phoenixd_playbook.yml diff --git a/scripts/build-xmrblocks.sh b/scripts/build-xmrblocks.sh new file mode 100755 index 0000000..44e3ede --- /dev/null +++ b/scripts/build-xmrblocks.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# +# Build xmrblocks (the Onion Monero Blockchain Explorer) reproducibly. +# +# WHY THIS SCRIPT EXISTS +# ---------------------- +# xmrblocks is not a client of monerod. It links 21 static libraries out of a +# compiled monero source tree and opens the LMDB directly, so building it means +# first compiling monero itself. Measured on Debian 13: a single cc1plus peaks +# at ~3.1 GB and a -j4 build peaks at ~5.5 GB. monerod-box has 2 GB, so it +# cannot build this at any level of parallelism - the binary has to be built +# elsewhere and shipped. +# +# Shipping a binary that only one machine can produce is how estates end up +# with artefacts nobody can rebuild. This script is the answer to that: it runs +# the whole build inside a debian:13 container, so any machine with Docker and +# ~6 GB of RAM produces the same binary. Nothing is installed on the host. +# +# WHAT IT GUARANTEES +# - the monero source is the GPG-signed release tarball, verified against +# binaryFate's key (the same chain roles/monero uses for the daemon) +# - the explorer is pinned to an exact commit +# - the one upstream patch is applied from an anchor, and the script FAILS +# if the anchor is missing rather than silently skipping it +# +# WHAT IT IS NOT +# Not bit-for-bit reproducible. Debian's toolchain, timestamps and build +# paths all leak in. It is repeatable, not deterministic; the MANIFEST +# records what went in so two builds can be compared. +# +# Usage: scripts/build-xmrblocks.sh [output-dir] +# Needs: docker, ~6 GB RAM available to it, ~2 GB disk +set -euo pipefail + +# --- Pins. Change these together and rebuild. -------------------------------- +MONERO_VERSION="0.18.5.1" +MONERO_SRC_TARBALL="monero-source-v${MONERO_VERSION}.tar.bz2" +MONERO_SRC_URL="https://downloads.getmonero.org/cli/${MONERO_SRC_TARBALL}" +# From the GPG-signed hashes.txt. Verified by the container, not trusted from here. +MONERO_SRC_SHA256="9ec6ed0fd37db9d81cf7738a5f0536cf9aec6ed8ef8fd48649a59a6aaf20de3d" +BINARYFATE_FPR="81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92" + +EXPLORER_REPO="https://github.com/moneroexamples/onion-monero-blockchain-explorer.git" +EXPLORER_COMMIT="29593b709edeb23daba035009ccd134a5862543a" + +BUILD_IMAGE="debian:13" # must match the TARGET host's Debian release +JOBS="${JOBS:-4}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SIGNING_KEY="${REPO_ROOT}/ansible/roles/monero/files/binaryfate.asc" +OUT_DIR="${1:-$HOME/personal_infra_artifacts/xmrblocks}" + +command -v docker >/dev/null || { echo "docker is required" >&2; exit 1; } +[ -f "$SIGNING_KEY" ] || { echo "missing signing key: $SIGNING_KEY" >&2; exit 1; } + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +cp "$SIGNING_KEY" "$WORK/binaryfate.asc" +mkdir -p "$OUT_DIR" + +echo "==> building xmrblocks" +echo " monero ${MONERO_VERSION} (signed source tarball)" +echo " explorer ${EXPLORER_COMMIT}" +echo " image ${BUILD_IMAGE}, -j${JOBS}" +echo " output ${OUT_DIR}" + +cat > "$WORK/build.sh" </dev/null + +cd /build + +echo "--> verifying the monero source tarball" +export GNUPGHOME=/build/gnupg; mkdir -p \$GNUPGHOME; chmod 700 \$GNUPGHOME +gpg --batch --import /build/binaryfate.asc 2>/dev/null +gpg --batch --with-colons --fingerprint | awk -F: '/^fpr:/{print \$10}' \\ + | grep -qx "${BINARYFATE_FPR}" || { echo "signing key is not binaryFate's"; exit 1; } +curl -sSL -o hashes.txt https://www.getmonero.org/downloads/hashes.txt +gpg --batch --status-fd 2 --output hashes-verified.txt --decrypt hashes.txt 2>gpg.err +grep -q "VALIDSIG.*${BINARYFATE_FPR}" gpg.err || { echo "hashes.txt did not verify"; exit 1; } +curl -sSL -o "${MONERO_SRC_TARBALL}" "${MONERO_SRC_URL}" +ACTUAL=\$(sha256sum "${MONERO_SRC_TARBALL}" | cut -d' ' -f1) +[ "\$ACTUAL" = "${MONERO_SRC_SHA256}" ] || { echo "tarball sha mismatch"; exit 1; } +grep -Fxq "\$ACTUAL ${MONERO_SRC_TARBALL}" hashes-verified.txt \\ + || { echo "tarball is not the one binaryFate signed"; exit 1; } +echo " source verified against the signed hashes" + +tar xjf "${MONERO_SRC_TARBALL}" +SRC=/build/monero-source-v${MONERO_VERSION} + +echo "--> building the monero libraries xmrblocks links against" +# Only the libraries. Upstream's Dockerfile runs a full \`make\`, which also +# builds monerod, both wallets and the test suite - none of which are linked +# into xmrblocks. USE_DEVICE_TREZOR=OFF drops the generated protobuf units too; +# an explorer has no use for hardware wallets. +mkdir -p \$SRC/build/release && cd \$SRC/build/release +cmake -D CMAKE_BUILD_TYPE=Release -D BUILD_TESTS=OFF -D USE_DEVICE_TREZOR=OFF ../.. >/dev/null +cmake --build . -j${JOBS} --target \\ + common blocks cryptonote_basic cryptonote_core multisig cryptonote_protocol \\ + daemonizer mnemonics epee lmdb device wallet-crypto blockchain_db ringct \\ + wallet cncrypto easylogging version checkpoints randomx hardforks >/dev/null + +echo "--> fetching the explorer at ${EXPLORER_COMMIT}" +cd /build +git clone -q "${EXPLORER_REPO}" explorer +cd explorer && git checkout -q "${EXPLORER_COMMIT}" + +# monero v0.18.5.0 changed get_pruned_transaction_hash from returning the hash +# to returning bool with an out-param. The explorer still calls the old form, +# so it does not compile against the daemon we actually run. Anchored on the +# exact line: if upstream fixes this, the anchor vanishes and we stop loudly +# rather than building something unexpected. +OLD=' txd.hash = get_pruned_transaction_hash(tx, tx.prunable_hash);' +NEW=' get_pruned_transaction_hash(tx, tx.prunable_hash, txd.hash);' +if grep -qF "\$OLD" src/page.h; then + sed -i "s|\$OLD|\$NEW|" src/page.h + echo " applied get_pruned_transaction_hash patch" +elif grep -qF "\$NEW" src/page.h; then + echo " upstream already uses the new signature - patch not needed" +else + echo "PATCH ANCHOR NOT FOUND in src/page.h - upstream changed; review before building" >&2 + exit 1 +fi + +echo "--> building xmrblocks" +mkdir -p build && cd build +# Xmr_MINIUPNPC_LIBRARY: FindMonero.cmake searches for miniupnpc *inside* the +# monero build tree with NO_DEFAULT_PATH, but monero no longer vendors it. +# Point it at the system library instead of patching their cmake. +cmake -D MONERO_DIR=\$SRC \\ + -D Xmr_MINIUPNPC_LIBRARY=/usr/lib/x86_64-linux-gnu/libminiupnpc.a .. >/dev/null +cmake --build . -j${JOBS} >/dev/null +[ -x xmrblocks ] || { echo "no xmrblocks produced"; exit 1; } +./xmrblocks --help >/dev/null 2>&1 || true + +echo "--> collecting" +install -m 0755 xmrblocks /out/xmrblocks +rm -rf /out/templates && cp -r templates /out/templates +chmod -R a+rX /out/templates + +{ + echo "artifact: xmrblocks" + echo "built: \$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "build-image: ${BUILD_IMAGE}" + echo "monero-version: ${MONERO_VERSION}" + echo "monero-src-sha: ${MONERO_SRC_SHA256}" + echo "explorer-commit: ${EXPLORER_COMMIT}" + echo "cmake-flags: USE_DEVICE_TREZOR=OFF BUILD_TESTS=OFF CMAKE_BUILD_TYPE=Release" + echo "patches: get_pruned_transaction_hash -> out-param form (monero >= 0.18.5.0)" + echo "xmrblocks-sha256: \$(sha256sum /out/xmrblocks | cut -d' ' -f1)" + echo "templates-sha256: \$(find /out/templates -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" + echo "needed-libs:" + objdump -p /out/xmrblocks 2>/dev/null | awk '/NEEDED/{print " " \$2}' | sort +} > /out/MANIFEST +INNER +chmod +x "$WORK/build.sh" + +docker run --rm \ + -v "$WORK":/build \ + -v "$OUT_DIR":/out \ + "$BUILD_IMAGE" bash /build/build.sh + +echo +echo "==> done" +sed -n '1,12p' "$OUT_DIR/MANIFEST" +echo +echo " pin xmrblocks_sha256 in ansible/roles/xmrblocks/defaults/main.yml to:" +grep '^xmrblocks-sha256:' "$OUT_DIR/MANIFEST" | awk '{print " " $2}' diff --git a/tofu/nodito/vms.auto.tfvars b/tofu/nodito/vms.auto.tfvars index 8e50853..40877bd 100644 --- a/tofu/nodito/vms.auto.tfvars +++ b/tofu/nodito/vms.auto.tfvars @@ -58,7 +58,11 @@ vms = { vmid = 2400 name = "monerod-box" cores = 4 - memory_mb = 2048 + # Raised from 2048 on 2026-09-20: monerod was OOM-killed hourly once the + # chain passed ~39% (1.74 GB anon-rss against 1973 MB of RAM), and the + # explorer shares this box. Applied in Proxmox; recorded here because the + # vm resource sets ignore_changes = all and will not reconcile it. + memory_mb = 6144 disk_size_gb = 10 ipconfig0 = "ip=dhcp" },