From 2a80f0e3cad5815073842ee5672251516e73c0ef Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 19 Oct 2026 22:10:25 +0200 Subject: [PATCH] bitcoin-knots: ride out block-time RPC stalls The RPC probe flapped while every other knots check stayed green. Each burst lined up with a block: validation on the saturated HDD holds the node busy while Fulcrum, mempool and DATUM all fetch the block, and bitcoind rejected calls with "work queue depth exceeded" (93 times on 2026-10-18). - 403: the probe runs every minute, so failure-threshold 4 rides out a 1-3 minute burst while a real outage still alerts within ~5 minutes. Probes now take an optional failure_threshold over a shared probe_alert. - rpcworkqueue=64 and rpcthreads=8, so bursts queue instead of failing. Profiling later showed the stalls are per-input UTXO lookups at ~5 ms each - HDD seek latency - so this treats the symptom. The fix is chainstate on SSD. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../infra/403_service_probe_registration.yml | 28 +++++++++++++------ ansible/roles/bitcoin_knots/defaults/main.yml | 9 ++++++ .../bitcoin_knots/templates/bitcoin.conf.j2 | 2 ++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index b716faa..a46682a 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -37,7 +37,13 @@ vars: probes: - - {name: bitcoin-knots, host: knots_box_local} + # failure_threshold 4: this probe runs every MINUTE, and bitcoind turns RPC + # away for 1-3 minutes when a block lands - validation on the saturated + # HDD plus Fulcrum/mempool/DATUM all fetching the block at once (93 + # "work queue depth exceeded" in one day, 2026-10-18). At 1, every slow + # block paged. 4 consecutive failed minutes is past any block burst seen, + # and a real outage still alerts within ~5 minutes. + - {name: bitcoin-knots, host: knots_box_local, failure_threshold: 4} - {name: datum-gateway, host: knots_box_local} - {name: fulcrum, host: fulcrum_box_local} - {name: phoenixd, host: vipy} @@ -54,6 +60,16 @@ - {name: navidrome, host: music_box_local} - {name: jar-lid-archive-staging, host: jar_lid_staging_local} + # The alert every probe gets. failure-threshold is 1 - the heartbeat window + # is the tolerance, see the note above - unless a probe sets + # failure_threshold, for checks that legitimately fail for a minute or two. + probe_alert: + type: signal + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + tasks: - name: Build the probe endpoint list ansible.builtin.set_fact: @@ -61,19 +77,13 @@ 'name': item.name, 'group': 'probe', 'token': gatus_push_tokens[item.host], - 'heartbeat': item.heartbeat | default('11m')}] }}" + 'heartbeat': item.heartbeat | default('11m'), + 'alerts': [probe_alert | combine({'failure-threshold': item.failure_threshold | default(1)})]}] }}" loop: "{{ probes }}" - name: Register the probe endpoints ansible.builtin.include_role: name: gatus_endpoint vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h gatus_endpoint_name: probes gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/roles/bitcoin_knots/defaults/main.yml b/ansible/roles/bitcoin_knots/defaults/main.yml index 1045e77..01a352d 100644 --- a/ansible/roles/bitcoin_knots/defaults/main.yml +++ b/ansible/roles/bitcoin_knots/defaults/main.yml @@ -22,6 +22,15 @@ bitcoin_rpc_port: 8332 bitcoin_p2p_port: 8333 bitcoin_rpc_bind: "0.0.0.0" +# RPC capacity. When a block lands, Fulcrum, mempool and DATUM all fetch it at +# once while validation holds the node busy on a saturated HDD; with the +# defaults the queue overflowed and bitcoind REJECTED calls ("http work queue +# depth exceeded" - 93 times on 2026-10-18), which also made the health probe +# flap. A deeper queue lets those bursts wait instead of failing. Costs a few +# idle threads and a little memory. +bitcoin_rpcworkqueue: 64 +bitcoin_rpcthreads: 8 + # Build options bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores bitcoin_build_prefix: /usr/local diff --git a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 index 5277a3c..4187e24 100644 --- a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 +++ b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 @@ -11,6 +11,8 @@ rpcpassword={{ bitcoin_rpc_password }} rpcbind={{ bitcoin_rpc_bind }} rpcport={{ bitcoin_rpc_port }} rpcallowip=0.0.0.0/0 +rpcworkqueue={{ bitcoin_rpcworkqueue }} +rpcthreads={{ bitcoin_rpcthreads }} # Network Configuration listen=1