diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index b716faa..a46682a 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -37,7 +37,13 @@ vars: probes: - - {name: bitcoin-knots, host: knots_box_local} + # failure_threshold 4: this probe runs every MINUTE, and bitcoind turns RPC + # away for 1-3 minutes when a block lands - validation on the saturated + # HDD plus Fulcrum/mempool/DATUM all fetching the block at once (93 + # "work queue depth exceeded" in one day, 2026-10-18). At 1, every slow + # block paged. 4 consecutive failed minutes is past any block burst seen, + # and a real outage still alerts within ~5 minutes. + - {name: bitcoin-knots, host: knots_box_local, failure_threshold: 4} - {name: datum-gateway, host: knots_box_local} - {name: fulcrum, host: fulcrum_box_local} - {name: phoenixd, host: vipy} @@ -54,6 +60,16 @@ - {name: navidrome, host: music_box_local} - {name: jar-lid-archive-staging, host: jar_lid_staging_local} + # The alert every probe gets. failure-threshold is 1 - the heartbeat window + # is the tolerance, see the note above - unless a probe sets + # failure_threshold, for checks that legitimately fail for a minute or two. + probe_alert: + type: signal + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h + tasks: - name: Build the probe endpoint list ansible.builtin.set_fact: @@ -61,19 +77,13 @@ 'name': item.name, 'group': 'probe', 'token': gatus_push_tokens[item.host], - 'heartbeat': item.heartbeat | default('11m')}] }}" + 'heartbeat': item.heartbeat | default('11m'), + 'alerts': [probe_alert | combine({'failure-threshold': item.failure_threshold | default(1)})]}] }}" loop: "{{ probes }}" - name: Register the probe endpoints ansible.builtin.include_role: name: gatus_endpoint vars: - gatus_endpoint_default_alerts: - - type: signal - # 1, not 3: the heartbeat window is the tolerance. See the note above. - failure-threshold: 1 - success-threshold: 2 - send-on-resolved: true - minimum-reminder-interval: 6h gatus_endpoint_name: probes gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/roles/bitcoin_knots/defaults/main.yml b/ansible/roles/bitcoin_knots/defaults/main.yml index 1045e77..01a352d 100644 --- a/ansible/roles/bitcoin_knots/defaults/main.yml +++ b/ansible/roles/bitcoin_knots/defaults/main.yml @@ -22,6 +22,15 @@ bitcoin_rpc_port: 8332 bitcoin_p2p_port: 8333 bitcoin_rpc_bind: "0.0.0.0" +# RPC capacity. When a block lands, Fulcrum, mempool and DATUM all fetch it at +# once while validation holds the node busy on a saturated HDD; with the +# defaults the queue overflowed and bitcoind REJECTED calls ("http work queue +# depth exceeded" - 93 times on 2026-10-18), which also made the health probe +# flap. A deeper queue lets those bursts wait instead of failing. Costs a few +# idle threads and a little memory. +bitcoin_rpcworkqueue: 64 +bitcoin_rpcthreads: 8 + # Build options bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores bitcoin_build_prefix: /usr/local diff --git a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 index 5277a3c..4187e24 100644 --- a/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 +++ b/ansible/roles/bitcoin_knots/templates/bitcoin.conf.j2 @@ -11,6 +11,8 @@ rpcpassword={{ bitcoin_rpc_password }} rpcbind={{ bitcoin_rpc_bind }} rpcport={{ bitcoin_rpc_port }} rpcallowip=0.0.0.0/0 +rpcworkqueue={{ bitcoin_rpcworkqueue }} +rpcthreads={{ bitcoin_rpcthreads }} # Network Configuration listen=1