diff --git a/show-status.sh b/show-status.sh index b2e8d10b..af190464 100755 --- a/show-status.sh +++ b/show-status.sh @@ -19,7 +19,9 @@ any_failure=false # Function to run the command and handle the result check_sync_status() { local part=$1 - result=$("$BASEPATH/sync-status.sh" "${part%.yml}") + # Cap the whole per-node branch (belt-and-suspenders over check-health's own cap), so no single + # node can ever block the 'wait' below — that is what wedged the fleet rpc-update for hours. + result=$(timeout "${SYNC_TIMEOUT:-60}" "$BASEPATH/sync-status.sh" "${part%.yml}") code=0 diff --git a/sync-status.sh b/sync-status.sh index 699a233d..1f0058ec 100755 --- a/sync-status.sh +++ b/sync-status.sh @@ -176,17 +176,22 @@ for path in $pathlist; do unset LAGGING_LAG SYNCING_LAG fi - # Call the health check script with RPC_URL, ref, and chain-type flag + # Call the health check script with RPC_URL, ref, and chain-type flag. + # HARD CAP each call: a single slow/unresponsive reference RPC must never hang the parallel + # fleet status sweep (show-status.sh waits on ALL of these). On timeout -> exit 124, reported. + HC_TIMEOUT="${HC_TIMEOUT:-30}" if $is_aztec; then - $BASEPATH/check-health.sh "$RPC_URL" --aztec $ref + timeout "$HC_TIMEOUT" $BASEPATH/check-health.sh "$RPC_URL" --aztec $ref elif $is_starknet; then - $BASEPATH/check-health.sh "$RPC_URL" --starknet $ref + timeout "$HC_TIMEOUT" $BASEPATH/check-health.sh "$RPC_URL" --starknet $ref elif $is_cosmos; then - $BASEPATH/check-health.sh "$RPC_URL" --cosmos $ref + timeout "$HC_TIMEOUT" $BASEPATH/check-health.sh "$RPC_URL" --cosmos $ref else - $BASEPATH/check-health.sh "$RPC_URL" $ref + timeout "$HC_TIMEOUT" $BASEPATH/check-health.sh "$RPC_URL" $ref fi - exit $? + rc=$? + [ "$rc" -eq 124 ] && echo "timeout (health check exceeded ${HC_TIMEOUT}s)" + exit "$rc" fi done