#!/bin/bash BASEPATH="$(dirname "$0")" source $BASEPATH/.env IFS=':' read -ra parts <<< $COMPOSE_FILE blacklist=( "drpc.yml" "drpc-free.yml" "drpc-home.yml" # dshackles "arbitrum-one-mainnet-arbnode-archive-trace.yml" # always behind and no reference rpc "ethereum-beacon-mainnet-lighthouse-pruned-blobs" # can't handle beacon rest api yet "rpc.yml" "monitoring.yml" "ftp.yml" "backup-http.yml" "base.yml" # no rpcs ) # Flag to track if any invocation failed for the alert scripts any_failure=false # Function to run the command and handle the result check_sync_status() { local part=$1 # Cap the whole per-node branch (belt-and-suspenders over check-health's own cap), so no single # node can ever block the 'wait' below — that is what wedged the fleet rpc-update for hours. result=$(timeout "${SYNC_TIMEOUT:-60}" "$BASEPATH/sync-status.sh" "${part%.yml}") # Capture the status IMMEDIATELY. Any command in between - including a plain # assignment like `code=0` - overwrites $? with its own (always 0) status. rc=$? code=0 if [ "$rc" -ne 0 ]; then if [[ "$result" == *"syncing"* ]] || [[ "$result" == *"lagging"* ]]; then # sync-status exits 1 for syncing/lagging; those are expected states, # not failures. code=0 else code=1 fi fi echo "${part%.yml}: $result" # NOTE: do NOT set any_failure here. This function runs backgrounded (`&`), so # it executes in a subshell and any variable it sets is discarded. Failure is # propagated to the parent through this return code, collected by `wait` below. return "$code" } for part in "${parts[@]}"; do include=true for word in "${blacklist[@]}"; do if echo "$part" | grep -qE "$word"; then include=false fi done # Check if any parameters were passed if [ $# -gt 0 ]; then # Put parameters into an array (list) params=("$@") # Check if a string is part of the list if [[ " ${params[@]} " =~ " $1 " ]]; then include=$include # don't change anything else include=false fi fi if $include; then check_sync_status "$part" & pids+=($!) # Save the process ID for waiting later fi done # Wait for all background processes to finish. `wait` runs in the PARENT shell, so # this is where a failing node can actually flip any_failure - the checker itself # cannot, being a subshell. Previously the status was discarded here, which silently # neutered the exit code. for pid in "${pids[@]}"; do wait "$pid" || any_failure=true done # Fenced nodes (fleet-state maintenance windows) are dropped from COMPOSE_FILE # by rpc-update, so they never appear above even though their containers keep # running. The .fenced marker is written by the same rpc-update task that # fences them and is rewritten every reconcile (empty once windows expire). if [ -s "$BASEPATH/.fenced" ]; then echo "" echo "Fenced (maintenance window - running but unmanaged, not checked above):" sed 's/^/ /' "$BASEPATH/.fenced" fi # If any invocation failed, return a failure exit code if $any_failure; then exit 1 fi