#!/bin/bash # ANSI color codes RED='\033[0;31m' YELLOW='\033[0;33m' GREEN='\033[0;32m' NC='\033[0m' # No Color # Enable debugging if needed # set -x # Check if timestamp.sh exists and is executable if [ ! -x "$(dirname "$0")/timestamp.sh" ]; then echo -e "${RED}Error: timestamp.sh script not found or not executable${NC}" echo "Looking in: $(dirname "$0")" ls -la "$(dirname "$0")" | grep timestamp.sh exit 1 fi s_to_human_readable() { local seconds=$1 local days=$((seconds / 86400)) seconds=$((seconds % 86400)) local hours=$((seconds / 3600)) seconds=$((seconds % 3600)) local minutes=$((seconds / 60)) seconds=$((seconds % 60)) printf "%d days, %02d hours, %02d minutes, %02d seconds\n" $days $hours $minutes $seconds } BASEPATH="$(dirname "$0")" source $BASEPATH/.env 2>/dev/null || true compose_base="$1" seconds_to_measure=${2:-10} if [ -z "$compose_base" ]; then echo -e "${RED}Error: compose file path (sans .yml) required${NC}" exit 1 fi compose_yml="$BASEPATH/${compose_base}.yml" if [ ! -f "$compose_yml" ]; then echo -e "${RED}Error: compose file not found: $compose_yml${NC}" exit 1 fi if [ -n "$NO_SSL" ]; then PROTO="http" DOMAIN="${DOMAIN:-0.0.0.0}" else PROTO="https" fi # Resolve traefik RPC path from compose labels (first non-blacklisted path). compose_rpc_path() { local pathlist path pathlist=$(grep -oP "stripprefix\.prefixes.*?/\K[^\"]+" "$compose_yml" 2>/dev/null) for path in $pathlist; do echo "$path" return 0 done return 1 } compose_rpc_url() { local path path=$(compose_rpc_path) || return 1 echo "$PROTO://$DOMAIN/$path" } compose_chain_slug() { grep -oP '^\s*chain:\s*\K\S+' "$compose_yml" 2>/dev/null | head -1 } # Registry protocol via x-upstreams chain label; path-substring fallback for legacy composes. resolve_protocol() { local chain_slug protocol path chain_slug=$(compose_chain_slug) protocol="" if [ -n "$chain_slug" ]; then protocol=$($BASEPATH/reference-rpc-endpoint.sh --protocol "$chain_slug" 2>/dev/null) || protocol="" fi if [ -z "$protocol" ]; then path=$(compose_rpc_path 2>/dev/null) || path="" if echo "$path" | grep -qi "starknet"; then protocol="starknet" elif echo "$path" | grep -qi "aztec"; then protocol="aztec" elif echo "$path" | grep -qiE "bitcoin|digibyte|dogecoin|dash"; then protocol="bitcoin" elif echo "$path" | grep -qi "ripple"; then protocol="ripple" elif echo "$path" | grep -qiE "cosmos|babylon|haqq|gaiad|tac"; then protocol="cosmos" else protocol="eth" fi fi echo "$protocol" } # Bitcoin-family RPC URL (traefik path) and basic-auth from x-upstreams (fallback username:password). bitcoin_rpc_url() { compose_rpc_url } bitcoin_rpc_auth() { # Prefer creds from compose x-upstreams basic-auth; per-chain rpcauth may differ on-host. local user pass user=$(grep -A5 'basic-auth:' "$compose_yml" 2>/dev/null | grep -oP '^\s*username:\s*\K\S+' | head -1) pass=$(grep -A5 'basic-auth:' "$compose_yml" 2>/dev/null | grep -oP '^\s*password:\s*\K\S+' | head -1) [ -z "$user" ] && user="username" [ -z "$pass" ] && pass="password" echo "${user}:${pass}" } bitcoin_rpc_call() { local method="$1" local params="${2:-[]}" local url auth url=$(bitcoin_rpc_url) || return 1 auth=$(bitcoin_rpc_auth) curl -L --ipv4 -m 5 -s -u "$auth" -X POST -H "Content-Type: application/json" \ --data "{\"jsonrpc\":\"1.0\",\"id\":1,\"method\":\"$method\",\"params\":$params}" "$url" } bitcoin_parse_blocks_headers() { local info="$1" local blocks headers blocks=$(echo "$info" | jq -r '.result.blocks // empty' 2>/dev/null) headers=$(echo "$info" | jq -r '.result.headers // empty' 2>/dev/null) if [[ ! "$blocks" =~ ^[0-9]+$ ]] || [[ ! "$headers" =~ ^[0-9]+$ ]]; then return 1 fi echo "$blocks $headers" } bitcoin_reference_height() { local chain_slug="$1" local ref_urls ref_url resp height ref_urls=$($BASEPATH/reference-rpc-endpoint.sh --chain "$chain_slug" 2>/dev/null) || return 1 ref_url=$(echo "$ref_urls" | awk '{print $1}') [ -z "$ref_url" ] && return 1 resp=$(curl -L --ipv4 -m 5 -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"1.0","id":1,"method":"getblockcount","params":[]}' "$ref_url") if [ -z "$resp" ] || echo "$resp" | grep -qi 'Unknown network'; then return 2 fi height=$(echo "$resp" | jq -r '.result // empty' 2>/dev/null) if [[ ! "$height" =~ ^[0-9]+$ ]]; then return 2 fi echo "$height" } bitcoin_pick_seed_peer() { local chain_slug="$1" path path=$(compose_rpc_path 2>/dev/null) || path="" if echo "$chain_slug $path" | grep -qi "digibyte"; then echo "seed.digibyte.co:12024" return 0 fi if echo "$chain_slug $path" | grep -qi "dogecoin"; then # TODO: dogecoin wire seeds return 1 fi if echo "$chain_slug $path" | grep -qi "dash"; then # TODO: dash wire seeds return 1 fi # Bitcoin mainnet / testnet / cash family default echo "seed.bitcoin.sipa.be:8333" } bitcoin_try_addnode() { local chain_slug="$1" peer resp peer="" local ref_urls ref_url pinfo ref_urls=$($BASEPATH/reference-rpc-endpoint.sh --chain "$chain_slug" 2>/dev/null) || ref_urls="" ref_url=$(echo "$ref_urls" | awk '{print $1}') if [ -n "$ref_url" ]; then pinfo=$(curl -L --ipv4 -m 5 -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"1.0","id":1,"method":"getpeerinfo","params":[]}' "$ref_url" 2>/dev/null) peer=$(echo "$pinfo" | jq -r '.result[0].addr // empty' 2>/dev/null) fi if [ -z "$peer" ] || [ "$peer" = "null" ]; then peer=$(bitcoin_pick_seed_peer "$chain_slug") || return 0 fi resp=$(bitcoin_rpc_call "addnode" "[\"$peer\", \"onetry\"]") if echo "$resp" | jq -e '.error == null' >/dev/null 2>&1; then echo "Issued harmless addnode onetry to $peer" fi } report_block_catchup_rate() { local gap_change="$1" local interval="$2" local block_time_ms="$3" local gap_remaining="$4" local label="${5:-blocks}" local block_time_sec realtime_units effective catchup_rate block_time_sec=$(echo "scale=6; $block_time_ms / 1000" | bc) realtime_units=$(echo "scale=6; $interval / $block_time_sec" | bc) effective=$(echo "scale=6; $realtime_units - $gap_change" | bc) catchup_rate=$(echo "scale=3; $effective / $realtime_units" | bc) echo "First measurement: $label behind = $((gap_remaining - gap_change)) $label" echo "Second measurement: $label behind = $gap_remaining $label" echo "Gap change in $interval seconds: $gap_change $label" echo "Calculated catchup rate: ${catchup_rate}x realtime" if (( $(echo "$catchup_rate < 0" | bc -l) )); then local falling_behind_rate falling_behind_rate=$(echo "scale=2; -1 * $catchup_rate" | bc) echo -e "${RED}Node is FALLING BEHIND by ${falling_behind_rate}x realtime${NC}" echo -e "${RED}The gap increased by $gap_change $label over $interval seconds${NC}" return 1 elif (( $(echo "$catchup_rate < 1" | bc -l) )); then echo -e "${YELLOW}WARNING: Node is processing at ${catchup_rate}x realtime${NC}" echo -e "${YELLOW}This is too slow - node will never catch up to chain head${NC}" return 2 else echo -e "${GREEN}Node is processing at ${catchup_rate}x realtime${NC}" local time_to_catchup time_to_catchup=$(echo "scale=0; $gap_remaining * $block_time_sec / ($catchup_rate - 1)" | bc) echo -e "${GREEN}Estimated time until synced:${NC}" s_to_human_readable "$time_to_catchup" return 0 fi } bitcoin_catchup() { local chain_slug="$1" local interval="$2" local block_time_ms info1 info2 blocks1 headers1 blocks2 headers2 local ref1 ref2 use_pseudo=false gap1 gap2 gap_change block_time_ms=$($BASEPATH/reference-rpc-endpoint.sh --block-time-ms "$chain_slug" 2>/dev/null) || block_time_ms="600000" echo "Checking sync status for $compose_base (bitcoin-family)..." info1=$(bitcoin_rpc_call "getblockchaininfo") if [ -z "$info1" ]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi read -r blocks1 headers1 < <(bitcoin_parse_blocks_headers "$info1") || { echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 } ref1=$(bitcoin_reference_height "$chain_slug") local ref_rc1=$? if [ "$ref_rc1" -eq 2 ]; then use_pseudo=true ref1=$headers1 echo "Reference RPC unavailable (Unknown network) — using local headers as pseudo-reference" elif [ "$ref_rc1" -ne 0 ]; then use_pseudo=true ref1=$headers1 echo "Reference RPC unavailable — using local headers as pseudo-reference" fi gap1=$((ref1 - blocks1)) echo "Current chain head is $gap1 blocks behind reference" s_to_human_readable $(echo "scale=0; $gap1 * $block_time_ms / 1000" | bc) echo "Measuring progress over $interval seconds..." sleep "$interval" info2=$(bitcoin_rpc_call "getblockchaininfo") if [ -z "$info2" ]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi read -r blocks2 headers2 < <(bitcoin_parse_blocks_headers "$info2") || { echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 } if $use_pseudo; then ref2=$headers2 else ref2=$(bitcoin_reference_height "$chain_slug") local ref_rc2=$? if [ "$ref_rc2" -ne 0 ]; then ref2=$headers2 use_pseudo=true fi fi gap2=$((ref2 - blocks2)) gap_change=$((gap2 - gap1)) report_block_catchup_rate "$gap_change" "$interval" "$block_time_ms" "$gap2" "blocks" local rc=$? if [ "$rc" -eq 1 ] && [ -n "$chain_slug" ]; then bitcoin_try_addnode "$chain_slug" fi exit "$rc" } ripple_rpc_call() { local url url=$(compose_rpc_url) || return 1 curl -L --ipv4 -m 5 -s -X POST -H "Content-Type: application/json" \ --data '{"method":"server_info"}' "$url" } ripple_validated_seq() { local resp="$1" seq seq=$(echo "$resp" | jq -r '.result.info.validated_ledger.seq // .info.validated_ledger.seq // empty' 2>/dev/null) if [[ ! "$seq" =~ ^[0-9]+$ ]]; then return 1 fi echo "$seq" } ripple_reference_seq() { local chain_slug="$1" ref_urls ref_url resp ref_urls=$($BASEPATH/reference-rpc-endpoint.sh --chain "$chain_slug" 2>/dev/null) || return 1 ref_url=$(echo "$ref_urls" | awk '{print $1}') [ -z "$ref_url" ] && return 1 resp=$(curl -L --ipv4 -m 5 -s -X POST -H "Content-Type: application/json" \ --data '{"method":"server_info"}' "$ref_url") ripple_validated_seq "$resp" } ripple_catchup() { local chain_slug="$1" local interval="$2" local block_time_ms resp1 resp2 seq1 seq2 ref1 ref2 gap1 gap2 gap_change block_time_ms=$($BASEPATH/reference-rpc-endpoint.sh --block-time-ms "$chain_slug" 2>/dev/null) || block_time_ms="4000" echo "Checking sync status for $compose_base (ripple)..." resp1=$(ripple_rpc_call) if [ -z "$resp1" ]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi seq1=$(ripple_validated_seq "$resp1") || { echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 } ref1=$(ripple_reference_seq "$chain_slug") || ref1="" if [ -z "$ref1" ]; then gap1=0 echo "Reference RPC unavailable — measuring local validated-ledger progress only" else gap1=$((ref1 - seq1)) echo "Current chain head is $gap1 ledgers behind reference" s_to_human_readable $(echo "scale=0; $gap1 * $block_time_ms / 1000" | bc) fi echo "Measuring progress over $interval seconds..." sleep "$interval" resp2=$(ripple_rpc_call) if [ -z "$resp2" ]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi seq2=$(ripple_validated_seq "$resp2") || { echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 } ref2=$(ripple_reference_seq "$chain_slug") || ref2="" if [ -z "$ref1" ] || [ -z "$ref2" ]; then local seq_gain=$((seq2 - seq1)) local realtime_units effective catchup_rate realtime_units=$(echo "scale=6; $interval * 1000 / $block_time_ms" | bc) effective=$(echo "scale=6; $seq_gain" | bc) catchup_rate=$(echo "scale=3; $effective / $realtime_units" | bc) echo "Validated ledger gain in $interval seconds: $seq_gain ledgers" echo "Calculated catchup rate: ${catchup_rate}x realtime" if (( $(echo "$catchup_rate < 1" | bc -l) )); then if (( $(echo "$catchup_rate < 0" | bc -l) )); then echo -e "${RED}Node is FALLING BEHIND${NC}" exit 1 fi echo -e "${YELLOW}WARNING: Node is processing at ${catchup_rate}x realtime${NC}" exit 2 fi echo -e "${GREEN}Node is processing at ${catchup_rate}x realtime${NC}" exit 0 fi gap2=$((ref2 - seq2)) gap_change=$((gap2 - gap1)) report_block_catchup_rate "$gap_change" "$interval" "$block_time_ms" "$gap2" "ledgers" exit $? } cosmos_catching_up() { local url status catching_up url=$(compose_rpc_url) || return 1 status=$(curl -L --ipv4 -m 5 -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","id":1,"method":"status"}' "$url") catching_up=$(echo "$status" | jq -r '.result.sync_info.catching_up // .sync_info.catching_up // empty' 2>/dev/null) echo "$catching_up" } cosmos_catchup() { local catching_up echo "Checking sync status for $compose_base (cosmos/cometbft)..." catching_up=$(cosmos_catching_up) if [ "$catching_up" = "true" ]; then echo "CometBFT reports catching_up=true (node is still syncing)" elif [ -z "$catching_up" ]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi # Standard timestamp-lag algorithm via timestamp.sh (cosmos /status source). echo "Debug: Running ./timestamp.sh $compose_base" timestamp_output=$(./timestamp.sh "$compose_base" 2>&1) timestamp_exit_code=$? echo "Debug: timestamp.sh output: '$timestamp_output'" echo "Debug: timestamp.sh exit code: $timestamp_exit_code" latest_block_timestamp_decimal=$(echo "$timestamp_output" | grep -o '[0-9]\+' | head -1) if [[ $timestamp_exit_code -ne 0 || -z "$latest_block_timestamp_decimal" ]]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi current_time=$(date +%s) time_difference=$((current_time - latest_block_timestamp_decimal)) echo "Current chain head is $time_difference seconds behind real time" if [ "$catching_up" = "true" ]; then echo "(catching_up=true)" fi s_to_human_readable $time_difference echo "Measuring progress over $seconds_to_measure seconds..." sleep $seconds_to_measure catching_up=$(cosmos_catching_up) if [ "$catching_up" = "true" ]; then echo "CometBFT still reports catching_up=true" fi echo "Debug: Running ./timestamp.sh $compose_base (second measurement)" timestamp_output=$(./timestamp.sh "$compose_base" 2>&1) timestamp_exit_code=$? echo "Debug: timestamp.sh output (second): '$timestamp_output'" echo "Debug: timestamp.sh exit code (second): $timestamp_exit_code" latest_block_timestamp2=$(echo "$timestamp_output" | grep -o '[0-9]\+' | head -1) if [[ $timestamp_exit_code -ne 0 || -z "$latest_block_timestamp2" ]]; then echo -e "${RED}rpc-not-open (expected mid-sync for stage-sync/bootstrap clients — avalanchego, erigon/reth stage-sync, statesync bootstraps); re-run at chainhead${NC}" exit 1 fi current_time2=$(date +%s) time_difference2=$((current_time2 - latest_block_timestamp2)) gap_change=$((time_difference2 - time_difference)) effective_catchup_seconds=$((seconds_to_measure - gap_change)) catchup_rate=$(echo "scale=3; $effective_catchup_seconds / $seconds_to_measure" | bc) echo "First measurement: Time behind = $time_difference seconds" echo "Second measurement: Time behind = $time_difference2 seconds" if [ "$catching_up" = "true" ]; then echo "(catching_up=true)" fi echo "Gap change in $seconds_to_measure seconds: $gap_change seconds" echo "Calculated catchup rate: $catchup_rate× realtime" if (( $(echo "$catchup_rate < 0" | bc -l) )); then falling_behind_rate=$(echo "scale=2; -1 * $catchup_rate" | bc) echo -e "${RED}Node is FALLING BEHIND by $falling_behind_rate× realtime${NC}" echo -e "${RED}The gap increased by $gap_change seconds over $seconds_to_measure seconds${NC}" exit 1 elif (( $(echo "$catchup_rate < 1" | bc -l) )); then echo -e "${YELLOW}WARNING: Node is processing at $catchup_rate× realtime${NC}" echo -e "${YELLOW}This is too slow - node will never catch up to chain head${NC}" exit 2 else echo -e "${GREEN}Node is processing at $catchup_rate× realtime${NC}" time_to_catchup=$(echo "scale=0; $time_difference2 / ($catchup_rate - 1)" | bc) echo -e "${GREEN}Estimated time until synced:${NC}" s_to_human_readable $time_to_catchup exit 0 fi } protocol=$(resolve_protocol) chain_slug=$(compose_chain_slug) case "$protocol" in bitcoin) bitcoin_catchup "$chain_slug" "$seconds_to_measure" ;; ripple) ripple_catchup "$chain_slug" "$seconds_to_measure" ;; cosmos) cosmos_catchup ;; eth|starknet|aztec) ;; *) echo "unsupported protocol $protocol for catchup" exit 1 ;; esac # --- eth / starknet / aztec: original timestamp-lag path (unchanged) --- seconds_to_measure=${2:-10} # First measurement echo "Checking sync status for $1..." echo "Debug: Running ./timestamp.sh $1" timestamp_output=$(./timestamp.sh $1 2>&1) timestamp_exit_code=$? echo "Debug: timestamp.sh output: '$timestamp_output'" echo "Debug: timestamp.sh exit code: $timestamp_exit_code" # Try to extract a number, ignoring any errors or other text latest_block_timestamp_decimal=$(echo "$timestamp_output" | grep -o '[0-9]\+' | head -1) if [[ $timestamp_exit_code -ne 0 || -z "$latest_block_timestamp_decimal" ]]; then echo -e "${RED}Error: Failed to get valid block timestamp. Is the node running?${NC}" echo -e "${RED}timestamp.sh returned: $timestamp_output${NC}" exit 1 fi current_time=$(date +%s) time_difference=$((current_time - latest_block_timestamp_decimal)) # Check for reasonable time difference echo "Current chain head is $time_difference seconds behind real time" s_to_human_readable $time_difference # Wait to measure progress echo "Measuring progress over $seconds_to_measure seconds..." sleep $seconds_to_measure # Second measurement echo "Debug: Running ./timestamp.sh $1 (second measurement)" timestamp_output=$(./timestamp.sh $1 2>&1) timestamp_exit_code=$? echo "Debug: timestamp.sh output (second): '$timestamp_output'" echo "Debug: timestamp.sh exit code (second): $timestamp_exit_code" # Try to extract a number, ignoring any errors or other text latest_block_timestamp2=$(echo "$timestamp_output" | grep -o '[0-9]\+' | head -1) if [[ $timestamp_exit_code -ne 0 || -z "$latest_block_timestamp2" ]]; then echo -e "${RED}Error: Failed to get valid block timestamp on second measurement${NC}" echo -e "${RED}timestamp.sh returned: $timestamp_output${NC}" exit 1 fi current_time2=$(date +%s) time_difference2=$((current_time2 - latest_block_timestamp2)) # Calculate the gap change # Positive = gap increased (falling behind) # Negative = gap decreased (catching up) gap_change=$((time_difference2 - time_difference)) # Calculate catch-up rate (how many seconds of blockchain time processed in 1 second real time) effective_catchup_seconds=$((seconds_to_measure - gap_change)) catchup_rate=$(echo "scale=3; $effective_catchup_seconds / $seconds_to_measure" | bc) # Display debug info echo "First measurement: Time behind = $time_difference seconds" echo "Second measurement: Time behind = $time_difference2 seconds" echo "Gap change in $seconds_to_measure seconds: $gap_change seconds" echo "Calculated catchup rate: $catchup_rate× realtime" # Interpret results if (( $(echo "$catchup_rate < 0" | bc -l) )); then # Node is falling behind falling_behind_rate=$(echo "scale=2; -1 * $catchup_rate" | bc) echo -e "${RED}Node is FALLING BEHIND by $falling_behind_rate× realtime${NC}" echo -e "${RED}The gap increased by $gap_change seconds over $seconds_to_measure seconds${NC}" exit 1 elif (( $(echo "$catchup_rate < 1" | bc -l) )); then # Node is processing slower than realtime echo -e "${YELLOW}WARNING: Node is processing at $catchup_rate× realtime${NC}" echo -e "${YELLOW}This is too slow - node will never catch up to chain head${NC}" exit 2 else # Node is catching up echo -e "${GREEN}Node is processing at $catchup_rate× realtime${NC}" # Calculate time until caught up time_to_catchup=$(echo "scale=0; $time_difference2 / ($catchup_rate - 1)" | bc) echo -e "${GREEN}Estimated time until synced:${NC}" s_to_human_readable $time_to_catchup exit 0 fi