diff --git a/README.md b/README.md index 0a8f1f8e..c179f013 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,15 @@ This directory includes several useful scripts to help you manage and monitor yo ./stop.sh && ./rm.sh && ./delete-volumes.sh && ./force-recreate.sh && ./logs.sh ``` +#### Debugging tips + +To get the configuration name for one of the commands use `./show-status.sh` which lists all the configrations and their status to copy paste for further inspection with e.g. `./catchup.sh ` or repeated use of `./latest.sh ` which will give you and idea if the sync is actually progressing and if it is on the canonical chain. +Note: some configurations use staged sync which means that there is no measurable progress on the RPC in between bacthes of processed blocks. In any case `./logs.sh ` will give you insights into problems, potentially filtered by a LLM to spot common errors. It could be that clients are syncing slower than the chain progresses. + +#### Further automation + +You can chain `./success-if-almost-synced.sh ` with other scripts to create more complex automation, e.g. notify you once a node synced up to chainhead or adding the node to the dshackle configuration or taking a backup to clone the node to a different server. + #### OP Wheel Usage Example ```bash diff --git a/catchup.sh b/catchup.sh index 3d9a0942..640ca1e6 100755 --- a/catchup.sh +++ b/catchup.sh @@ -1,5 +1,11 @@ #!/bin/bash +# ANSI color codes +RED='\033[0;31m' +YELLOW='\033[0;33m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + s_to_human_readable() { local ms=$1 local days=$((ms / 86400)) @@ -17,28 +23,48 @@ BASEPATH="$(dirname "$0")" source $BASEPATH/.env seconds_to_measure=${2:-10} +# Assume 1 block per 12 seconds as default chain block time +chain_block_time=${3:-12} +# First measurement latest_block_timestamp_decimal=$(./timestamp.sh $1) current_time=$(date +%s) time_difference=$((current_time - latest_block_timestamp_decimal)) -#echo "$latest_block_timestamp_decimal $current_time $time_difference" +echo "Current chain head is $time_difference seconds behind real time" +# s_to_human_readable $time_difference -#s_to_human_readable $time_difference +# Wait to measure progress sleep $seconds_to_measure +# Second measurement latest_block_timestamp_decimal=$(./timestamp.sh $1) current_time=$(date +%s) time_difference2=$((current_time - latest_block_timestamp_decimal)) -#echo "$latest_block_timestamp_decimal $current_time $time_difference2" - -#s_to_human_readable $time_difference2 +# Calculate catchup rate progress=$((time_difference - time_difference2)) -progress_per_second=$((progress / seconds_to_measure)) -#echo "$progress_per_second" -#s_to_human_readable $progress_per_second +progress_per_second=$(echo "scale=4; $progress / $seconds_to_measure" | bc) -result=$(echo "scale=0; $time_difference2 / $progress_per_second" | bc) -#echo "$result" -s_to_human_readable $result +# Calculate time to catch up +if (( $(echo "$progress_per_second <= 0" | bc -l) )); then + echo -e "${RED}ERROR: Node is not catching up! It's falling behind by $(echo "scale=2; -1 * $progress_per_second" | bc) seconds per second.${NC}" + exit 1 +fi + +# Time until caught up +time_to_catchup=$(echo "scale=0; $time_difference2 / $progress_per_second" | bc) + +# Calculate if catchup rate is faster than chain growth rate +# Chain growth is typically 1 second of block time per second of real time +chain_growth_rate=1.0 # 1 second per second as baseline +catchup_needed=$(echo "scale=4; $chain_growth_rate + 0.01" | bc) # Slight buffer for safety + +if (( $(echo "$progress_per_second < $catchup_needed" | bc -l) )); then + echo -e "${YELLOW}WARNING: Node catchup rate ($progress_per_second seconds/second) is slower than chain growth rate ($chain_growth_rate seconds/second).${NC}" + echo -e "${YELLOW}The node will likely never catch up to the chain head!${NC}" +else + echo -e "${GREEN}Node catchup rate is good: $progress_per_second seconds/second${NC}" + echo -e "${GREEN}Estimated time to sync:${NC}" + s_to_human_readable $time_to_catchup +fi