fix

README.md

@@ -135,6 +135,15 @@ This directory includes several useful scripts to help you manage and monitor yo
 ./stop.sh <config-name> && ./rm.sh <config-name> && ./delete-volumes.sh <config-name> && ./force-recreate.sh <config-name> && ./logs.sh <config-name>
 ```
 
+#### Debugging tips
+
+To get the configuration name for one of the commands use `./show-status.sh` which lists all the configrations and their status to copy paste for further inspection with e.g. `./catchup.sh <config-name>` or repeated use of `./latest.sh <config-name>` which will give you and idea if the sync is actually progressing and if it is on the canonical chain.
+Note: some configurations use staged sync which means that there is no measurable progress on the RPC in between bacthes of processed blocks. In any case `./logs.sh <config-name>` will give you insights into problems, potentially filtered by a LLM to spot common errors. It could be that clients are syncing slower than the chain progresses.
+
+#### Further automation
+
+You can chain `./success-if-almost-synced.sh <config-name> <age-of-last-block-in-seconds-to-be-considered-almost-synced>` with other scripts to create more complex automation, e.g. notify you once a node synced up to chainhead or adding the node to the dshackle configuration or taking a backup to clone the node to a different server.
+
 #### OP Wheel Usage Example
 
 ```bash

catchup.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+# ANSI color codes
+RED='\033[0;31m'
+YELLOW='\033[0;33m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
 s_to_human_readable() {
     local ms=$1
     local days=$((ms / 86400))
@@ -17,28 +23,48 @@ BASEPATH="$(dirname "$0")"
 source $BASEPATH/.env
 
 seconds_to_measure=${2:-10}
+# Assume 1 block per 12 seconds as default chain block time
+chain_block_time=${3:-12}
 
+# First measurement
 latest_block_timestamp_decimal=$(./timestamp.sh $1)
 current_time=$(date +%s)
 time_difference=$((current_time - latest_block_timestamp_decimal))	       
 
-#echo "$latest_block_timestamp_decimal $current_time $time_difference"
+echo "Current chain head is $time_difference seconds behind real time"
-
 # s_to_human_readable $time_difference
+
+# Wait to measure progress
 sleep $seconds_to_measure
 
+# Second measurement
 latest_block_timestamp_decimal=$(./timestamp.sh $1)
 current_time=$(date +%s)
 time_difference2=$((current_time - latest_block_timestamp_decimal))	       
 
-#echo "$latest_block_timestamp_decimal $current_time $time_difference2"
+# Calculate catchup rate
-
-#s_to_human_readable $time_difference2
 progress=$((time_difference - time_difference2))
-progress_per_second=$((progress / seconds_to_measure))
+progress_per_second=$(echo "scale=4; $progress / $seconds_to_measure" | bc)
-#echo "$progress_per_second"
-#s_to_human_readable $progress_per_second
 
-result=$(echo "scale=0; $time_difference2 / $progress_per_second" | bc)
+# Calculate time to catch up
-#echo "$result"
+if (( $(echo "$progress_per_second <= 0" | bc -l) )); then
-s_to_human_readable $result
+    echo -e "${RED}ERROR: Node is not catching up! It's falling behind by $(echo "scale=2; -1 * $progress_per_second" | bc) seconds per second.${NC}"
+    exit 1
+fi
+
+# Time until caught up
+time_to_catchup=$(echo "scale=0; $time_difference2 / $progress_per_second" | bc)
+
+# Calculate if catchup rate is faster than chain growth rate
+# Chain growth is typically 1 second of block time per second of real time
+chain_growth_rate=1.0  # 1 second per second as baseline
+catchup_needed=$(echo "scale=4; $chain_growth_rate + 0.01" | bc)  # Slight buffer for safety
+
+if (( $(echo "$progress_per_second < $catchup_needed" | bc -l) )); then
+    echo -e "${YELLOW}WARNING: Node catchup rate ($progress_per_second seconds/second) is slower than chain growth rate ($chain_growth_rate seconds/second).${NC}"
+    echo -e "${YELLOW}The node will likely never catch up to the chain head!${NC}"
+else
+    echo -e "${GREEN}Node catchup rate is good: $progress_per_second seconds/second${NC}"
+    echo -e "${GREEN}Estimated time to sync:${NC}"
+    s_to_human_readable $time_to_catchup
+fi