Compare commits
5 Commits
zircuit-co
...
cf91b6f14d
| Author | SHA1 | Date | |
|---|---|---|---|
| cf91b6f14d | |||
| 529a91f0d7 | |||
| 502a75b2fd | |||
| 80b0e4253e | |||
| c5f9e5000f |
@@ -49,12 +49,12 @@ port = 8080
|
|||||||
peer-chain-height-polling-interval = "5s"
|
peer-chain-height-polling-interval = "5s"
|
||||||
el-sync-status-refresh-interval = "5s"
|
el-sync-status-refresh-interval = "5s"
|
||||||
sync-target-selection = "Highest"
|
sync-target-selection = "Highest"
|
||||||
desync-tolerance = 100000
|
desync-tolerance = 64
|
||||||
|
|
||||||
[syncing.download]
|
[syncing.download]
|
||||||
block-range-request-timeout = "10s"
|
block-range-request-timeout = "10s"
|
||||||
blocks-batch-size = 10
|
blocks-batch-size = 5
|
||||||
blocks-parallelism = 10
|
blocks-parallelism = 2
|
||||||
max-retries = 5
|
max-retries = 5
|
||||||
backoff-delay = "1s"
|
backoff-delay = "1s"
|
||||||
use-unconditional-random-download-peer = false
|
use-unconditional-random-download-peer = false
|
||||||
@@ -22,26 +22,25 @@ check_sync_status() {
|
|||||||
# Cap the whole per-node branch (belt-and-suspenders over check-health's own cap), so no single
|
# Cap the whole per-node branch (belt-and-suspenders over check-health's own cap), so no single
|
||||||
# node can ever block the 'wait' below — that is what wedged the fleet rpc-update for hours.
|
# node can ever block the 'wait' below — that is what wedged the fleet rpc-update for hours.
|
||||||
result=$(timeout "${SYNC_TIMEOUT:-60}" "$BASEPATH/sync-status.sh" "${part%.yml}")
|
result=$(timeout "${SYNC_TIMEOUT:-60}" "$BASEPATH/sync-status.sh" "${part%.yml}")
|
||||||
|
# Capture the status IMMEDIATELY. Any command in between - including a plain
|
||||||
|
# assignment like `code=0` - overwrites $? with its own (always 0) status.
|
||||||
|
rc=$?
|
||||||
|
|
||||||
code=0
|
code=0
|
||||||
|
if [ "$rc" -ne 0 ]; then
|
||||||
if [ $? -ne 0 ]; then
|
if [[ "$result" == *"syncing"* ]] || [[ "$result" == *"lagging"* ]]; then
|
||||||
if [[ "$result" == *"syncing"* ]]; then
|
# sync-status exits 1 for syncing/lagging; those are expected states,
|
||||||
# Allow exit status 1 if result contains "syncing"
|
# not failures.
|
||||||
code=0
|
|
||||||
elif [[ "$result" == *"lagging"* ]]; then
|
|
||||||
# Allow exit status 1 if result contains "lagging"
|
|
||||||
code=0
|
code=0
|
||||||
else
|
else
|
||||||
any_failure=true
|
|
||||||
code=1
|
code=1
|
||||||
fi
|
fi
|
||||||
else
|
|
||||||
code=1
|
|
||||||
any_failure=true
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "${part%.yml}: $result"
|
echo "${part%.yml}: $result"
|
||||||
|
# NOTE: do NOT set any_failure here. This function runs backgrounded (`&`), so
|
||||||
|
# it executes in a subshell and any variable it sets is discarded. Failure is
|
||||||
|
# propagated to the parent through this return code, collected by `wait` below.
|
||||||
return "$code"
|
return "$code"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,9 +73,12 @@ for part in "${parts[@]}"; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# Wait for all background processes to finish
|
# Wait for all background processes to finish. `wait` runs in the PARENT shell, so
|
||||||
|
# this is where a failing node can actually flip any_failure - the checker itself
|
||||||
|
# cannot, being a subshell. Previously the status was discarded here, which silently
|
||||||
|
# neutered the exit code.
|
||||||
for pid in "${pids[@]}"; do
|
for pid in "${pids[@]}"; do
|
||||||
wait "$pid"
|
wait "$pid" || any_failure=true
|
||||||
done
|
done
|
||||||
|
|
||||||
# Fenced nodes (fleet-state maintenance windows) are dropped from COMPOSE_FILE
|
# Fenced nodes (fleet-state maintenance windows) are dropped from COMPOSE_FILE
|
||||||
|
|||||||
Reference in New Issue
Block a user