Add optional/ephemeral volume notation for restore resilience
Introduce x-optional-volumes for beacon sidecars (prysm, nimbus, lighthouse, teku, lodestar) that are checkpoint-sync capable and rebuild state quickly. Mark reth sockets volumes as ephemeral (runtime IPC artifacts). Script changes: - volume-utils.sh: add is_optional_volume_key() function - restore-volumes.sh: tolerate missing optional/ephemeral volume archives - clone-backup.sh: skip missing optional/ephemeral volumes gracefully - list-restorable.sh: only require archives for non-optional persistent volumes This fixes the de-25 ethereum prysm incident (entire 1.1T restore failed due to missing prysm archive) and arc-testnet reth sockets restore loops. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -160,6 +160,9 @@ volumes:
|
||||
arc-mainnet-reth-archive-trace-execution:
|
||||
arc-mainnet-reth-archive-trace-sockets:
|
||||
|
||||
x-ephemeral-volumes:
|
||||
- arc-mainnet-reth-archive-trace-sockets
|
||||
|
||||
x-upstreams:
|
||||
- id: $${ID}
|
||||
labels:
|
||||
|
||||
@@ -161,6 +161,9 @@ volumes:
|
||||
arc-mainnet-reth-pruned-trace-execution:
|
||||
arc-mainnet-reth-pruned-trace-sockets:
|
||||
|
||||
x-ephemeral-volumes:
|
||||
- arc-mainnet-reth-pruned-trace-sockets
|
||||
|
||||
x-upstreams:
|
||||
- id: $${ID}
|
||||
labels:
|
||||
|
||||
@@ -160,6 +160,9 @@ volumes:
|
||||
arc-testnet-reth-archive-trace-execution:
|
||||
arc-testnet-reth-archive-trace-sockets:
|
||||
|
||||
x-ephemeral-volumes:
|
||||
- arc-testnet-reth-archive-trace-sockets
|
||||
|
||||
x-upstreams:
|
||||
- id: $${ID}
|
||||
labels:
|
||||
|
||||
@@ -161,6 +161,9 @@ volumes:
|
||||
arc-testnet-reth-pruned-trace-execution:
|
||||
arc-testnet-reth-pruned-trace-sockets:
|
||||
|
||||
x-ephemeral-volumes:
|
||||
- arc-testnet-reth-pruned-trace-sockets
|
||||
|
||||
x-upstreams:
|
||||
- id: $${ID}
|
||||
labels:
|
||||
|
||||
@@ -161,6 +161,9 @@ volumes:
|
||||
arc-testnet-reth-pruned-execution:
|
||||
arc-testnet-reth-pruned-sockets:
|
||||
|
||||
x-ephemeral-volumes:
|
||||
- arc-testnet-reth-pruned-sockets
|
||||
|
||||
x-upstreams:
|
||||
- id: $${ID}
|
||||
labels:
|
||||
|
||||
@@ -298,6 +298,7 @@ cleanup_all_ports() {
|
||||
# Transfer backup file using screen method with better error handling
|
||||
transfer_backup() {
|
||||
local key=$1
|
||||
local compose_file=$2
|
||||
local volume_name="rpc_$key"
|
||||
|
||||
# Find the newest restorable backup file (skip near-empty artifacts)
|
||||
@@ -309,6 +310,13 @@ transfer_backup() {
|
||||
echo "Note: no backup for secrets/jwt volume $volume_name (regenerated at create-node), skipping"
|
||||
return 2
|
||||
fi
|
||||
# Check if this volume is optional or ephemeral - if so, skip gracefully
|
||||
if [[ -n "$compose_file" && -f "$compose_file" ]]; then
|
||||
if is_ephemeral_volume_key "$key" "$compose_file" || is_optional_volume_key "$key" "$compose_file"; then
|
||||
echo "Note: no backup for optional/ephemeral volume $volume_name (will be created empty at destination), skipping"
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
echo "Warning: No restorable backup file found for $volume_name, skipping"
|
||||
return 1
|
||||
fi
|
||||
@@ -502,6 +510,7 @@ LISTENER
|
||||
# Fallback: Direct SSH pipe
|
||||
transfer_backup_ssh() {
|
||||
local key=$1
|
||||
local compose_file=$2
|
||||
local volume_name="rpc_$key"
|
||||
|
||||
# Find the newest restorable backup file (skip near-empty artifacts)
|
||||
@@ -513,6 +522,13 @@ transfer_backup_ssh() {
|
||||
echo "Note: no backup for secrets/jwt volume $volume_name (regenerated at create-node), skipping"
|
||||
return 2
|
||||
fi
|
||||
# Check if this volume is optional or ephemeral - if so, skip gracefully
|
||||
if [[ -n "$compose_file" && -f "$compose_file" ]]; then
|
||||
if is_ephemeral_volume_key "$key" "$compose_file" || is_optional_volume_key "$key" "$compose_file"; then
|
||||
echo "Note: no backup for optional/ephemeral volume $volume_name (will be created empty at destination), skipping"
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
echo "Warning: No restorable backup file found for $volume_name, skipping"
|
||||
return 1
|
||||
fi
|
||||
@@ -642,7 +658,7 @@ main() {
|
||||
|
||||
for key in $keys; do
|
||||
# Try nc method first
|
||||
transfer_backup "$key"
|
||||
transfer_backup "$key" "$dir/$1.yml"
|
||||
rc=$?
|
||||
|
||||
if [[ $rc -eq 2 ]]; then
|
||||
@@ -650,7 +666,7 @@ main() {
|
||||
skipped_volumes="$skipped_volumes $key"
|
||||
elif [[ $rc -ne 0 ]]; then
|
||||
echo "NC transfer failed, trying direct SSH..."
|
||||
transfer_backup_ssh "$key"
|
||||
transfer_backup_ssh "$key" "$dir/$1.yml"
|
||||
rc=$?
|
||||
|
||||
if [[ $rc -eq 0 ]]; then
|
||||
|
||||
@@ -958,8 +958,7 @@
|
||||
"type": "archive",
|
||||
"volumes": [
|
||||
"arc-mainnet-reth-archive-trace-consensus",
|
||||
"arc-mainnet-reth-archive-trace-execution",
|
||||
"arc-mainnet-reth-archive-trace-sockets"
|
||||
"arc-mainnet-reth-archive-trace-execution"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -976,8 +975,7 @@
|
||||
"type": "pruned",
|
||||
"volumes": [
|
||||
"arc-mainnet-reth-pruned-trace-consensus",
|
||||
"arc-mainnet-reth-pruned-trace-execution",
|
||||
"arc-mainnet-reth-pruned-trace-sockets"
|
||||
"arc-mainnet-reth-pruned-trace-execution"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -994,8 +992,7 @@
|
||||
"type": "archive",
|
||||
"volumes": [
|
||||
"arc-testnet-reth-archive-trace-consensus",
|
||||
"arc-testnet-reth-archive-trace-execution",
|
||||
"arc-testnet-reth-archive-trace-sockets"
|
||||
"arc-testnet-reth-archive-trace-execution"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1012,8 +1009,7 @@
|
||||
"type": "pruned",
|
||||
"volumes": [
|
||||
"arc-testnet-reth-pruned-trace-consensus",
|
||||
"arc-testnet-reth-pruned-trace-execution",
|
||||
"arc-testnet-reth-pruned-trace-sockets"
|
||||
"arc-testnet-reth-pruned-trace-execution"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1028,8 +1024,7 @@
|
||||
"type": "pruned",
|
||||
"volumes": [
|
||||
"arc-testnet-reth-pruned-consensus",
|
||||
"arc-testnet-reth-pruned-execution",
|
||||
"arc-testnet-reth-pruned-sockets"
|
||||
"arc-testnet-reth-pruned-execution"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -199,4 +199,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-besu-pruned-bonsai_nimbus]
|
||||
...
|
||||
@@ -205,4 +205,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-hoodi-geth-pruned-pebble-path_nimbus]
|
||||
...
|
||||
@@ -206,4 +206,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-hoodi-geth-pruned-pebble-path_prysm]
|
||||
...
|
||||
@@ -210,4 +210,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-geth-minimal-pebble-path_prysm]
|
||||
...
|
||||
@@ -269,4 +269,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-geth-pruned-leveldb-hash_nimbus]
|
||||
...
|
||||
@@ -271,4 +271,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-geth-pruned-pebble-path_nimbus]
|
||||
...
|
||||
@@ -205,4 +205,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-geth-pruned-pebble-path_nimbus]
|
||||
...
|
||||
@@ -206,4 +206,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-geth-pruned-pebble-path_prysm]
|
||||
...
|
||||
@@ -205,4 +205,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-geth-pruned-pebble-path_nimbus]
|
||||
...
|
||||
@@ -206,4 +206,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-geth-pruned-pebble-path_prysm]
|
||||
...
|
||||
@@ -179,4 +179,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-hoodi-nethermind-pruned-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -179,4 +179,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-nethermind-pruned-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -179,4 +179,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-nethermind-pruned-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-hoodi-reth-archive-trace_prysm]
|
||||
...
|
||||
@@ -195,4 +195,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-hoodi-reth-pruned-trace_prysm]
|
||||
...
|
||||
@@ -201,4 +201,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-archive-trace_lighthouse-pruned-blobs]
|
||||
...
|
||||
@@ -193,4 +193,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-archive-trace_nimbus]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-archive-trace_prysm]
|
||||
...
|
||||
@@ -252,4 +252,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-minimal-trace_nimbus]
|
||||
...
|
||||
@@ -200,4 +200,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-pruned-trace_lighthouse]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-pruned-trace_nimbus]
|
||||
...
|
||||
@@ -195,4 +195,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-mainnet-reth-pruned-trace_prysm]
|
||||
...
|
||||
@@ -193,4 +193,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-reth-archive-trace_nimbus]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-reth-archive-trace_prysm]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-reth-pruned-trace_nimbus]
|
||||
...
|
||||
@@ -195,4 +195,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [ethereum-sepolia-reth-pruned-trace_prysm]
|
||||
...
|
||||
@@ -181,4 +181,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-chiado-nethermind-archive-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -184,4 +184,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-chiado-nethermind-minimal-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -179,4 +179,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-chiado-nethermind-pruned-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -181,4 +181,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-mainnet-nethermind-archive-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -184,4 +184,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-mainnet-nethermind-minimal-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -179,4 +179,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-mainnet-nethermind-pruned-rocksdb-trace_nimbus]
|
||||
...
|
||||
@@ -193,4 +193,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-chiado-reth-archive-trace_nimbus]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-chiado-reth-pruned-trace_nimbus]
|
||||
...
|
||||
@@ -193,4 +193,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-mainnet-reth-archive-trace_nimbus]
|
||||
...
|
||||
@@ -194,4 +194,5 @@ x-upstreams:
|
||||
generic:
|
||||
rpc:
|
||||
url: $${RPC_URL}
|
||||
x-optional-volumes: [gnosis-mainnet-reth-pruned-trace_nimbus]
|
||||
...
|
||||
@@ -13,6 +13,20 @@
|
||||
|
||||
dir="$(dirname "$0")"
|
||||
source "$dir/volume-utils.sh"
|
||||
|
||||
# Get required persistent volume keys (all volumes minus ephemeral minus optional)
|
||||
get_required_volume_keys() {
|
||||
local compose_file=$1
|
||||
local key
|
||||
|
||||
while IFS= read -r key; do
|
||||
[[ -z "$key" ]] && continue
|
||||
# Skip if ephemeral or optional
|
||||
if ! is_ephemeral_volume_key "$key" "$compose_file" && ! is_optional_volume_key "$key" "$compose_file"; then
|
||||
echo "$key"
|
||||
fi
|
||||
done < <(get_volume_keys "$compose_file")
|
||||
}
|
||||
registry_file="${dir}/compose_registry.json"
|
||||
backup_dir="/backup"
|
||||
|
||||
@@ -115,12 +129,14 @@ while IFS= read -r entry; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check each volume for backup file
|
||||
# Check each required volume for backup file
|
||||
all_backups_exist=true
|
||||
missing_volumes=()
|
||||
backup_info=()
|
||||
|
||||
while IFS= read -r volume; do
|
||||
# Get required persistent volumes (excluding ephemeral and optional)
|
||||
# Use process substitution to avoid subshell issues
|
||||
while IFS= read -r volume < <(get_required_volume_keys "$compose_path"); do
|
||||
volume_name="rpc_${volume}"
|
||||
# jwt/secrets volumes are regenerated at create-node — never backed up by
|
||||
# design, so they must not gate restorability (kept plasma marked
|
||||
|
||||
@@ -168,6 +168,13 @@ while IFS= read -r key; do
|
||||
fi
|
||||
|
||||
if [ -z "$newest_file" ]; then
|
||||
# Check if this volume is optional or ephemeral - if so, warn and continue
|
||||
if is_ephemeral_volume_key "$key" "$dir/$1.yml" || is_optional_volume_key "$key" "$dir/$1.yml"; then
|
||||
echo "WARN: No backup found for volume 'rpc_$key' (volume is optional/ephemeral) - creating empty volume"
|
||||
# Create empty volume and continue
|
||||
mkdir -p "$data_dir"
|
||||
continue
|
||||
fi
|
||||
echo "Error: No backup found for volume 'rpc_$key'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -218,6 +218,22 @@ is_ephemeral_volume_key() {
|
||||
return 1
|
||||
}
|
||||
|
||||
is_optional_volume_key() {
|
||||
local key=$1
|
||||
local compose_file=$2
|
||||
|
||||
if [[ -n "$compose_file" && -f "$compose_file" ]]; then
|
||||
local optional
|
||||
optional=$(yaml2json < "$compose_file" 2>/dev/null | jq -r '.["x-optional-volumes"] // [] | .[]' 2>/dev/null)
|
||||
while IFS= read -r vol; do
|
||||
[[ -z "$vol" ]] && continue
|
||||
[[ "$key" == "$vol" ]] && return 0
|
||||
done <<< "$optional"
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
get_volume_keys() {
|
||||
local compose_file=$1
|
||||
yaml2json < "$compose_file" 2>/dev/null | jq -r '.volumes | keys[]' 2>/dev/null
|
||||
|
||||
Reference in New Issue
Block a user