Add optional/ephemeral volume notation for restore resilience

Introduce x-optional-volumes for beacon sidecars (prysm, nimbus, lighthouse,
teku, lodestar) that are checkpoint-sync capable and rebuild state quickly.
Mark reth sockets volumes as ephemeral (runtime IPC artifacts).

Script changes:
- volume-utils.sh: add is_optional_volume_key() function
- restore-volumes.sh: tolerate missing optional/ephemeral volume archives
- clone-backup.sh: skip missing optional/ephemeral volumes gracefully
- list-restorable.sh: only require archives for non-optional persistent volumes

This fixes the de-25 ethereum prysm incident (entire 1.1T restore failed
due to missing prysm archive) and arc-testnet reth sockets restore loops.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
2026-08-04 06:10:55 +00:00
parent d21f2669ae
commit d896378eb7
46 changed files with 115 additions and 14 deletions

View File

@@ -298,6 +298,7 @@ cleanup_all_ports() {
# Transfer backup file using screen method with better error handling
transfer_backup() {
local key=$1
local compose_file=$2
local volume_name="rpc_$key"
# Find the newest restorable backup file (skip near-empty artifacts)
@@ -309,6 +310,13 @@ transfer_backup() {
echo "Note: no backup for secrets/jwt volume $volume_name (regenerated at create-node), skipping"
return 2
fi
# Check if this volume is optional or ephemeral - if so, skip gracefully
if [[ -n "$compose_file" && -f "$compose_file" ]]; then
if is_ephemeral_volume_key "$key" "$compose_file" || is_optional_volume_key "$key" "$compose_file"; then
echo "Note: no backup for optional/ephemeral volume $volume_name (will be created empty at destination), skipping"
return 2
fi
fi
echo "Warning: No restorable backup file found for $volume_name, skipping"
return 1
fi
@@ -502,6 +510,7 @@ LISTENER
# Fallback: Direct SSH pipe
transfer_backup_ssh() {
local key=$1
local compose_file=$2
local volume_name="rpc_$key"
# Find the newest restorable backup file (skip near-empty artifacts)
@@ -513,6 +522,13 @@ transfer_backup_ssh() {
echo "Note: no backup for secrets/jwt volume $volume_name (regenerated at create-node), skipping"
return 2
fi
# Check if this volume is optional or ephemeral - if so, skip gracefully
if [[ -n "$compose_file" && -f "$compose_file" ]]; then
if is_ephemeral_volume_key "$key" "$compose_file" || is_optional_volume_key "$key" "$compose_file"; then
echo "Note: no backup for optional/ephemeral volume $volume_name (will be created empty at destination), skipping"
return 2
fi
fi
echo "Warning: No restorable backup file found for $volume_name, skipping"
return 1
fi
@@ -642,7 +658,7 @@ main() {
for key in $keys; do
# Try nc method first
transfer_backup "$key"
transfer_backup "$key" "$dir/$1.yml"
rc=$?
if [[ $rc -eq 2 ]]; then
@@ -650,7 +666,7 @@ main() {
skipped_volumes="$skipped_volumes $key"
elif [[ $rc -ne 0 ]]; then
echo "NC transfer failed, trying direct SSH..."
transfer_backup_ssh "$key"
transfer_backup_ssh "$key" "$dir/$1.yml"
rc=$?
if [[ $rc -eq 0 ]]; then