diff --git a/arc/reth/arc-mainnet-reth-archive-trace.yml b/arc/reth/arc-mainnet-reth-archive-trace.yml index 289d6841..02f08a37 100644 --- a/arc/reth/arc-mainnet-reth-archive-trace.yml +++ b/arc/reth/arc-mainnet-reth-archive-trace.yml @@ -160,6 +160,9 @@ volumes: arc-mainnet-reth-archive-trace-execution: arc-mainnet-reth-archive-trace-sockets: +x-ephemeral-volumes: + - arc-mainnet-reth-archive-trace-sockets + x-upstreams: - id: $${ID} labels: diff --git a/arc/reth/arc-mainnet-reth-pruned-trace.yml b/arc/reth/arc-mainnet-reth-pruned-trace.yml index 3961f0d4..2f35834b 100644 --- a/arc/reth/arc-mainnet-reth-pruned-trace.yml +++ b/arc/reth/arc-mainnet-reth-pruned-trace.yml @@ -161,6 +161,9 @@ volumes: arc-mainnet-reth-pruned-trace-execution: arc-mainnet-reth-pruned-trace-sockets: +x-ephemeral-volumes: + - arc-mainnet-reth-pruned-trace-sockets + x-upstreams: - id: $${ID} labels: diff --git a/arc/reth/arc-testnet-reth-archive-trace.yml b/arc/reth/arc-testnet-reth-archive-trace.yml index f1a461d0..8927bc27 100644 --- a/arc/reth/arc-testnet-reth-archive-trace.yml +++ b/arc/reth/arc-testnet-reth-archive-trace.yml @@ -160,6 +160,9 @@ volumes: arc-testnet-reth-archive-trace-execution: arc-testnet-reth-archive-trace-sockets: +x-ephemeral-volumes: + - arc-testnet-reth-archive-trace-sockets + x-upstreams: - id: $${ID} labels: diff --git a/arc/reth/arc-testnet-reth-pruned-trace.yml b/arc/reth/arc-testnet-reth-pruned-trace.yml index eb68b5c4..34f3066a 100644 --- a/arc/reth/arc-testnet-reth-pruned-trace.yml +++ b/arc/reth/arc-testnet-reth-pruned-trace.yml @@ -161,6 +161,9 @@ volumes: arc-testnet-reth-pruned-trace-execution: arc-testnet-reth-pruned-trace-sockets: +x-ephemeral-volumes: + - arc-testnet-reth-pruned-trace-sockets + x-upstreams: - id: $${ID} labels: diff --git a/arc/reth/arc-testnet-reth-pruned.yml b/arc/reth/arc-testnet-reth-pruned.yml index d05bff5f..9e24d938 100644 --- a/arc/reth/arc-testnet-reth-pruned.yml +++ b/arc/reth/arc-testnet-reth-pruned.yml @@ -161,6 +161,9 @@ volumes: arc-testnet-reth-pruned-execution: arc-testnet-reth-pruned-sockets: +x-ephemeral-volumes: + - arc-testnet-reth-pruned-sockets + x-upstreams: - id: $${ID} labels: diff --git a/clone-backup.sh b/clone-backup.sh index f2f189e7..0680de1e 100755 --- a/clone-backup.sh +++ b/clone-backup.sh @@ -298,6 +298,7 @@ cleanup_all_ports() { # Transfer backup file using screen method with better error handling transfer_backup() { local key=$1 + local compose_file=$2 local volume_name="rpc_$key" # Find the newest restorable backup file (skip near-empty artifacts) @@ -309,6 +310,13 @@ transfer_backup() { echo "Note: no backup for secrets/jwt volume $volume_name (regenerated at create-node), skipping" return 2 fi + # Check if this volume is optional or ephemeral - if so, skip gracefully + if [[ -n "$compose_file" && -f "$compose_file" ]]; then + if is_ephemeral_volume_key "$key" "$compose_file" || is_optional_volume_key "$key" "$compose_file"; then + echo "Note: no backup for optional/ephemeral volume $volume_name (will be created empty at destination), skipping" + return 2 + fi + fi echo "Warning: No restorable backup file found for $volume_name, skipping" return 1 fi @@ -502,6 +510,7 @@ LISTENER # Fallback: Direct SSH pipe transfer_backup_ssh() { local key=$1 + local compose_file=$2 local volume_name="rpc_$key" # Find the newest restorable backup file (skip near-empty artifacts) @@ -513,6 +522,13 @@ transfer_backup_ssh() { echo "Note: no backup for secrets/jwt volume $volume_name (regenerated at create-node), skipping" return 2 fi + # Check if this volume is optional or ephemeral - if so, skip gracefully + if [[ -n "$compose_file" && -f "$compose_file" ]]; then + if is_ephemeral_volume_key "$key" "$compose_file" || is_optional_volume_key "$key" "$compose_file"; then + echo "Note: no backup for optional/ephemeral volume $volume_name (will be created empty at destination), skipping" + return 2 + fi + fi echo "Warning: No restorable backup file found for $volume_name, skipping" return 1 fi @@ -642,7 +658,7 @@ main() { for key in $keys; do # Try nc method first - transfer_backup "$key" + transfer_backup "$key" "$dir/$1.yml" rc=$? if [[ $rc -eq 2 ]]; then @@ -650,7 +666,7 @@ main() { skipped_volumes="$skipped_volumes $key" elif [[ $rc -ne 0 ]]; then echo "NC transfer failed, trying direct SSH..." - transfer_backup_ssh "$key" + transfer_backup_ssh "$key" "$dir/$1.yml" rc=$? if [[ $rc -eq 0 ]]; then diff --git a/compose_registry.json b/compose_registry.json index 726d911f..542def5f 100644 --- a/compose_registry.json +++ b/compose_registry.json @@ -958,8 +958,7 @@ "type": "archive", "volumes": [ "arc-mainnet-reth-archive-trace-consensus", - "arc-mainnet-reth-archive-trace-execution", - "arc-mainnet-reth-archive-trace-sockets" + "arc-mainnet-reth-archive-trace-execution" ] }, { @@ -976,8 +975,7 @@ "type": "pruned", "volumes": [ "arc-mainnet-reth-pruned-trace-consensus", - "arc-mainnet-reth-pruned-trace-execution", - "arc-mainnet-reth-pruned-trace-sockets" + "arc-mainnet-reth-pruned-trace-execution" ] }, { @@ -994,8 +992,7 @@ "type": "archive", "volumes": [ "arc-testnet-reth-archive-trace-consensus", - "arc-testnet-reth-archive-trace-execution", - "arc-testnet-reth-archive-trace-sockets" + "arc-testnet-reth-archive-trace-execution" ] }, { @@ -1012,8 +1009,7 @@ "type": "pruned", "volumes": [ "arc-testnet-reth-pruned-trace-consensus", - "arc-testnet-reth-pruned-trace-execution", - "arc-testnet-reth-pruned-trace-sockets" + "arc-testnet-reth-pruned-trace-execution" ] }, { @@ -1028,8 +1024,7 @@ "type": "pruned", "volumes": [ "arc-testnet-reth-pruned-consensus", - "arc-testnet-reth-pruned-execution", - "arc-testnet-reth-pruned-sockets" + "arc-testnet-reth-pruned-execution" ] }, { diff --git a/ethereum/besu/ethereum-mainnet-besu-pruned-bonsai--nimbus.yml b/ethereum/besu/ethereum-mainnet-besu-pruned-bonsai--nimbus.yml index 86b7fd18..44d5dc14 100644 --- a/ethereum/besu/ethereum-mainnet-besu-pruned-bonsai--nimbus.yml +++ b/ethereum/besu/ethereum-mainnet-besu-pruned-bonsai--nimbus.yml @@ -199,4 +199,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-besu-pruned-bonsai_nimbus] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path--nimbus.yml b/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path--nimbus.yml index 455c88f6..b858ddea 100644 --- a/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path--nimbus.yml +++ b/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path--nimbus.yml @@ -205,4 +205,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-hoodi-geth-pruned-pebble-path_nimbus] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path.yml b/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path.yml index 587286c0..7efe6994 100644 --- a/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path.yml +++ b/ethereum/geth/ethereum-hoodi-geth-pruned-pebble-path.yml @@ -206,4 +206,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-hoodi-geth-pruned-pebble-path_prysm] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-mainnet-geth-minimal-pebble-path.yml b/ethereum/geth/ethereum-mainnet-geth-minimal-pebble-path.yml index ff96c11b..fba8b921 100644 --- a/ethereum/geth/ethereum-mainnet-geth-minimal-pebble-path.yml +++ b/ethereum/geth/ethereum-mainnet-geth-minimal-pebble-path.yml @@ -210,4 +210,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-geth-minimal-pebble-path_prysm] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-mainnet-geth-pruned-leveldb-hash--nimbus--fireeth.yml b/ethereum/geth/ethereum-mainnet-geth-pruned-leveldb-hash--nimbus--fireeth.yml index 0ab11fcf..3535930a 100644 --- a/ethereum/geth/ethereum-mainnet-geth-pruned-leveldb-hash--nimbus--fireeth.yml +++ b/ethereum/geth/ethereum-mainnet-geth-pruned-leveldb-hash--nimbus--fireeth.yml @@ -269,4 +269,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-geth-pruned-leveldb-hash_nimbus] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus--fireeth.yml b/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus--fireeth.yml index 60747453..a02343bc 100644 --- a/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus--fireeth.yml +++ b/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus--fireeth.yml @@ -271,4 +271,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-geth-pruned-pebble-path_nimbus] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus.yml b/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus.yml index 1e86e648..caee52c6 100644 --- a/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus.yml +++ b/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path--nimbus.yml @@ -205,4 +205,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-geth-pruned-pebble-path_nimbus] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path.yml b/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path.yml index 1c7dda21..15c9bcf0 100644 --- a/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path.yml +++ b/ethereum/geth/ethereum-mainnet-geth-pruned-pebble-path.yml @@ -206,4 +206,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-geth-pruned-pebble-path_prysm] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path--nimbus.yml b/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path--nimbus.yml index 6a89b63c..ca0d36d9 100644 --- a/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path--nimbus.yml +++ b/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path--nimbus.yml @@ -205,4 +205,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-geth-pruned-pebble-path_nimbus] ... \ No newline at end of file diff --git a/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path.yml b/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path.yml index 5cc22335..244c4d77 100644 --- a/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path.yml +++ b/ethereum/geth/ethereum-sepolia-geth-pruned-pebble-path.yml @@ -206,4 +206,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-geth-pruned-pebble-path_prysm] ... \ No newline at end of file diff --git a/ethereum/nethermind/ethereum-hoodi-nethermind-pruned-rocksdb-trace--nimbus.yml b/ethereum/nethermind/ethereum-hoodi-nethermind-pruned-rocksdb-trace--nimbus.yml index b10def66..30b478d1 100644 --- a/ethereum/nethermind/ethereum-hoodi-nethermind-pruned-rocksdb-trace--nimbus.yml +++ b/ethereum/nethermind/ethereum-hoodi-nethermind-pruned-rocksdb-trace--nimbus.yml @@ -179,4 +179,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-hoodi-nethermind-pruned-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/nethermind/ethereum-mainnet-nethermind-pruned-rocksdb-trace--nimbus.yml b/ethereum/nethermind/ethereum-mainnet-nethermind-pruned-rocksdb-trace--nimbus.yml index f14eb808..8785b10d 100644 --- a/ethereum/nethermind/ethereum-mainnet-nethermind-pruned-rocksdb-trace--nimbus.yml +++ b/ethereum/nethermind/ethereum-mainnet-nethermind-pruned-rocksdb-trace--nimbus.yml @@ -179,4 +179,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-nethermind-pruned-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/nethermind/ethereum-sepolia-nethermind-pruned-rocksdb-trace--nimbus.yml b/ethereum/nethermind/ethereum-sepolia-nethermind-pruned-rocksdb-trace--nimbus.yml index b6b54797..33cfcb2d 100644 --- a/ethereum/nethermind/ethereum-sepolia-nethermind-pruned-rocksdb-trace--nimbus.yml +++ b/ethereum/nethermind/ethereum-sepolia-nethermind-pruned-rocksdb-trace--nimbus.yml @@ -179,4 +179,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-nethermind-pruned-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-hoodi-reth-archive-trace.yml b/ethereum/reth/ethereum-hoodi-reth-archive-trace.yml index 5339ade6..ee3c8f5f 100644 --- a/ethereum/reth/ethereum-hoodi-reth-archive-trace.yml +++ b/ethereum/reth/ethereum-hoodi-reth-archive-trace.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-hoodi-reth-archive-trace_prysm] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-hoodi-reth-pruned-trace.yml b/ethereum/reth/ethereum-hoodi-reth-pruned-trace.yml index 18462eb9..ee247063 100644 --- a/ethereum/reth/ethereum-hoodi-reth-pruned-trace.yml +++ b/ethereum/reth/ethereum-hoodi-reth-pruned-trace.yml @@ -195,4 +195,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-hoodi-reth-pruned-trace_prysm] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-archive-trace--lighthouse-pruned-blobs.yml b/ethereum/reth/ethereum-mainnet-reth-archive-trace--lighthouse-pruned-blobs.yml index e3449c15..1ef3a562 100644 --- a/ethereum/reth/ethereum-mainnet-reth-archive-trace--lighthouse-pruned-blobs.yml +++ b/ethereum/reth/ethereum-mainnet-reth-archive-trace--lighthouse-pruned-blobs.yml @@ -201,4 +201,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-archive-trace_lighthouse-pruned-blobs] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-archive-trace--nimbus.yml b/ethereum/reth/ethereum-mainnet-reth-archive-trace--nimbus.yml index 7b2fcf27..538d0fd5 100644 --- a/ethereum/reth/ethereum-mainnet-reth-archive-trace--nimbus.yml +++ b/ethereum/reth/ethereum-mainnet-reth-archive-trace--nimbus.yml @@ -193,4 +193,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-archive-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-archive-trace.yml b/ethereum/reth/ethereum-mainnet-reth-archive-trace.yml index 20649425..2c9274f2 100644 --- a/ethereum/reth/ethereum-mainnet-reth-archive-trace.yml +++ b/ethereum/reth/ethereum-mainnet-reth-archive-trace.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-archive-trace_prysm] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml b/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml index dbea22b1..2366d790 100644 --- a/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml +++ b/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml @@ -252,4 +252,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-minimal-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-pruned-trace--lighthouse.yml b/ethereum/reth/ethereum-mainnet-reth-pruned-trace--lighthouse.yml index a1be9adf..9b9be2b8 100644 --- a/ethereum/reth/ethereum-mainnet-reth-pruned-trace--lighthouse.yml +++ b/ethereum/reth/ethereum-mainnet-reth-pruned-trace--lighthouse.yml @@ -200,4 +200,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-pruned-trace_lighthouse] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-pruned-trace--nimbus.yml b/ethereum/reth/ethereum-mainnet-reth-pruned-trace--nimbus.yml index 4ceb6225..f66ce710 100644 --- a/ethereum/reth/ethereum-mainnet-reth-pruned-trace--nimbus.yml +++ b/ethereum/reth/ethereum-mainnet-reth-pruned-trace--nimbus.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-pruned-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-mainnet-reth-pruned-trace.yml b/ethereum/reth/ethereum-mainnet-reth-pruned-trace.yml index 451f175c..09fd0549 100644 --- a/ethereum/reth/ethereum-mainnet-reth-pruned-trace.yml +++ b/ethereum/reth/ethereum-mainnet-reth-pruned-trace.yml @@ -195,4 +195,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-mainnet-reth-pruned-trace_prysm] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-sepolia-reth-archive-trace--nimbus.yml b/ethereum/reth/ethereum-sepolia-reth-archive-trace--nimbus.yml index 75ba78ce..3a2424b5 100644 --- a/ethereum/reth/ethereum-sepolia-reth-archive-trace--nimbus.yml +++ b/ethereum/reth/ethereum-sepolia-reth-archive-trace--nimbus.yml @@ -193,4 +193,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-reth-archive-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-sepolia-reth-archive-trace.yml b/ethereum/reth/ethereum-sepolia-reth-archive-trace.yml index 1538c896..d9dd9d5a 100644 --- a/ethereum/reth/ethereum-sepolia-reth-archive-trace.yml +++ b/ethereum/reth/ethereum-sepolia-reth-archive-trace.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-reth-archive-trace_prysm] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-sepolia-reth-pruned-trace--nimbus.yml b/ethereum/reth/ethereum-sepolia-reth-pruned-trace--nimbus.yml index e662a813..eb106bf7 100644 --- a/ethereum/reth/ethereum-sepolia-reth-pruned-trace--nimbus.yml +++ b/ethereum/reth/ethereum-sepolia-reth-pruned-trace--nimbus.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-reth-pruned-trace_nimbus] ... \ No newline at end of file diff --git a/ethereum/reth/ethereum-sepolia-reth-pruned-trace.yml b/ethereum/reth/ethereum-sepolia-reth-pruned-trace.yml index 982de7b9..ecd7d48c 100644 --- a/ethereum/reth/ethereum-sepolia-reth-pruned-trace.yml +++ b/ethereum/reth/ethereum-sepolia-reth-pruned-trace.yml @@ -195,4 +195,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [ethereum-sepolia-reth-pruned-trace_prysm] ... \ No newline at end of file diff --git a/gnosis/nethermind/gnosis-chiado-nethermind-archive-rocksdb-trace.yml b/gnosis/nethermind/gnosis-chiado-nethermind-archive-rocksdb-trace.yml index 40cee643..f512005c 100644 --- a/gnosis/nethermind/gnosis-chiado-nethermind-archive-rocksdb-trace.yml +++ b/gnosis/nethermind/gnosis-chiado-nethermind-archive-rocksdb-trace.yml @@ -181,4 +181,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-chiado-nethermind-archive-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/nethermind/gnosis-chiado-nethermind-minimal-rocksdb-trace.yml b/gnosis/nethermind/gnosis-chiado-nethermind-minimal-rocksdb-trace.yml index 3e05a597..f8b2a7dd 100644 --- a/gnosis/nethermind/gnosis-chiado-nethermind-minimal-rocksdb-trace.yml +++ b/gnosis/nethermind/gnosis-chiado-nethermind-minimal-rocksdb-trace.yml @@ -184,4 +184,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-chiado-nethermind-minimal-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/nethermind/gnosis-chiado-nethermind-pruned-rocksdb-trace.yml b/gnosis/nethermind/gnosis-chiado-nethermind-pruned-rocksdb-trace.yml index dac483d6..6e7400ff 100644 --- a/gnosis/nethermind/gnosis-chiado-nethermind-pruned-rocksdb-trace.yml +++ b/gnosis/nethermind/gnosis-chiado-nethermind-pruned-rocksdb-trace.yml @@ -179,4 +179,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-chiado-nethermind-pruned-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/nethermind/gnosis-mainnet-nethermind-archive-rocksdb-trace.yml b/gnosis/nethermind/gnosis-mainnet-nethermind-archive-rocksdb-trace.yml index 8c2a3a00..ef3bd619 100644 --- a/gnosis/nethermind/gnosis-mainnet-nethermind-archive-rocksdb-trace.yml +++ b/gnosis/nethermind/gnosis-mainnet-nethermind-archive-rocksdb-trace.yml @@ -181,4 +181,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-mainnet-nethermind-archive-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/nethermind/gnosis-mainnet-nethermind-minimal-rocksdb-trace.yml b/gnosis/nethermind/gnosis-mainnet-nethermind-minimal-rocksdb-trace.yml index 4cfa1f0e..fb1278d1 100644 --- a/gnosis/nethermind/gnosis-mainnet-nethermind-minimal-rocksdb-trace.yml +++ b/gnosis/nethermind/gnosis-mainnet-nethermind-minimal-rocksdb-trace.yml @@ -184,4 +184,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-mainnet-nethermind-minimal-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/nethermind/gnosis-mainnet-nethermind-pruned-rocksdb-trace.yml b/gnosis/nethermind/gnosis-mainnet-nethermind-pruned-rocksdb-trace.yml index 5468e272..d00b1e3c 100644 --- a/gnosis/nethermind/gnosis-mainnet-nethermind-pruned-rocksdb-trace.yml +++ b/gnosis/nethermind/gnosis-mainnet-nethermind-pruned-rocksdb-trace.yml @@ -179,4 +179,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-mainnet-nethermind-pruned-rocksdb-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/reth/gnosis-chiado-reth-archive-trace.yml b/gnosis/reth/gnosis-chiado-reth-archive-trace.yml index 841b2c7f..3f559733 100644 --- a/gnosis/reth/gnosis-chiado-reth-archive-trace.yml +++ b/gnosis/reth/gnosis-chiado-reth-archive-trace.yml @@ -193,4 +193,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-chiado-reth-archive-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/reth/gnosis-chiado-reth-pruned-trace.yml b/gnosis/reth/gnosis-chiado-reth-pruned-trace.yml index 0d4a9ecb..5e1cc7c1 100644 --- a/gnosis/reth/gnosis-chiado-reth-pruned-trace.yml +++ b/gnosis/reth/gnosis-chiado-reth-pruned-trace.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-chiado-reth-pruned-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/reth/gnosis-mainnet-reth-archive-trace.yml b/gnosis/reth/gnosis-mainnet-reth-archive-trace.yml index cacbd5f5..cb96fee7 100644 --- a/gnosis/reth/gnosis-mainnet-reth-archive-trace.yml +++ b/gnosis/reth/gnosis-mainnet-reth-archive-trace.yml @@ -193,4 +193,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-mainnet-reth-archive-trace_nimbus] ... \ No newline at end of file diff --git a/gnosis/reth/gnosis-mainnet-reth-pruned-trace.yml b/gnosis/reth/gnosis-mainnet-reth-pruned-trace.yml index 6ef71aaf..54355e28 100644 --- a/gnosis/reth/gnosis-mainnet-reth-pruned-trace.yml +++ b/gnosis/reth/gnosis-mainnet-reth-pruned-trace.yml @@ -194,4 +194,5 @@ x-upstreams: generic: rpc: url: $${RPC_URL} +x-optional-volumes: [gnosis-mainnet-reth-pruned-trace_nimbus] ... \ No newline at end of file diff --git a/list-restorable.sh b/list-restorable.sh index aff058da..60d8e763 100755 --- a/list-restorable.sh +++ b/list-restorable.sh @@ -13,6 +13,20 @@ dir="$(dirname "$0")" source "$dir/volume-utils.sh" + +# Get required persistent volume keys (all volumes minus ephemeral minus optional) +get_required_volume_keys() { + local compose_file=$1 + local key + + while IFS= read -r key; do + [[ -z "$key" ]] && continue + # Skip if ephemeral or optional + if ! is_ephemeral_volume_key "$key" "$compose_file" && ! is_optional_volume_key "$key" "$compose_file"; then + echo "$key" + fi + done < <(get_volume_keys "$compose_file") +} registry_file="${dir}/compose_registry.json" backup_dir="/backup" @@ -115,12 +129,14 @@ while IFS= read -r entry; do continue fi - # Check each volume for backup file + # Check each required volume for backup file all_backups_exist=true missing_volumes=() backup_info=() - while IFS= read -r volume; do + # Get required persistent volumes (excluding ephemeral and optional) + # Use process substitution to avoid subshell issues + while IFS= read -r volume < <(get_required_volume_keys "$compose_path"); do volume_name="rpc_${volume}" # jwt/secrets volumes are regenerated at create-node — never backed up by # design, so they must not gate restorability (kept plasma marked diff --git a/restore-volumes.sh b/restore-volumes.sh index 739d862c..6a92b49f 100755 --- a/restore-volumes.sh +++ b/restore-volumes.sh @@ -168,6 +168,13 @@ while IFS= read -r key; do fi if [ -z "$newest_file" ]; then + # Check if this volume is optional or ephemeral - if so, warn and continue + if is_ephemeral_volume_key "$key" "$dir/$1.yml" || is_optional_volume_key "$key" "$dir/$1.yml"; then + echo "WARN: No backup found for volume 'rpc_$key' (volume is optional/ephemeral) - creating empty volume" + # Create empty volume and continue + mkdir -p "$data_dir" + continue + fi echo "Error: No backup found for volume 'rpc_$key'" exit 1 fi diff --git a/volume-utils.sh b/volume-utils.sh index 6a99fc9b..fa6e1fd8 100755 --- a/volume-utils.sh +++ b/volume-utils.sh @@ -218,6 +218,22 @@ is_ephemeral_volume_key() { return 1 } +is_optional_volume_key() { + local key=$1 + local compose_file=$2 + + if [[ -n "$compose_file" && -f "$compose_file" ]]; then + local optional + optional=$(yaml2json < "$compose_file" 2>/dev/null | jq -r '.["x-optional-volumes"] // [] | .[]' 2>/dev/null) + while IFS= read -r vol; do + [[ -z "$vol" ]] && continue + [[ "$key" == "$vol" ]] && return 0 + done <<< "$optional" + fi + + return 1 +} + get_volume_keys() { local compose_file=$1 yaml2json < "$compose_file" 2>/dev/null | jq -r '.volumes | keys[]' 2>/dev/null