StakeSquid/ethereum-rpc-docker
1
0
Fork 0
Code Issues Pull Requests 3 Packages Projects Releases Wiki Activity

backup: harden lifecycle fencing and skip near-empty archives

Browse Source 
backup-node.sh now stops services, fences COMPOSE_FILE during backup, and
restores via EXIT trap; consumers skip <1MB .tar.zst artifacts so purged-
volume junk cannot shadow real backups.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Claude Agent
2026-07-03 04:38:00 +00:00
parent 6c0df2bce5
commit 58adc446e2
6 changed files with 279 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
backup-node.sh
View File

@@ -1,10 +1,73 @@
#!/bin/bash
#
# Backup persistent compose volumes to /backup or WebDAV.
#
# Usage:
# ./backup-node.sh [--hot] [--force] <compose-name> [webdav-url]
#
# Lifecycle (default, omitted with --hot):
# stop services -> fence compose out of COMPOSE_FILE -> tar+zstd -> unfence+start (EXIT trap).
# Hot backups skip stop/fence; leveldb/pebble live tars are NOT restore-trustworthy.
#
# --force Allow backing up an existing but very small datadir (<1GB reported size).
# Does NOT bypass a missing volume path (always fatal).
BASEPATH="$(dirname "$0")"
source "$BASEPATH/volume-utils.sh"
backup_dir="/backup"
remote_target="$2"
HOT_BACKUP=false
FORCE_SMALL=false
_pos=()
for _a in "$@"; do
case "$_a" in
--hot) HOT_BACKUP=true ;;
--force) FORCE_SMALL=true ;;
*) _pos+=("$_a") ;;
esac
done
set -- "${_pos[@]}"
if [ -z "${1:-}" ] || [ ! -f "/root/rpc/$1.yml" ]; then
echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist."
exit 1
fi
compose_name="$1"
remote_target="${2:-}"
lifecycle_active=false
fenced=false
stopped=false
backup_lifecycle_cleanup() {
local rc=$?
if [[ "$lifecycle_active" != true ]]; then
exit "$rc"
fi
if [[ "$fenced" == true ]]; then
unfence_compose_in_env "$compose_name" || true
fenced=false
fi
if [[ "$stopped" == true ]]; then
"$BASEPATH/start.sh" "$compose_name" || true
stopped=false
fi
exit "$rc"
}
if [[ "$HOT_BACKUP" == true ]]; then
echo "WARNING: --hot backup skips stop/fence; live leveldb/pebble archives may not restore cleanly"
else
trap backup_lifecycle_cleanup EXIT
lifecycle_active=true
echo "Stopping services for $compose_name before backup..."
"$BASEPATH/stop.sh" "$compose_name"
stopped=true
if fence_compose_in_env "$compose_name"; then
fenced=true
fi
fi
if [[ -n "$remote_target" ]] && is_local_backup_url "$remote_target"; then
echo "Target URL points to this server, using local /backup instead of $remote_target"
remote_target=""
@@ -62,13 +125,19 @@ generate_volume_metadata() {
}
# Read the JSON input and extract the list of keys
keys=$(get_persistent_volume_keys "/root/rpc/$1.yml")
keys=$(get_persistent_volume_keys "/root/rpc/$compose_name.yml")
# Iterate over the list of keys
for key in $keys; do
echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data"
source_folder="/var/lib/docker/volumes/rpc_$key/_data"
if [[ ! -d "$source_folder" ]]; then
echo "FATAL: volume data directory missing, refusing backup: $source_folder" >&2
exit 1
fi
folder_size=$(du -shL "$source_folder" | awk '{
size = $1
sub(/[Kk]$/, "", size) # Remove 'K' suffix if present
@@ -86,6 +155,11 @@ for key in $keys; do
}')
folder_size_gb=$(printf "%.0f" "$folder_size")
if (( folder_size_gb < 1 )) && [[ "$FORCE_SMALL" != true ]]; then
echo "FATAL: existing datadir too small for backup (${folder_size_gb}G at $source_folder); use --force if intentional" >&2
exit 1
fi
timestamp=$(date +'%Y-%m-%d-%H-%M-%S')
target_file="rpc_$key-${timestamp}-${folder_size_gb}G.tar.zst"
@@ -119,5 +193,5 @@ done
echo ""
echo "=== Overall Size Summary ==="
if [[ -f "$BASEPATH/show-size.sh" ]]; then
"$BASEPATH/show-size.sh" "$1" 2>&1
"$BASEPATH/show-size.sh" "$compose_name" 2>&1
fi

16
cleanup-backups.sh
View File

@@ -14,6 +14,8 @@
set -euo pipefail
BASEPATH="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=volume-utils.sh
source "$BASEPATH/volume-utils.sh"
BACKUP_DIR="${BACKUP_DIR:-/backup}"
TRASH_DIR="${TRASH_DIR:-$BACKUP_DIR/trash}"
KEEP_COUNT="${KEEP_COUNT:-3}"
@@ -163,6 +165,20 @@ cleanup_volume_backups() {
continue
fi
if ! is_restorable_backup_archive "$file"; then
local near_size
near_size=$(backup_archive_size_bytes "$file")
local near_meta
near_meta=$(metadata_for_archive "$file")
add_trashed_bytes "$file"
trash_file "$file" "near-empty artifact (${near_size} bytes, excluded from KEEP_COUNT)"
if [[ -f "$near_meta" ]]; then
add_trashed_bytes "$near_meta"
trash_file "$near_meta" "metadata for near-empty artifact"
fi
continue
fi
volume_files["$volume_key"]+="$file"$'\n'
done < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'rpc_*-*.tar.zst' -print0 2>/dev/null)

14
clone-backup.sh
View File

@@ -294,11 +294,12 @@ transfer_backup() {
local key=$1
local volume_name="rpc_$key"
# Find the newest backup file
local backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
# Find the newest restorable backup file (skip near-empty artifacts)
local backup_file
backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name")
if [[ -z "$backup_file" ]] || [[ ! -f "$backup_file" ]]; then
echo "Warning: No backup file found for $volume_name, skipping"
echo "Warning: No restorable backup file found for $volume_name, skipping"
return 1
fi
@@ -482,11 +483,12 @@ transfer_backup_ssh() {
local key=$1
local volume_name="rpc_$key"
# Find the newest backup file
local backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
# Find the newest restorable backup file (skip near-empty artifacts)
local backup_file
backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name")
if [[ -z "$backup_file" ]] || [[ ! -f "$backup_file" ]]; then
echo "Warning: No backup file found for $volume_name, skipping"
echo "Warning: No restorable backup file found for $volume_name, skipping"
return 1
fi

3
list-restorable.sh
View File

@@ -12,6 +12,7 @@
# ./list-restorable.sh <network> --all <chain> # --all can be in any position
dir="$(dirname "$0")"
source "$dir/volume-utils.sh"
registry_file="${dir}/compose_registry.json"
backup_dir="/backup"
@@ -122,7 +123,7 @@ while IFS= read -r entry; do
while IFS= read -r volume; do
volume_name="rpc_${volume}"
# Look for backup files matching pattern: rpc_${volume}-[0-9]*G.tar.zst
backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name")
if [ -z "$backup_file" ]; then
all_backups_exist=false

4
restore-volumes.sh
View File

@@ -135,9 +135,9 @@ while IFS= read -r key; do
declare newest_file
if [[ -n "$remote_source" ]]; then
newest_file=$($dir/list-backups.sh "$remote_source" | grep "rpc_$key-20" | sort | tail -n 1)
newest_file=$($dir/list-backups.sh "$remote_source" | select_newest_remote_backup_from_list "$remote_source" "rpc_$key")
else
newest_file=$(ls -1 "$backup_dir"/"rpc_$key"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
newest_file=$(select_newest_local_backup "$backup_dir" "rpc_$key")
fi
if [ -z "$newest_file" ]; then

174
volume-utils.sh
View File

@@ -3,6 +3,180 @@
# Shared helpers for compose volume operations.
# Ephemeral volumes (init-container config) are excluded from backup/restore/size.
# Minimum on-disk bytes for a backup archive to be considered restorable (near-empty
# zstd headers from purged/missing volumes must not win sort|tail newest selection).
MIN_RESTORABLE_BACKUP_BYTES=$((1024 * 1024))
RPC_ENV_FILE="/root/rpc/.env"
backup_archive_size_bytes() {
local file=$1
stat -c%s "$file" 2>/dev/null || echo 0
}
is_restorable_backup_archive() {
local file=$1
local size
[[ -f "$file" ]] || return 1
size=$(backup_archive_size_bytes "$file")
[[ "$size" =~ ^[0-9]+$ ]] && (( size >= MIN_RESTORABLE_BACKUP_BYTES ))
}
# Pick the lexicographically newest backup that is not near-empty (<1MB). Prints one
# skip line per rejected artifact; returns empty when none qualify.
select_newest_local_backup() {
local backup_dir=$1
local volume_name=$2
local pattern="${backup_dir}/${volume_name}"-[0-9]*G.tar.zst
local newest="" f size
shopt -s nullglob
local files=( $pattern )
shopt -u nullglob
if (( ${#files[@]} == 0 )); then
return 0
fi
IFS=$'\n' files=( $(printf '%s\n' "${files[@]}" | sort) )
unset IFS
for f in "${files[@]}"; do
size=$(backup_archive_size_bytes "$f")
if [[ ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then
echo "skip near-empty backup artifact: $(basename "$f") (${size:-0} bytes)" >&2
continue
fi
newest="$f"
done
if [[ -n "$newest" ]]; then
echo "$newest"
fi
}
remote_backup_archive_size_bytes() {
local remote_source=$1
local archive_name=$2
curl --ipv4 -fsSI "${remote_source}${archive_name}" 2>/dev/null \
| awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
}
# Newest restorable backup filename (not path) from a remote WebDAV listing on stdin.
select_newest_remote_backup_from_list() {
local remote_source=$1
local volume_name=$2
local newest="" f size
while IFS= read -r f; do
[[ -z "$f" ]] && continue
[[ "$f" == "${volume_name}-"* ]] || continue
[[ "$f" == *.tar.zst ]] || continue
size=$(remote_backup_archive_size_bytes "$remote_source" "$f")
if [[ -z "$size" || ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then
echo "skip near-empty backup artifact: $f (${size:-unknown} bytes)" >&2
continue
fi
newest="$f"
done < <(sort)
if [[ -n "$newest" ]]; then
echo "$newest"
fi
}
compose_fence_marker_path() {
local compose_name=$1
local env_dir
env_dir=$(dirname "$RPC_ENV_FILE")
echo "${env_dir}/.env.fence-$(echo "$compose_name" | tr '/' '_')"
}
# Remove a compose yml segment from COMPOSE_FILE; store it in a sidecar marker for restore.
fence_compose_in_env() {
local compose_name=$1
local yml_segment="${compose_name}.yml"
local env_file="$RPC_ENV_FILE"
local marker
marker=$(compose_fence_marker_path "$compose_name")
[[ -f "$env_file" ]] || { echo "WARN: $env_file missing, cannot fence $compose_name" >&2; return 1; }
[[ -f "$marker" ]] && { echo "WARN: fence marker already exists: $marker" >&2; return 1; }
local line compose_file new_compose removed=0
line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
[[ -n "$line" ]] || { echo "WARN: COMPOSE_FILE not found in $env_file" >&2; return 1; }
compose_file="${line#COMPOSE_FILE=}"
compose_file="${compose_file#\"}"
compose_file="${compose_file%\"}"
local part new_parts=()
IFS=':' read -ra parts <<< "$compose_file"
for part in "${parts[@]}"; do
if [[ "$part" == "$yml_segment" ]]; then
if ! printf '%s\n' "$part" > "$marker"; then
echo "WARN: failed to write fence marker $marker" >&2
return 1
fi
removed=1
else
new_parts+=("$part")
fi
done
unset IFS
if (( removed == 0 )); then
echo "WARN: $yml_segment not present in COMPOSE_FILE (already fenced?)" >&2
return 1
fi
new_compose=$(IFS=':'; echo "${new_parts[*]}")
if grep -q '^COMPOSE_FILE=' "$env_file"; then
sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${new_compose}|" "$env_file"
else
echo "COMPOSE_FILE=${new_compose}" >> "$env_file"
fi
echo "Fenced $yml_segment from COMPOSE_FILE (marker: $marker)"
}
# Restore a fenced compose segment before service-scoped start (start needs the entry).
unfence_compose_in_env() {
local compose_name=$1
local yml_segment="${compose_name}.yml"
local env_file="$RPC_ENV_FILE"
local marker
marker=$(compose_fence_marker_path "$compose_name")
[[ -f "$marker" ]] || return 0
local segment line compose_file
segment=$(cat "$marker")
[[ -n "$segment" ]] || { rm -f "$marker"; return 0; }
if [[ -f "$env_file" ]]; then
line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
compose_file="${line#COMPOSE_FILE=}"
compose_file="${compose_file#\"}"
compose_file="${compose_file%\"}"
case ":$compose_file:" in
*:"$segment":*) ;;
*)
if [[ -n "$compose_file" ]]; then
compose_file="${compose_file}:${segment}"
else
compose_file="$segment"
fi
sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${compose_file}|" "$env_file"
;;
esac
else
echo "COMPOSE_FILE=${segment}" > "$env_file"
fi
rm -f "$marker"
echo "Unfenced $yml_segment into COMPOSE_FILE"
}
is_ephemeral_volume_key() {
local key=$1
local compose_file=$2