ethereum-rpc-docker/backup-node.sh

#!/bin/bash
#
# Backup persistent compose volumes to /backup or WebDAV.
#
# Usage:
#   ./backup-node.sh [--hot] [--force] <compose-name> [webdav-url]
#
# Lifecycle (default, omitted with --hot):
#   stop services -> fence compose out of COMPOSE_FILE -> tar+zstd -> unfence+start (EXIT trap).
#   Hot backups skip stop/fence; leveldb/pebble live tars are NOT restore-trustworthy.
#
# --force  Allow backing up an existing but very small datadir (<1GB reported size).
#          Does NOT bypass a missing volume path (always fatal).

BASEPATH="$(dirname "$0")"
source "$BASEPATH/volume-utils.sh"
backup_dir="/backup"

HOT_BACKUP=false
FORCE_SMALL=false
_pos=()
for _a in "$@"; do
    case "$_a" in
        --hot) HOT_BACKUP=true ;;
        --force) FORCE_SMALL=true ;;
        *) _pos+=("$_a") ;;
    esac
done
set -- "${_pos[@]}"

if [ -z "${1:-}" ] || [ ! -f "/root/rpc/$1.yml" ]; then
    echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist."
    exit 1
fi

compose_name="$1"
remote_target="${2:-}"
lifecycle_active=false
fenced=false
stopped=false

backup_lifecycle_cleanup() {
    local rc=$?
    if [[ "$lifecycle_active" != true ]]; then
        exit "$rc"
    fi
    if [[ "$fenced" == true ]]; then
        unfence_compose_in_env "$compose_name" || true
        fenced=false
    fi
    if [[ "$stopped" == true ]]; then
        "$BASEPATH/start.sh" "$compose_name" || true
        stopped=false
    fi
    exit "$rc"
}

if [[ "$HOT_BACKUP" == true ]]; then
    echo "WARNING: --hot backup skips stop/fence; live leveldb/pebble archives may not restore cleanly"
else
    trap backup_lifecycle_cleanup EXIT
    lifecycle_active=true
    echo "Stopping services for $compose_name before backup..."
    "$BASEPATH/stop.sh" "$compose_name"
    stopped=true
    if fence_compose_in_env "$compose_name"; then
        fenced=true
    fi
fi

if [[ -n "$remote_target" ]] && is_local_backup_url "$remote_target"; then
    echo "Target URL points to this server, using local /backup instead of $remote_target"
    remote_target=""
fi

if [[ -n "$remote_target" ]]; then
    echo "upload backup via webdav to $remote_target"
else
    if [ ! -d "$backup_dir" ]; then
	echo "Error: /backup directory does not exist"
	exit 1
    fi
fi

# Function to generate metadata for a single volume
generate_volume_metadata() {
    local volume_key=$1
    local source_folder=$2
    local metadata_file=$3
    
    prefix="/var/lib/docker/volumes/rpc_$volume_key"
    static_file_list="$BASEPATH/static-file-path-list.txt"
    
    # Initialize metadata file
    echo "Static file paths and sizes for volume: rpc_$volume_key" > "$metadata_file"
    echo "Generated: $(date)" >> "$metadata_file"
    echo "" >> "$metadata_file"
    
    # Check each static file path
    if [[ -f "$static_file_list" ]]; then
        while IFS= read -r path; do
            [[ -z "$path" ]] && continue
            # Match rule (see static-file-path-list.txt): an entry with NO slash is root-level
            # only (so `snapshots` won't catch postgres pg_logical/snapshots); an entry WITH a
            # slash is a path-suffix, so any prefix matches (network-prefixed nitro freezers,
            # nested l2geth geth/geth/...). Record the CONCRETE relative path so restore-volumes
            # can recreate the exact static-file -> /slowdisk symlink.
            # Match real dirs AND symlinks: an already-OFFLOADED static dir is a symlink to
            # /slowdisk, which find -type d alone would skip (then it would be dropped from the
            # manifest and the next restore would not re-offload it).
            local matches=() m rel
            if [[ "$path" == */* ]]; then
                while IFS= read -r m; do matches+=("$m"); done < <(find "$prefix/_data" \( -type d -o -type l \) -path "*/$path" 2>/dev/null)
            else
                m="$prefix/_data/$path"; { [ -d "$m" ] || [ -L "$m" ]; } && matches+=("$m")
            fi
            for m in "${matches[@]}"; do
                rel="${m#"$prefix/_data/"}"
                size=$(du -sL "$m" 2>/dev/null | awk '{print $1}')
                size_formatted=$(echo "$(( ${size:-0} * 1024 ))" | numfmt --to=iec --suffix=B --format="%.2f")
                echo "$size_formatted  $rel" >> "$metadata_file"
            done
        done < "$static_file_list"
    fi
}

# Read the JSON input and extract the list of keys
keys=$(get_persistent_volume_keys "/root/rpc/$compose_name.yml")

# Iterate over the list of keys
for key in $keys; do
    echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data"

    source_folder="/var/lib/docker/volumes/rpc_$key/_data"

    if [[ ! -d "$source_folder" ]]; then
	echo "FATAL: volume data directory missing, refusing backup: $source_folder" >&2
	exit 1
    fi

    folder_size=$(du -shL "$source_folder" | awk '{
     size = $1
     sub(/[Kk]$/, "", size)  # Remove 'K' suffix if present
     sub(/[Mm]$/, "", size)  # Remove 'M' suffix if present
     sub(/[Gg]$/, "", size)  # Remove 'G' suffix if present
     sub(/[Tt]$/, "", size)  # Remove 'T' suffix if present
     if ($1 ~ /[Kk]$/) {
         size *= 0.001   # Convert kilobytes to gigabytes
     } else if ($1 ~ /[Mm]$/) {
         size *= 0.001   # Convert megabytes to gigabytes
     } else if ($1 ~ /[Tt]$/) {
     	 size *= 1000 # convert terabytes to gigabytes
     }
     print size
    }')

    folder_size_gb=$(printf "%.0f" "$folder_size")

    if (( folder_size_gb < 1 )) && [[ "$FORCE_SMALL" != true ]]; then
	# The torn-backup floor applies to the MAIN data volume only (key == compose
	# basename). Sidecar volumes (eigenda-proxy, configs, ...) are legitimately
	# tiny — a 0G sidecar aborting the whole backup left no manifest and blocked
	# the refresh lane on every ronin bench-retire (4 cycles, 2026-07).
	if [[ "$key" == "$(basename "$compose_name")" ]]; then
	    echo "FATAL: existing datadir too small for backup (${folder_size_gb}G at $source_folder); use --force if intentional" >&2
	    exit 1
	fi
	echo "NOTE: sidecar volume rpc_$key is ${folder_size_gb}G — allowing small backup (floor guards the main data volume only)"
    fi
    
    timestamp=$(date +'%Y-%m-%d-%H-%M-%S')
    target_file="rpc_$key-${timestamp}-${folder_size_gb}G.tar.zst"
    metadata_file_name="rpc_$key-${timestamp}-${folder_size_gb}G.txt"

    #echo "$target_file"

    if [[ -n "$remote_target" ]]; then
	# Upload volume archive
	tar -cf - --dereference "$source_folder" | pv -pterb -s $(du -sb "$source_folder" | awk '{print $1}') | zstd | curl -X PUT --upload-file - "$remote_target/null/uploading-$target_file"
        curl -X MOVE -H "Destination: /null/$target_file" "$remote_target/null/uploading-$target_file"
        
        # Generate and upload metadata file
        echo "Generating metadata for volume: rpc_$key"
        temp_metadata="/tmp/$metadata_file_name"
        generate_volume_metadata "$key" "$source_folder" "$temp_metadata"
        curl -X PUT --upload-file "$temp_metadata" "$remote_target/null/$metadata_file_name"
        rm -f "$temp_metadata"
    else    
        # Create volume archive
        tar -cf - --dereference "$source_folder" | pv -pterb -s $(du -sb "$source_folder" | awk '{print $1}') | zstd -o "/backup/uploading-$target_file"
        mv "/backup/uploading-$target_file" "/backup/$target_file"
        
        # Generate metadata file
        echo "Generating metadata for volume: rpc_$key"
        generate_volume_metadata "$key" "$source_folder" "/backup/$metadata_file_name"
    fi
done

# Run show-size.sh to display overall summary
echo ""
echo "=== Overall Size Summary ==="
if [[ -f "$BASEPATH/show-size.sh" ]]; then
    "$BASEPATH/show-size.sh" "$compose_name" 2>&1
fi