ethereum-rpc-docker/restore-volumes.sh

#!/bin/bash
dir="$(dirname "$0")"
source "$dir/volume-utils.sh"

# Pull out the --no-slowdisk flag (position-independent); keep the positional args
# ($1 = compose name, $2 = optional remote source) intact for the rest of the script.
no_slowdisk_flag=0
_pos=()
for _a in "$@"; do
    case "$_a" in
        --no-slowdisk) no_slowdisk_flag=1 ;;
        *) _pos+=("$_a") ;;
    esac
done
set -- "${_pos[@]}"

# Static-file offload gate. Hosts with a real dedicated extra disk mounted at /slowdisk set
# SLOWDISK=True in their .env (Python-templated boolean, capitalized); that enables the
# offload. The --no-slowdisk flag disables it even when SLOWDISK=True (e.g. extra disk full).
[ -f "$dir/.env" ] && source "$dir/.env"
SLOWDISK="${SLOWDISK:-}"
[ "$no_slowdisk_flag" = 1 ] && SLOWDISK=False
SLOWDISK_FLOOR_GB="${SLOWDISK_FLOOR_GB:-150}"

remote_source="$2"
if [[ -n "$remote_source" ]] && is_local_backup_url "$remote_source"; then
    echo "Source URL points to this server, using local /backup instead of $remote_source"
    remote_source=""
fi

# Path to the backup directory
backup_dir="/backup"

# Path to the volume directory
volume_dir="/var/lib/docker/volumes"

if [ ! -d "$volume_dir" ]; then
    echo "Error: /var/lib/docker/volumes directory does not exist"
    exit 1
fi

# Pre-create static-file -> /slowdisk symlinks from a backup's ".txt" manifest, so the
# immutable "ancient"/freezer dirs land on the (SSD) /slowdisk during extraction while the
# hot/dynamic state stays on the primary disk. tar then extracts THROUGH the symlinks via
# --keep-directory-symlink (it keeps the dir-symlinks instead of clobbering them).
# Target naming matches delete-volumes.sh / delete_slowdisk_targets_for_key cleanup.
# GATED on SLOWDISK (see top): offload runs only when SLOWDISK is the Python-templated boolean
# "True" (set in the host .env on dedicated-extra-disk hosts) and the --no-slowdisk flag was
# not passed. Case matters — the value comes through as capitalized "True"/"False". Safe
# fallbacks (normal extract): SLOWDISK not True, /slowdisk missing, no manifest, no static paths,
# or slowdisk free space below SLOWDISK_FLOOR_GB after projected static payload.
slowdisk_free_gb() {
    local free
    free=$(df -BG /slowdisk 2>/dev/null | awk 'NR==2 {gsub(/G/,"",$4); print $4}')
    [ -n "$free" ] && [[ "$free" =~ ^[0-9]+$ ]] || return 1
    echo "$free"
}

# Best cheap size signal for incoming static payload: backup archive bytes (local stat or
# remote Content-Length). Archive is compressed (.tar.zst); treat size as a LOWER bound and
# assume ~2x when estimating uncompressed footprint on /slowdisk.
backup_archive_bytes() {
    local newest_file=$1
    if [[ -n "$remote_source" ]]; then
        local cached="$backup_dir/$(basename "$newest_file")"
        if [[ -f "$cached" && ! -e "${cached}.aria2" ]]; then
            stat -c%s "$cached" 2>/dev/null && return 0
        fi
        curl --ipv4 -fsSI "${remote_source}${newest_file}" 2>/dev/null \
            | awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
    else
        stat -c%s "$newest_file" 2>/dev/null
    fi
}

# Pre-extract floor gate: return 0 if offload may proceed, 1 to fall back to primary extract.
slowdisk_floor_allows_offload() {
    local newest_file=$1 archive_bytes need_gb free_gb floor_gb
    floor_gb="${SLOWDISK_FLOOR_GB:-150}"
    free_gb=$(slowdisk_free_gb) || return 0
    archive_bytes=$(backup_archive_bytes "$newest_file")
    if [[ -z "$archive_bytes" || ! "$archive_bytes" =~ ^[0-9]+$ ]]; then
        return 0
    fi
    # ~2x uncompressed factor (archive is compressed zstd tar)
    need_gb=$(( (archive_bytes * 2 + 1024*1024*1024 - 1) / (1024*1024*1024) ))
    if (( free_gb - need_gb < floor_gb )); then
        echo "slowdisk floor gate: skipping offload (free ${free_gb} GB, projected need ${need_gb} GB, floor ${floor_gb} GB) — extracting to primary"
        return 1
    fi
    return 0
}

warn_slowdisk_floor_breach() {
    local free_gb floor_gb
    floor_gb="${SLOWDISK_FLOOR_GB:-150}"
    free_gb=$(slowdisk_free_gb) || return 0
    if (( free_gb < floor_gb )); then
        echo "WARNING: slowdisk floor breach after static offload (free ${free_gb} GB < floor ${floor_gb} GB) — OS headroom compromised"
    fi
}

prep_static_offload() {
    local key=$1 meta=$2 data_dir=$3 newest_file=$4 rel target
    case "$SLOWDISK" in
        True|true) ;;  # offload enabled (Python "True"; also accept manual lowercase "true")
        *) echo "  static offload disabled (SLOWDISK=$SLOWDISK) — normal extract"; return 0 ;;
    esac
    [ -d /slowdisk ] || { echo "  /slowdisk absent — no static offload"; return 0; }
    [ -f "$meta" ]   || { echo "  no manifest ($meta) — no static offload"; return 0; }
    # Fit check against the manifest's actual static sizes (not the old whole-archive x2
    # estimate, which false-refuses restores whose statics are a fraction of the archive).
    # When the statics do NOT fit, FAIL LOUDLY instead of silently extracting everything
    # onto the primary disk: a silent NVMe fallback violates the caller's capacity math
    # (operator directive 2026-07-09). --no-slowdisk is the conscious override.
    local need_gb free_gb floor_gb growth_pct
    floor_gb="${SLOWDISK_FLOOR_GB:-150}"
    # Static files GROW (reth appends segments as the chain advances, ~5%/mo), and /slowdisk
    # is usually the SATA root partition — filling it starves the OS/logs. Each offload must
    # bring its own 90d growth headroom on top of the floor (operator directive 2026-07-09).
    growth_pct="${SLOWDISK_GROWTH_PCT:-15}"
    free_gb=$(slowdisk_free_gb || echo "")
    need_gb=$(awk -v g="$growth_pct" 'NR>3 && NF>=2 {
        s=$1; mult=1
        if (s ~ /TB$/) mult=1024; else if (s ~ /MB$/) mult=1/1024; else if (s ~ /KB$/) mult=1/1048576
        gsub(/[A-Za-z]/, "", s); total+=s*mult
    } END {printf "%d", total*1.05*(1+g/100) + 1}' "$meta")
    if [[ "$free_gb" =~ ^[0-9]+$ && "$need_gb" =~ ^[0-9]+$ ]] && (( free_gb - need_gb < floor_gb )); then
        echo "ERROR: static offload does not fit on /slowdisk (need ~${need_gb}G incl. ${growth_pct}% growth headroom, free ${free_gb}G, floor ${floor_gb}G)." >&2
        echo "       This restore can only fit with --no-slowdisk (extracts everything onto the" >&2
        echo "       primary disk — NVMe capacity accounting will differ from the offload plan)." >&2
        exit 1
    fi
    static_offload_used=1
    # manifest data lines (after the 3-line header) are "<size>  <relpath>"
    while IFS= read -r rel; do
        [ -z "$rel" ] && continue
        rel="${rel#/}"
        case "$rel" in *..*) echo "  skip unsafe static path '$rel'"; continue;; esac
        target="/slowdisk/rpc_${key}__data_${rel//\//__}"
        echo "  offload static '$rel' -> $target"
        mkdir -p "$target" "$data_dir/$(dirname "$rel")" || { echo "  WARN: mkdir failed for '$rel', skipping"; continue; }
        ln -sfn "$target" "$data_dir/$rel"
        # tar CANNOT extract through a symlink whose target is on another device -
        # open() fails with EXDEV "Invalid cross-device link" even with
        # --keep-directory-symlink (reproduced GNU tar 1.34 + 1.35, 2026-07-11;
        # killed the katana->uk-4 provision). Rewrite the member paths so tar
        # writes DIRECTLY into the /slowdisk target; the symlink stays for the
        # container's runtime view.
        static_transform_args+=(--transform "s|^\\(\\./\\)\\?var/lib/docker/volumes/rpc_${key}/_data/${rel}|slowdisk/rpc_${key}__data_${rel//\//__}|")
    done < <(awk 'NR>3 && NF>=2 {print $NF}' "$meta")
}

# Read the JSON input and extract the list of keys
keys=$(get_persistent_volume_keys "$dir/$1.yml" | grep -E '^[0-9a-z]')

echo "$keys"

while IFS= read -r key; do
    [ -z "$key" ] && continue
    data_dir="$volume_dir/rpc_$key/_data"
    declare newest_file

    if [[ -n "$remote_source" ]]; then
	newest_file=$($dir/list-backups.sh "$remote_source" | select_newest_remote_backup_from_list "$remote_source" "rpc_$key")
    else
	newest_file=$(select_newest_local_backup "$backup_dir" "rpc_$key")
    fi

    if [ -z "$newest_file" ]; then
	# Check if this volume is optional or ephemeral - if so, warn and continue
	if is_ephemeral_volume_key "$key" "$dir/$1.yml" || is_optional_volume_key "$key" "$dir/$1.yml"; then
		echo "WARN: No backup found for volume 'rpc_$key' (volume is optional/ephemeral) - creating empty volume"
		# Create empty volume and continue
		mkdir -p "$data_dir"
		continue
	fi
	echo "Error: No backup found for volume 'rpc_$key'"
	exit 1
    fi
    meta_file="${newest_file%.tar.zst}.txt"
    echo "=== restoring rpc_$key  <-  $newest_file ==="
    static_offload_used=0
    static_transform_args=()

    # 1) wipe live data AND any /slowdisk static targets for this key (no leak on re-restore)
    delete_slowdisk_targets_for_key "$key"
    [ -d "$data_dir" ] && rm -rf "$data_dir"/*
    mkdir -p "$data_dir"

    # 2) obtain the manifest (fetch the sidecar .txt for remote restores) and pre-create the
    #    offload. (The old reth guard is gone: op-reth v2.3.2 verified running + WRITING through
    #    a symlinked static_files dir on rpc-uk-4, 2026-07-09. Ancient reths that predate this
    #    refuse to start - symptom is loud, fix is --no-slowdisk.)
    local_meta="$meta_file"
    if [[ -n "$remote_source" ]]; then
	local_meta="$backup_dir/$(basename "$meta_file")"
	[ -d "$backup_dir" ] || local_meta="/tmp/$(basename "$meta_file")"
	if [ ! -f "$local_meta" ]; then
	    curl --ipv4 -fsS "${remote_source}${meta_file}" -o "$local_meta" 2>/dev/null || local_meta=""
	fi
    fi
    if [ -n "$local_meta" ]; then
	prep_static_offload "$key" "$local_meta" "$data_dir" "$newest_file"
    fi

    # 3) extract THROUGH the pre-created symlinks (keep them, don't clobber)
    if [[ -n "$remote_source" ]]; then
	if [ ! -d "$backup_dir" ]; then
	    echo "No /backup cache: streaming + extracting $newest_file directly"
	    curl --ipv4 -# "${remote_source}${newest_file}" | zstd -d | tar -xf - --keep-directory-symlink "${static_transform_args[@]}" -C /
	    if [ $? -ne 0 ]; then
		echo "Error processing $newest_file" >&2
		exit 1
	    fi
	else
	    backup_file="$backup_dir/$(basename "$newest_file")"
	    if [ ! -e "$backup_file" ] || [ -e "${backup_file}.aria2" ]; then
		aria2c -c -Z -x8 -j8 -s8 -d "$backup_dir" "${remote_source}${newest_file}"
	    fi
	    tar -I zstd -xf "$backup_file" --keep-directory-symlink "${static_transform_args[@]}" -C /
	    if [ $? -ne 0 ]; then
		echo "Error processing $newest_file" >&2
		exit 1
	    fi
	fi
    else
	tar -I zstd -xf "$newest_file" --keep-directory-symlink "${static_transform_args[@]}" -C /
	    if [ $? -ne 0 ]; then
		echo "Error processing $newest_file" >&2
		exit 1
	    fi
    fi
    if [ "$static_offload_used" = 1 ]; then
	warn_slowdisk_floor_breach
    fi
    echo "Backup '$newest_file' restored"
done <<< "$keys"

"$dir/delete-node-keys.sh" "$1"

echo "node $1 restored."