diff --git a/clone-backup.sh b/clone-backup.sh index 4274f069..9758a5a6 100755 --- a/clone-backup.sh +++ b/clone-backup.sh @@ -2,6 +2,23 @@ # Script to clone node volumes from backup location to a remote server # Uses the same transfer method as clone-node.sh but streams zstd archives directly from backup location +# +# Usage: ./clone-backup.sh [--no-slowdisk] +# --no-slowdisk : do NOT offload static files to /slowdisk (no symlinks); +# extract everything onto the local disk instead. Required to +# proceed when /slowdisk is too small for the static files. + +# Pull the --no-slowdisk flag out of the args so the positional / +# parsing below is unaffected regardless of where the flag is placed. +NO_SLOWDISK=false +_args=() +for _a in "$@"; do + case "$_a" in + --no-slowdisk) NO_SLOWDISK=true ;; + *) _args+=("$_a") ;; + esac +done +set -- "${_args[@]}" if [[ -n $2 ]]; then DEST_HOST="$2.stakesquid.eu" @@ -108,7 +125,7 @@ release_port() { # Check if SLOWDISK mode is enabled on target check_slowdisk_enabled() { - $SSH_CMD "$DEST_HOST" "grep -q '^SLOWDISK=true' /root/rpc/.env 2>/dev/null" + $SSH_CMD "$DEST_HOST" "grep -qi '^SLOWDISK=true' /root/rpc/.env 2>/dev/null" return $? } @@ -159,6 +176,40 @@ parse_metadata() { printf '%s\n' "${static_paths[@]}" } +# Decide whether to offload this volume's static files to /slowdisk. +# return 0 -> use /slowdisk (caller should create symlinks + extract) +# return 1 -> skip /slowdisk (extract static files onto local disk) +# Aborts the whole script if /slowdisk is too small and --no-slowdisk was not +# given, to avoid overfilling /slowdisk with a partially-extracted snapshot. +decide_slowdisk() { + local key=$1 + local static_size_kb=$2 + + # Explicit override: never offload, never create symlinks. + if [[ "$NO_SLOWDISK" == "true" ]]; then + echo " --no-slowdisk: skipping /slowdisk offload for '$key' (static files -> local disk)" + return 1 + fi + + # How much room is actually on /slowdisk right now? + local slowdisk_available + slowdisk_available=$($SSH_CMD "$DEST_HOST" "df -BK /slowdisk 2>/dev/null | tail -1 | awk '{print \$4}' | sed 's/K//'") + + if [[ -z "$slowdisk_available" ]] || [[ "$slowdisk_available" -lt "$static_size_kb" ]]; then + { + echo "" + echo "WARNING: /slowdisk on $DEST_HOST is too small for the static files of '$key'." + echo " available: $(( ${slowdisk_available:-0} / 1024 ))MB, needed: $(( static_size_kb / 1024 ))MB" + echo " Aborting to avoid overfilling /slowdisk with a partial snapshot." + echo " Re-run with --no-slowdisk to extract static files onto the local disk" + echo " instead (no /slowdisk offload, no symlinks created)." + } >&2 + exit 1 + fi + + return 0 +} + # Setup slowdisk directory structure with symlinks setup_slowdisk_structure() { local key=$1 @@ -267,16 +318,21 @@ transfer_backup() { local static_paths=($(echo "$metadata_output" | tail -n +2)) if [[ ${#static_paths[@]} -gt 0 ]] && [[ -n "$static_size_kb" ]]; then - # Setup slowdisk structure on remote - if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then - use_slowdisk=true - # Use --skip-old-files to avoid overwriting existing symlinks/directories - # But we still want to extract files into symlinked directories - tar_extract_opts="-xf - -C / --skip-old-files" - echo "SLOWDISK structure ready, will extract respecting symlinks" - else - echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" + # Check /slowdisk capacity (aborts if too small without --no-slowdisk). + if decide_slowdisk "$key" "$static_size_kb"; then + # Setup slowdisk structure on remote + if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then + use_slowdisk=true + # Use --skip-old-files to avoid overwriting existing symlinks/directories + # But we still want to extract files into symlinked directories + tar_extract_opts="-xf - -C / --skip-old-files" + echo "SLOWDISK structure ready, will extract respecting symlinks" + else + echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" + fi fi + # decide_slowdisk returned 1 (--no-slowdisk): leave tar_extract_opts at + # the default so static files are extracted onto the local disk. else echo "Warning: Could not parse metadata file, falling back to normal extraction" fi diff --git a/clone-node.sh b/clone-node.sh index d54d14b6..ed92c4dd 100755 --- a/clone-node.sh +++ b/clone-node.sh @@ -5,6 +5,21 @@ BASEPATH="$(dirname "$0")" source "$BASEPATH/volume-utils.sh" +# Usage: ./clone-node.sh [--no-slowdisk] +# --no-slowdisk : do NOT offload static files to /slowdisk on the target (no +# symlinks); extract everything to the local disk. Required to +# proceed when the target's /slowdisk is too small. Strip it +# out before the positional / parsing below. +NO_SLOWDISK=false +_args=() +for _a in "$@"; do + case "$_a" in + --no-slowdisk) NO_SLOWDISK=true ;; + *) _args+=("$_a") ;; + esac +done +set -- "${_args[@]}" + if [[ -n $2 ]]; then DEST_HOST="$2.stakesquid.eu" echo "Setting up optimized transfer to $DEST_HOST" @@ -104,7 +119,7 @@ release_port() { # Check if SLOWDISK mode is enabled on target check_slowdisk_enabled() { - $SSH_CMD "$DEST_HOST" "grep -q '^SLOWDISK=true' /root/rpc/.env 2>/dev/null" + $SSH_CMD "$DEST_HOST" "grep -qi '^SLOWDISK=true' /root/rpc/.env 2>/dev/null" return $? } @@ -141,6 +156,36 @@ detect_static_files() { printf '%s\n' "${static_paths[@]}" } +# Decide whether to offload this volume's static files to the target's /slowdisk. +# return 0 -> use /slowdisk (caller sets up symlinks + extracts through them) +# return 1 -> skip /slowdisk (extract static files onto local disk) +# Aborts the whole clone if /slowdisk is too small and --no-slowdisk was not +# given, to avoid overfilling /slowdisk with a partially-extracted volume. +decide_slowdisk() { + local key=$1 + local static_size_kb=$2 + + if [[ "$NO_SLOWDISK" == "true" ]]; then + echo " --no-slowdisk: skipping /slowdisk offload for '$key' (static files -> local disk)" + return 1 + fi + + local slowdisk_available + slowdisk_available=$($SSH_CMD "$DEST_HOST" "df -BK /slowdisk 2>/dev/null | tail -1 | awk '{print \$4}' | sed 's/K//'") + if [[ -z "$slowdisk_available" ]] || [[ "$slowdisk_available" -lt "$static_size_kb" ]]; then + { + echo "" + echo "WARNING: /slowdisk on $DEST_HOST is too small for the static files of '$key'." + echo " available: $(( ${slowdisk_available:-0} / 1024 ))MB, needed: $(( static_size_kb / 1024 ))MB" + echo " Aborting to avoid overfilling /slowdisk with a partial volume." + echo " Re-run with --no-slowdisk to clone without /slowdisk offload (static files on local disk)." + } >&2 + exit 1 + fi + + return 0 +} + # Setup slowdisk directory structure with symlinks setup_slowdisk_structure() { local key=$1 @@ -245,17 +290,21 @@ transfer_volume() { if [[ ${#static_paths[@]} -gt 0 ]] && [[ -n "$static_size_kb" ]]; then echo "Found ${#static_paths[@]} static paths (total: $((static_size_kb / 1024))MB)" - echo "Setting up SLOWDISK structure on target..." - - # Setup slowdisk structure on remote - if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then - use_slowdisk=true - # Use --skip-old-files to avoid overwriting existing symlinks/directories - tar_extract_opts="-xf - -C / --skip-old-files" - echo "SLOWDISK structure ready, will extract respecting symlinks" - else - echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" + # Check /slowdisk capacity (aborts if too small without --no-slowdisk). + if decide_slowdisk "$key" "$static_size_kb"; then + echo "Setting up SLOWDISK structure on target..." + # Setup slowdisk structure on remote + if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then + use_slowdisk=true + # Use --skip-old-files to avoid overwriting existing symlinks/directories + tar_extract_opts="-xf - -C / --skip-old-files" + echo "SLOWDISK structure ready, will extract respecting symlinks" + else + echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" + fi fi + # decide_slowdisk returned 1 (--no-slowdisk): leave tar_extract_opts + # at the default so static files extract onto the local disk. else echo "No static files detected, using normal extraction" fi @@ -421,15 +470,19 @@ transfer_volume_ssh() { if [[ ${#static_paths[@]} -gt 0 ]] && [[ -n "$static_size_kb" ]]; then echo "Found ${#static_paths[@]} static paths (total: $((static_size_kb / 1024))MB)" - echo "Setting up SLOWDISK structure on target..." - - # Setup slowdisk structure on remote - if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then - tar_extract_opts="-xf - -C / --skip-old-files" - echo "SLOWDISK structure ready, will extract respecting symlinks" - else - echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" + # Check /slowdisk capacity (aborts if too small without --no-slowdisk). + if decide_slowdisk "$key" "$static_size_kb"; then + echo "Setting up SLOWDISK structure on target..." + # Setup slowdisk structure on remote + if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then + tar_extract_opts="-xf - -C / --skip-old-files" + echo "SLOWDISK structure ready, will extract respecting symlinks" + else + echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" + fi fi + # decide_slowdisk returned 1 (--no-slowdisk): leave tar_extract_opts + # at the default so static files extract onto the local disk. else echo "No static files detected, using normal extraction" fi