backup: harden lifecycle fencing and skip near-empty archives

backup-node.sh now stops services, fences COMPOSE_FILE during backup, and
restores via EXIT trap; consumers skip <1MB .tar.zst artifacts so purged-
volume junk cannot shadow real backups.

Co-authored-by: Cursor <cursoragent@cursor.com>

backup-node.sh

@@ -1,10 +1,73 @@
 #!/bin/bash
+#
+# Backup persistent compose volumes to /backup or WebDAV.
+#
+# Usage:
+#   ./backup-node.sh [--hot] [--force] <compose-name> [webdav-url]
+#
+# Lifecycle (default, omitted with --hot):
+#   stop services -> fence compose out of COMPOSE_FILE -> tar+zstd -> unfence+start (EXIT trap).
+#   Hot backups skip stop/fence; leveldb/pebble live tars are NOT restore-trustworthy.
+#
+# --force  Allow backing up an existing but very small datadir (<1GB reported size).
+#          Does NOT bypass a missing volume path (always fatal).
 
 BASEPATH="$(dirname "$0")"
 source "$BASEPATH/volume-utils.sh"
 backup_dir="/backup"
 
-remote_target="$2"
+HOT_BACKUP=false
+FORCE_SMALL=false
+_pos=()
+for _a in "$@"; do
+    case "$_a" in
+        --hot) HOT_BACKUP=true ;;
+        --force) FORCE_SMALL=true ;;
+        *) _pos+=("$_a") ;;
+    esac
+done
+set -- "${_pos[@]}"
+
+if [ -z "${1:-}" ] || [ ! -f "/root/rpc/$1.yml" ]; then
+    echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist."
+    exit 1
+fi
+
+compose_name="$1"
+remote_target="${2:-}"
+lifecycle_active=false
+fenced=false
+stopped=false
+
+backup_lifecycle_cleanup() {
+    local rc=$?
+    if [[ "$lifecycle_active" != true ]]; then
+        exit "$rc"
+    fi
+    if [[ "$fenced" == true ]]; then
+        unfence_compose_in_env "$compose_name" || true
+        fenced=false
+    fi
+    if [[ "$stopped" == true ]]; then
+        "$BASEPATH/start.sh" "$compose_name" || true
+        stopped=false
+    fi
+    exit "$rc"
+}
+
+if [[ "$HOT_BACKUP" == true ]]; then
+    echo "WARNING: --hot backup skips stop/fence; live leveldb/pebble archives may not restore cleanly"
+else
+    trap backup_lifecycle_cleanup EXIT
+    lifecycle_active=true
+    echo "Stopping services for $compose_name before backup..."
+    "$BASEPATH/stop.sh" "$compose_name"
+    stopped=true
+    if fence_compose_in_env "$compose_name"; then
+        fenced=true
+    fi
+fi
+
 if [[ -n "$remote_target" ]] && is_local_backup_url "$remote_target"; then
     echo "Target URL points to this server, using local /backup instead of $remote_target"
     remote_target=""
@@ -62,13 +125,19 @@ generate_volume_metadata() {
 }
 
 # Read the JSON input and extract the list of keys
-keys=$(get_persistent_volume_keys "/root/rpc/$1.yml")
+keys=$(get_persistent_volume_keys "/root/rpc/$compose_name.yml")
 
 # Iterate over the list of keys
 for key in $keys; do
     echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data"
 
     source_folder="/var/lib/docker/volumes/rpc_$key/_data"
+
+    if [[ ! -d "$source_folder" ]]; then
+	echo "FATAL: volume data directory missing, refusing backup: $source_folder" >&2
+	exit 1
+    fi
+
     folder_size=$(du -shL "$source_folder" | awk '{
      size = $1
      sub(/[Kk]$/, "", size)  # Remove 'K' suffix if present
@@ -86,6 +155,11 @@ for key in $keys; do
     }')
 
     folder_size_gb=$(printf "%.0f" "$folder_size")
+
+    if (( folder_size_gb < 1 )) && [[ "$FORCE_SMALL" != true ]]; then
+	echo "FATAL: existing datadir too small for backup (${folder_size_gb}G at $source_folder); use --force if intentional" >&2
+	exit 1
+    fi
     
     timestamp=$(date +'%Y-%m-%d-%H-%M-%S')
     target_file="rpc_$key-${timestamp}-${folder_size_gb}G.tar.zst"
@@ -119,5 +193,5 @@ done
 echo ""
 echo "=== Overall Size Summary ==="
 if [[ -f "$BASEPATH/show-size.sh" ]]; then
-    "$BASEPATH/show-size.sh" "$1" 2>&1
+    "$BASEPATH/show-size.sh" "$compose_name" 2>&1
 fi

cleanup-backups.sh

@@ -14,6 +14,8 @@
 set -euo pipefail
 
 BASEPATH="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=volume-utils.sh
+source "$BASEPATH/volume-utils.sh"
 BACKUP_DIR="${BACKUP_DIR:-/backup}"
 TRASH_DIR="${TRASH_DIR:-$BACKUP_DIR/trash}"
 KEEP_COUNT="${KEEP_COUNT:-3}"
@@ -163,6 +165,20 @@ cleanup_volume_backups() {
             continue
         fi
 
+        if ! is_restorable_backup_archive "$file"; then
+            local near_size
+            near_size=$(backup_archive_size_bytes "$file")
+            local near_meta
+            near_meta=$(metadata_for_archive "$file")
+            add_trashed_bytes "$file"
+            trash_file "$file" "near-empty artifact (${near_size} bytes, excluded from KEEP_COUNT)"
+            if [[ -f "$near_meta" ]]; then
+                add_trashed_bytes "$near_meta"
+                trash_file "$near_meta" "metadata for near-empty artifact"
+            fi
+            continue
+        fi
+
         volume_files["$volume_key"]+="$file"$'\n'
     done < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'rpc_*-*.tar.zst' -print0 2>/dev/null)
 

clone-backup.sh

@@ -294,11 +294,12 @@ transfer_backup() {
     local key=$1
     local volume_name="rpc_$key"
     
-    # Find the newest backup file
+    # Find the newest restorable backup file (skip near-empty artifacts)
-    local backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
+    local backup_file
+    backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name")
     
     if [[ -z "$backup_file" ]] || [[ ! -f "$backup_file" ]]; then
-        echo "Warning: No backup file found for $volume_name, skipping"
+        echo "Warning: No restorable backup file found for $volume_name, skipping"
         return 1
     fi
     
@@ -482,11 +483,12 @@ transfer_backup_ssh() {
     local key=$1
     local volume_name="rpc_$key"
     
-    # Find the newest backup file
+    # Find the newest restorable backup file (skip near-empty artifacts)
-    local backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
+    local backup_file
+    backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name")
     
     if [[ -z "$backup_file" ]] || [[ ! -f "$backup_file" ]]; then
-        echo "Warning: No backup file found for $volume_name, skipping"
+        echo "Warning: No restorable backup file found for $volume_name, skipping"
         return 1
     fi
     

list-restorable.sh

@@ -12,6 +12,7 @@
 #   ./list-restorable.sh <network> --all <chain>  # --all can be in any position
 
 dir="$(dirname "$0")"
+source "$dir/volume-utils.sh"
 registry_file="${dir}/compose_registry.json"
 backup_dir="/backup"
 
@@ -122,7 +123,7 @@ while IFS= read -r entry; do
     while IFS= read -r volume; do
         volume_name="rpc_${volume}"
         # Look for backup files matching pattern: rpc_${volume}-[0-9]*G.tar.zst
-        backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
+        backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name")
         
         if [ -z "$backup_file" ]; then
             all_backups_exist=false

restore-volumes.sh

@@ -135,9 +135,9 @@ while IFS= read -r key; do
     declare newest_file
 
     if [[ -n "$remote_source" ]]; then
-	newest_file=$($dir/list-backups.sh "$remote_source" | grep "rpc_$key-20" | sort | tail -n 1)
+	newest_file=$($dir/list-backups.sh "$remote_source" | select_newest_remote_backup_from_list "$remote_source" "rpc_$key")
     else
-	newest_file=$(ls -1 "$backup_dir"/"rpc_$key"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1)
+	newest_file=$(select_newest_local_backup "$backup_dir" "rpc_$key")
     fi
 
     if [ -z "$newest_file" ]; then

volume-utils.sh

@@ -3,6 +3,180 @@
 # Shared helpers for compose volume operations.
 # Ephemeral volumes (init-container config) are excluded from backup/restore/size.
 
+# Minimum on-disk bytes for a backup archive to be considered restorable (near-empty
+# zstd headers from purged/missing volumes must not win sort|tail newest selection).
+MIN_RESTORABLE_BACKUP_BYTES=$((1024 * 1024))
+
+RPC_ENV_FILE="/root/rpc/.env"
+
+backup_archive_size_bytes() {
+  local file=$1
+  stat -c%s "$file" 2>/dev/null || echo 0
+}
+
+is_restorable_backup_archive() {
+  local file=$1
+  local size
+  [[ -f "$file" ]] || return 1
+  size=$(backup_archive_size_bytes "$file")
+  [[ "$size" =~ ^[0-9]+$ ]] && (( size >= MIN_RESTORABLE_BACKUP_BYTES ))
+}
+
+# Pick the lexicographically newest backup that is not near-empty (<1MB). Prints one
+# skip line per rejected artifact; returns empty when none qualify.
+select_newest_local_backup() {
+  local backup_dir=$1
+  local volume_name=$2
+  local pattern="${backup_dir}/${volume_name}"-[0-9]*G.tar.zst
+  local newest="" f size
+
+  shopt -s nullglob
+  local files=( $pattern )
+  shopt -u nullglob
+  if (( ${#files[@]} == 0 )); then
+    return 0
+  fi
+
+  IFS=$'\n' files=( $(printf '%s\n' "${files[@]}" | sort) )
+  unset IFS
+
+  for f in "${files[@]}"; do
+    size=$(backup_archive_size_bytes "$f")
+    if [[ ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then
+      echo "skip near-empty backup artifact: $(basename "$f") (${size:-0} bytes)" >&2
+      continue
+    fi
+    newest="$f"
+  done
+
+  if [[ -n "$newest" ]]; then
+    echo "$newest"
+  fi
+}
+
+remote_backup_archive_size_bytes() {
+  local remote_source=$1
+  local archive_name=$2
+  curl --ipv4 -fsSI "${remote_source}${archive_name}" 2>/dev/null \
+    | awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
+}
+
+# Newest restorable backup filename (not path) from a remote WebDAV listing on stdin.
+select_newest_remote_backup_from_list() {
+  local remote_source=$1
+  local volume_name=$2
+  local newest="" f size
+
+  while IFS= read -r f; do
+    [[ -z "$f" ]] && continue
+    [[ "$f" == "${volume_name}-"* ]] || continue
+    [[ "$f" == *.tar.zst ]] || continue
+    size=$(remote_backup_archive_size_bytes "$remote_source" "$f")
+    if [[ -z "$size" || ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then
+      echo "skip near-empty backup artifact: $f (${size:-unknown} bytes)" >&2
+      continue
+    fi
+    newest="$f"
+  done < <(sort)
+
+  if [[ -n "$newest" ]]; then
+    echo "$newest"
+  fi
+}
+
+compose_fence_marker_path() {
+  local compose_name=$1
+  local env_dir
+  env_dir=$(dirname "$RPC_ENV_FILE")
+  echo "${env_dir}/.env.fence-$(echo "$compose_name" | tr '/' '_')"
+}
+
+# Remove a compose yml segment from COMPOSE_FILE; store it in a sidecar marker for restore.
+fence_compose_in_env() {
+  local compose_name=$1
+  local yml_segment="${compose_name}.yml"
+  local env_file="$RPC_ENV_FILE"
+  local marker
+  marker=$(compose_fence_marker_path "$compose_name")
+
+  [[ -f "$env_file" ]] || { echo "WARN: $env_file missing, cannot fence $compose_name" >&2; return 1; }
+  [[ -f "$marker" ]] && { echo "WARN: fence marker already exists: $marker" >&2; return 1; }
+
+  local line compose_file new_compose removed=0
+  line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
+  [[ -n "$line" ]] || { echo "WARN: COMPOSE_FILE not found in $env_file" >&2; return 1; }
+
+  compose_file="${line#COMPOSE_FILE=}"
+  compose_file="${compose_file#\"}"
+  compose_file="${compose_file%\"}"
+
+  local part new_parts=()
+  IFS=':' read -ra parts <<< "$compose_file"
+  for part in "${parts[@]}"; do
+    if [[ "$part" == "$yml_segment" ]]; then
+      if ! printf '%s\n' "$part" > "$marker"; then
+        echo "WARN: failed to write fence marker $marker" >&2
+        return 1
+      fi
+      removed=1
+    else
+      new_parts+=("$part")
+    fi
+  done
+  unset IFS
+
+  if (( removed == 0 )); then
+    echo "WARN: $yml_segment not present in COMPOSE_FILE (already fenced?)" >&2
+    return 1
+  fi
+
+  new_compose=$(IFS=':'; echo "${new_parts[*]}")
+  if grep -q '^COMPOSE_FILE=' "$env_file"; then
+    sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${new_compose}|" "$env_file"
+  else
+    echo "COMPOSE_FILE=${new_compose}" >> "$env_file"
+  fi
+  echo "Fenced $yml_segment from COMPOSE_FILE (marker: $marker)"
+}
+
+# Restore a fenced compose segment before service-scoped start (start needs the entry).
+unfence_compose_in_env() {
+  local compose_name=$1
+  local yml_segment="${compose_name}.yml"
+  local env_file="$RPC_ENV_FILE"
+  local marker
+  marker=$(compose_fence_marker_path "$compose_name")
+
+  [[ -f "$marker" ]] || return 0
+
+  local segment line compose_file
+  segment=$(cat "$marker")
+  [[ -n "$segment" ]] || { rm -f "$marker"; return 0; }
+
+  if [[ -f "$env_file" ]]; then
+    line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
+    compose_file="${line#COMPOSE_FILE=}"
+    compose_file="${compose_file#\"}"
+    compose_file="${compose_file%\"}"
+    case ":$compose_file:" in
+      *:"$segment":*) ;;
+      *)
+        if [[ -n "$compose_file" ]]; then
+          compose_file="${compose_file}:${segment}"
+        else
+          compose_file="$segment"
+        fi
+        sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${compose_file}|" "$env_file"
+        ;;
+    esac
+  else
+    echo "COMPOSE_FILE=${segment}" > "$env_file"
+  fi
+
+  rm -f "$marker"
+  echo "Unfenced $yml_segment into COMPOSE_FILE"
+}
+
 is_ephemeral_volume_key() {
   local key=$1
   local compose_file=$2