backup: harden lifecycle fencing and skip near-empty archives

backup-node.sh now stops services, fences COMPOSE_FILE during backup, and
restores via EXIT trap; consumers skip <1MB .tar.zst artifacts so purged-
volume junk cannot shadow real backups.

Co-authored-by: Cursor <cursoragent@cursor.com>

volume-utils.sh

@@ -3,6 +3,180 @@
 # Shared helpers for compose volume operations.
 # Ephemeral volumes (init-container config) are excluded from backup/restore/size.
 
+# Minimum on-disk bytes for a backup archive to be considered restorable (near-empty
+# zstd headers from purged/missing volumes must not win sort|tail newest selection).
+MIN_RESTORABLE_BACKUP_BYTES=$((1024 * 1024))
+
+RPC_ENV_FILE="/root/rpc/.env"
+
+backup_archive_size_bytes() {
+  local file=$1
+  stat -c%s "$file" 2>/dev/null || echo 0
+}
+
+is_restorable_backup_archive() {
+  local file=$1
+  local size
+  [[ -f "$file" ]] || return 1
+  size=$(backup_archive_size_bytes "$file")
+  [[ "$size" =~ ^[0-9]+$ ]] && (( size >= MIN_RESTORABLE_BACKUP_BYTES ))
+}
+
+# Pick the lexicographically newest backup that is not near-empty (<1MB). Prints one
+# skip line per rejected artifact; returns empty when none qualify.
+select_newest_local_backup() {
+  local backup_dir=$1
+  local volume_name=$2
+  local pattern="${backup_dir}/${volume_name}"-[0-9]*G.tar.zst
+  local newest="" f size
+
+  shopt -s nullglob
+  local files=( $pattern )
+  shopt -u nullglob
+  if (( ${#files[@]} == 0 )); then
+    return 0
+  fi
+
+  IFS=$'\n' files=( $(printf '%s\n' "${files[@]}" | sort) )
+  unset IFS
+
+  for f in "${files[@]}"; do
+    size=$(backup_archive_size_bytes "$f")
+    if [[ ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then
+      echo "skip near-empty backup artifact: $(basename "$f") (${size:-0} bytes)" >&2
+      continue
+    fi
+    newest="$f"
+  done
+
+  if [[ -n "$newest" ]]; then
+    echo "$newest"
+  fi
+}
+
+remote_backup_archive_size_bytes() {
+  local remote_source=$1
+  local archive_name=$2
+  curl --ipv4 -fsSI "${remote_source}${archive_name}" 2>/dev/null \
+    | awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
+}
+
+# Newest restorable backup filename (not path) from a remote WebDAV listing on stdin.
+select_newest_remote_backup_from_list() {
+  local remote_source=$1
+  local volume_name=$2
+  local newest="" f size
+
+  while IFS= read -r f; do
+    [[ -z "$f" ]] && continue
+    [[ "$f" == "${volume_name}-"* ]] || continue
+    [[ "$f" == *.tar.zst ]] || continue
+    size=$(remote_backup_archive_size_bytes "$remote_source" "$f")
+    if [[ -z "$size" || ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then
+      echo "skip near-empty backup artifact: $f (${size:-unknown} bytes)" >&2
+      continue
+    fi
+    newest="$f"
+  done < <(sort)
+
+  if [[ -n "$newest" ]]; then
+    echo "$newest"
+  fi
+}
+
+compose_fence_marker_path() {
+  local compose_name=$1
+  local env_dir
+  env_dir=$(dirname "$RPC_ENV_FILE")
+  echo "${env_dir}/.env.fence-$(echo "$compose_name" | tr '/' '_')"
+}
+
+# Remove a compose yml segment from COMPOSE_FILE; store it in a sidecar marker for restore.
+fence_compose_in_env() {
+  local compose_name=$1
+  local yml_segment="${compose_name}.yml"
+  local env_file="$RPC_ENV_FILE"
+  local marker
+  marker=$(compose_fence_marker_path "$compose_name")
+
+  [[ -f "$env_file" ]] || { echo "WARN: $env_file missing, cannot fence $compose_name" >&2; return 1; }
+  [[ -f "$marker" ]] && { echo "WARN: fence marker already exists: $marker" >&2; return 1; }
+
+  local line compose_file new_compose removed=0
+  line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
+  [[ -n "$line" ]] || { echo "WARN: COMPOSE_FILE not found in $env_file" >&2; return 1; }
+
+  compose_file="${line#COMPOSE_FILE=}"
+  compose_file="${compose_file#\"}"
+  compose_file="${compose_file%\"}"
+
+  local part new_parts=()
+  IFS=':' read -ra parts <<< "$compose_file"
+  for part in "${parts[@]}"; do
+    if [[ "$part" == "$yml_segment" ]]; then
+      if ! printf '%s\n' "$part" > "$marker"; then
+        echo "WARN: failed to write fence marker $marker" >&2
+        return 1
+      fi
+      removed=1
+    else
+      new_parts+=("$part")
+    fi
+  done
+  unset IFS
+
+  if (( removed == 0 )); then
+    echo "WARN: $yml_segment not present in COMPOSE_FILE (already fenced?)" >&2
+    return 1
+  fi
+
+  new_compose=$(IFS=':'; echo "${new_parts[*]}")
+  if grep -q '^COMPOSE_FILE=' "$env_file"; then
+    sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${new_compose}|" "$env_file"
+  else
+    echo "COMPOSE_FILE=${new_compose}" >> "$env_file"
+  fi
+  echo "Fenced $yml_segment from COMPOSE_FILE (marker: $marker)"
+}
+
+# Restore a fenced compose segment before service-scoped start (start needs the entry).
+unfence_compose_in_env() {
+  local compose_name=$1
+  local yml_segment="${compose_name}.yml"
+  local env_file="$RPC_ENV_FILE"
+  local marker
+  marker=$(compose_fence_marker_path "$compose_name")
+
+  [[ -f "$marker" ]] || return 0
+
+  local segment line compose_file
+  segment=$(cat "$marker")
+  [[ -n "$segment" ]] || { rm -f "$marker"; return 0; }
+
+  if [[ -f "$env_file" ]]; then
+    line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
+    compose_file="${line#COMPOSE_FILE=}"
+    compose_file="${compose_file#\"}"
+    compose_file="${compose_file%\"}"
+    case ":$compose_file:" in
+      *:"$segment":*) ;;
+      *)
+        if [[ -n "$compose_file" ]]; then
+          compose_file="${compose_file}:${segment}"
+        else
+          compose_file="$segment"
+        fi
+        sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${compose_file}|" "$env_file"
+        ;;
+    esac
+  else
+    echo "COMPOSE_FILE=${segment}" > "$env_file"
+  fi
+
+  rm -f "$marker"
+  echo "Unfenced $yml_segment into COMPOSE_FILE"
+}
+
 is_ephemeral_volume_key() {
   local key=$1
   local compose_file=$2