backup: harden lifecycle fencing and skip near-empty archives

backup-node.sh now stops services, fences COMPOSE_FILE during backup, and
restores via EXIT trap; consumers skip <1MB .tar.zst artifacts so purged-
volume junk cannot shadow real backups.

Co-authored-by: Cursor <cursoragent@cursor.com>

backup-node.sh

@@ -1,10 +1,73 @@
 #!/bin/bash
+#
+# Backup persistent compose volumes to /backup or WebDAV.
+#
+# Usage:
+#   ./backup-node.sh [--hot] [--force] <compose-name> [webdav-url]
+#
+# Lifecycle (default, omitted with --hot):
+#   stop services -> fence compose out of COMPOSE_FILE -> tar+zstd -> unfence+start (EXIT trap).
+#   Hot backups skip stop/fence; leveldb/pebble live tars are NOT restore-trustworthy.
+#
+# --force  Allow backing up an existing but very small datadir (<1GB reported size).
+#          Does NOT bypass a missing volume path (always fatal).
 
 BASEPATH="$(dirname "$0")"
 source "$BASEPATH/volume-utils.sh"
 backup_dir="/backup"
 
-remote_target="$2"
+HOT_BACKUP=false
+FORCE_SMALL=false
+_pos=()
+for _a in "$@"; do
+    case "$_a" in
+        --hot) HOT_BACKUP=true ;;
+        --force) FORCE_SMALL=true ;;
+        *) _pos+=("$_a") ;;
+    esac
+done
+set -- "${_pos[@]}"
+
+if [ -z "${1:-}" ] || [ ! -f "/root/rpc/$1.yml" ]; then
+    echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist."
+    exit 1
+fi
+
+compose_name="$1"
+remote_target="${2:-}"
+lifecycle_active=false
+fenced=false
+stopped=false
+
+backup_lifecycle_cleanup() {
+    local rc=$?
+    if [[ "$lifecycle_active" != true ]]; then
+        exit "$rc"
+    fi
+    if [[ "$fenced" == true ]]; then
+        unfence_compose_in_env "$compose_name" || true
+        fenced=false
+    fi
+    if [[ "$stopped" == true ]]; then
+        "$BASEPATH/start.sh" "$compose_name" || true
+        stopped=false
+    fi
+    exit "$rc"
+}
+
+if [[ "$HOT_BACKUP" == true ]]; then
+    echo "WARNING: --hot backup skips stop/fence; live leveldb/pebble archives may not restore cleanly"
+else
+    trap backup_lifecycle_cleanup EXIT
+    lifecycle_active=true
+    echo "Stopping services for $compose_name before backup..."
+    "$BASEPATH/stop.sh" "$compose_name"
+    stopped=true
+    if fence_compose_in_env "$compose_name"; then
+        fenced=true
+    fi
+fi
+
 if [[ -n "$remote_target" ]] && is_local_backup_url "$remote_target"; then
     echo "Target URL points to this server, using local /backup instead of $remote_target"
     remote_target=""
@@ -62,13 +125,19 @@ generate_volume_metadata() {
 }
 
 # Read the JSON input and extract the list of keys
-keys=$(get_persistent_volume_keys "/root/rpc/$1.yml")
+keys=$(get_persistent_volume_keys "/root/rpc/$compose_name.yml")
 
 # Iterate over the list of keys
 for key in $keys; do
     echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data"
 
     source_folder="/var/lib/docker/volumes/rpc_$key/_data"
+
+    if [[ ! -d "$source_folder" ]]; then
+	echo "FATAL: volume data directory missing, refusing backup: $source_folder" >&2
+	exit 1
+    fi
+
     folder_size=$(du -shL "$source_folder" | awk '{
      size = $1
      sub(/[Kk]$/, "", size)  # Remove 'K' suffix if present
@@ -86,6 +155,11 @@ for key in $keys; do
     }')
 
     folder_size_gb=$(printf "%.0f" "$folder_size")
+
+    if (( folder_size_gb < 1 )) && [[ "$FORCE_SMALL" != true ]]; then
+	echo "FATAL: existing datadir too small for backup (${folder_size_gb}G at $source_folder); use --force if intentional" >&2
+	exit 1
+    fi
     
     timestamp=$(date +'%Y-%m-%d-%H-%M-%S')
     target_file="rpc_$key-${timestamp}-${folder_size_gb}G.tar.zst"
@@ -119,5 +193,5 @@ done
 echo ""
 echo "=== Overall Size Summary ==="
 if [[ -f "$BASEPATH/show-size.sh" ]]; then
-    "$BASEPATH/show-size.sh" "$1" 2>&1
+    "$BASEPATH/show-size.sh" "$compose_name" 2>&1
 fi