diff --git a/backup-node.sh b/backup-node.sh index 5485c146..4a4ac19c 100755 --- a/backup-node.sh +++ b/backup-node.sh @@ -1,10 +1,73 @@ #!/bin/bash +# +# Backup persistent compose volumes to /backup or WebDAV. +# +# Usage: +# ./backup-node.sh [--hot] [--force] [webdav-url] +# +# Lifecycle (default, omitted with --hot): +# stop services -> fence compose out of COMPOSE_FILE -> tar+zstd -> unfence+start (EXIT trap). +# Hot backups skip stop/fence; leveldb/pebble live tars are NOT restore-trustworthy. +# +# --force Allow backing up an existing but very small datadir (<1GB reported size). +# Does NOT bypass a missing volume path (always fatal). BASEPATH="$(dirname "$0")" source "$BASEPATH/volume-utils.sh" backup_dir="/backup" -remote_target="$2" +HOT_BACKUP=false +FORCE_SMALL=false +_pos=() +for _a in "$@"; do + case "$_a" in + --hot) HOT_BACKUP=true ;; + --force) FORCE_SMALL=true ;; + *) _pos+=("$_a") ;; + esac +done +set -- "${_pos[@]}" + +if [ -z "${1:-}" ] || [ ! -f "/root/rpc/$1.yml" ]; then + echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist." + exit 1 +fi + +compose_name="$1" +remote_target="${2:-}" +lifecycle_active=false +fenced=false +stopped=false + +backup_lifecycle_cleanup() { + local rc=$? + if [[ "$lifecycle_active" != true ]]; then + exit "$rc" + fi + if [[ "$fenced" == true ]]; then + unfence_compose_in_env "$compose_name" || true + fenced=false + fi + if [[ "$stopped" == true ]]; then + "$BASEPATH/start.sh" "$compose_name" || true + stopped=false + fi + exit "$rc" +} + +if [[ "$HOT_BACKUP" == true ]]; then + echo "WARNING: --hot backup skips stop/fence; live leveldb/pebble archives may not restore cleanly" +else + trap backup_lifecycle_cleanup EXIT + lifecycle_active=true + echo "Stopping services for $compose_name before backup..." + "$BASEPATH/stop.sh" "$compose_name" + stopped=true + if fence_compose_in_env "$compose_name"; then + fenced=true + fi +fi + if [[ -n "$remote_target" ]] && is_local_backup_url "$remote_target"; then echo "Target URL points to this server, using local /backup instead of $remote_target" remote_target="" @@ -62,13 +125,19 @@ generate_volume_metadata() { } # Read the JSON input and extract the list of keys -keys=$(get_persistent_volume_keys "/root/rpc/$1.yml") +keys=$(get_persistent_volume_keys "/root/rpc/$compose_name.yml") # Iterate over the list of keys for key in $keys; do echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data" source_folder="/var/lib/docker/volumes/rpc_$key/_data" + + if [[ ! -d "$source_folder" ]]; then + echo "FATAL: volume data directory missing, refusing backup: $source_folder" >&2 + exit 1 + fi + folder_size=$(du -shL "$source_folder" | awk '{ size = $1 sub(/[Kk]$/, "", size) # Remove 'K' suffix if present @@ -86,6 +155,11 @@ for key in $keys; do }') folder_size_gb=$(printf "%.0f" "$folder_size") + + if (( folder_size_gb < 1 )) && [[ "$FORCE_SMALL" != true ]]; then + echo "FATAL: existing datadir too small for backup (${folder_size_gb}G at $source_folder); use --force if intentional" >&2 + exit 1 + fi timestamp=$(date +'%Y-%m-%d-%H-%M-%S') target_file="rpc_$key-${timestamp}-${folder_size_gb}G.tar.zst" @@ -119,5 +193,5 @@ done echo "" echo "=== Overall Size Summary ===" if [[ -f "$BASEPATH/show-size.sh" ]]; then - "$BASEPATH/show-size.sh" "$1" 2>&1 + "$BASEPATH/show-size.sh" "$compose_name" 2>&1 fi diff --git a/cleanup-backups.sh b/cleanup-backups.sh index 113d2b44..b776aeff 100755 --- a/cleanup-backups.sh +++ b/cleanup-backups.sh @@ -14,6 +14,8 @@ set -euo pipefail BASEPATH="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=volume-utils.sh +source "$BASEPATH/volume-utils.sh" BACKUP_DIR="${BACKUP_DIR:-/backup}" TRASH_DIR="${TRASH_DIR:-$BACKUP_DIR/trash}" KEEP_COUNT="${KEEP_COUNT:-3}" @@ -163,6 +165,20 @@ cleanup_volume_backups() { continue fi + if ! is_restorable_backup_archive "$file"; then + local near_size + near_size=$(backup_archive_size_bytes "$file") + local near_meta + near_meta=$(metadata_for_archive "$file") + add_trashed_bytes "$file" + trash_file "$file" "near-empty artifact (${near_size} bytes, excluded from KEEP_COUNT)" + if [[ -f "$near_meta" ]]; then + add_trashed_bytes "$near_meta" + trash_file "$near_meta" "metadata for near-empty artifact" + fi + continue + fi + volume_files["$volume_key"]+="$file"$'\n' done < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'rpc_*-*.tar.zst' -print0 2>/dev/null) diff --git a/clone-backup.sh b/clone-backup.sh index 9bea0861..34b9f002 100755 --- a/clone-backup.sh +++ b/clone-backup.sh @@ -294,11 +294,12 @@ transfer_backup() { local key=$1 local volume_name="rpc_$key" - # Find the newest backup file - local backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1) + # Find the newest restorable backup file (skip near-empty artifacts) + local backup_file + backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name") if [[ -z "$backup_file" ]] || [[ ! -f "$backup_file" ]]; then - echo "Warning: No backup file found for $volume_name, skipping" + echo "Warning: No restorable backup file found for $volume_name, skipping" return 1 fi @@ -482,11 +483,12 @@ transfer_backup_ssh() { local key=$1 local volume_name="rpc_$key" - # Find the newest backup file - local backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1) + # Find the newest restorable backup file (skip near-empty artifacts) + local backup_file + backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name") if [[ -z "$backup_file" ]] || [[ ! -f "$backup_file" ]]; then - echo "Warning: No backup file found for $volume_name, skipping" + echo "Warning: No restorable backup file found for $volume_name, skipping" return 1 fi diff --git a/list-restorable.sh b/list-restorable.sh index 31908976..55483d49 100755 --- a/list-restorable.sh +++ b/list-restorable.sh @@ -12,6 +12,7 @@ # ./list-restorable.sh --all # --all can be in any position dir="$(dirname "$0")" +source "$dir/volume-utils.sh" registry_file="${dir}/compose_registry.json" backup_dir="/backup" @@ -122,7 +123,7 @@ while IFS= read -r entry; do while IFS= read -r volume; do volume_name="rpc_${volume}" # Look for backup files matching pattern: rpc_${volume}-[0-9]*G.tar.zst - backup_file=$(ls -1 "$backup_dir"/"${volume_name}"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1) + backup_file=$(select_newest_local_backup "$backup_dir" "$volume_name") if [ -z "$backup_file" ]; then all_backups_exist=false diff --git a/restore-volumes.sh b/restore-volumes.sh index 4a7d6aca..15e86540 100755 --- a/restore-volumes.sh +++ b/restore-volumes.sh @@ -135,9 +135,9 @@ while IFS= read -r key; do declare newest_file if [[ -n "$remote_source" ]]; then - newest_file=$($dir/list-backups.sh "$remote_source" | grep "rpc_$key-20" | sort | tail -n 1) + newest_file=$($dir/list-backups.sh "$remote_source" | select_newest_remote_backup_from_list "$remote_source" "rpc_$key") else - newest_file=$(ls -1 "$backup_dir"/"rpc_$key"-[0-9]*G.tar.zst 2>/dev/null | sort | tail -n 1) + newest_file=$(select_newest_local_backup "$backup_dir" "rpc_$key") fi if [ -z "$newest_file" ]; then diff --git a/volume-utils.sh b/volume-utils.sh index 089c0aa5..a201b9f6 100755 --- a/volume-utils.sh +++ b/volume-utils.sh @@ -3,6 +3,180 @@ # Shared helpers for compose volume operations. # Ephemeral volumes (init-container config) are excluded from backup/restore/size. +# Minimum on-disk bytes for a backup archive to be considered restorable (near-empty +# zstd headers from purged/missing volumes must not win sort|tail newest selection). +MIN_RESTORABLE_BACKUP_BYTES=$((1024 * 1024)) + +RPC_ENV_FILE="/root/rpc/.env" + +backup_archive_size_bytes() { + local file=$1 + stat -c%s "$file" 2>/dev/null || echo 0 +} + +is_restorable_backup_archive() { + local file=$1 + local size + [[ -f "$file" ]] || return 1 + size=$(backup_archive_size_bytes "$file") + [[ "$size" =~ ^[0-9]+$ ]] && (( size >= MIN_RESTORABLE_BACKUP_BYTES )) +} + +# Pick the lexicographically newest backup that is not near-empty (<1MB). Prints one +# skip line per rejected artifact; returns empty when none qualify. +select_newest_local_backup() { + local backup_dir=$1 + local volume_name=$2 + local pattern="${backup_dir}/${volume_name}"-[0-9]*G.tar.zst + local newest="" f size + + shopt -s nullglob + local files=( $pattern ) + shopt -u nullglob + if (( ${#files[@]} == 0 )); then + return 0 + fi + + IFS=$'\n' files=( $(printf '%s\n' "${files[@]}" | sort) ) + unset IFS + + for f in "${files[@]}"; do + size=$(backup_archive_size_bytes "$f") + if [[ ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then + echo "skip near-empty backup artifact: $(basename "$f") (${size:-0} bytes)" >&2 + continue + fi + newest="$f" + done + + if [[ -n "$newest" ]]; then + echo "$newest" + fi +} + +remote_backup_archive_size_bytes() { + local remote_source=$1 + local archive_name=$2 + curl --ipv4 -fsSI "${remote_source}${archive_name}" 2>/dev/null \ + | awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}' +} + +# Newest restorable backup filename (not path) from a remote WebDAV listing on stdin. +select_newest_remote_backup_from_list() { + local remote_source=$1 + local volume_name=$2 + local newest="" f size + + while IFS= read -r f; do + [[ -z "$f" ]] && continue + [[ "$f" == "${volume_name}-"* ]] || continue + [[ "$f" == *.tar.zst ]] || continue + size=$(remote_backup_archive_size_bytes "$remote_source" "$f") + if [[ -z "$size" || ! "$size" =~ ^[0-9]+$ ]] || (( size < MIN_RESTORABLE_BACKUP_BYTES )); then + echo "skip near-empty backup artifact: $f (${size:-unknown} bytes)" >&2 + continue + fi + newest="$f" + done < <(sort) + + if [[ -n "$newest" ]]; then + echo "$newest" + fi +} + +compose_fence_marker_path() { + local compose_name=$1 + local env_dir + env_dir=$(dirname "$RPC_ENV_FILE") + echo "${env_dir}/.env.fence-$(echo "$compose_name" | tr '/' '_')" +} + +# Remove a compose yml segment from COMPOSE_FILE; store it in a sidecar marker for restore. +fence_compose_in_env() { + local compose_name=$1 + local yml_segment="${compose_name}.yml" + local env_file="$RPC_ENV_FILE" + local marker + marker=$(compose_fence_marker_path "$compose_name") + + [[ -f "$env_file" ]] || { echo "WARN: $env_file missing, cannot fence $compose_name" >&2; return 1; } + [[ -f "$marker" ]] && { echo "WARN: fence marker already exists: $marker" >&2; return 1; } + + local line compose_file new_compose removed=0 + line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1) + [[ -n "$line" ]] || { echo "WARN: COMPOSE_FILE not found in $env_file" >&2; return 1; } + + compose_file="${line#COMPOSE_FILE=}" + compose_file="${compose_file#\"}" + compose_file="${compose_file%\"}" + + local part new_parts=() + IFS=':' read -ra parts <<< "$compose_file" + for part in "${parts[@]}"; do + if [[ "$part" == "$yml_segment" ]]; then + if ! printf '%s\n' "$part" > "$marker"; then + echo "WARN: failed to write fence marker $marker" >&2 + return 1 + fi + removed=1 + else + new_parts+=("$part") + fi + done + unset IFS + + if (( removed == 0 )); then + echo "WARN: $yml_segment not present in COMPOSE_FILE (already fenced?)" >&2 + return 1 + fi + + new_compose=$(IFS=':'; echo "${new_parts[*]}") + if grep -q '^COMPOSE_FILE=' "$env_file"; then + sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${new_compose}|" "$env_file" + else + echo "COMPOSE_FILE=${new_compose}" >> "$env_file" + fi + echo "Fenced $yml_segment from COMPOSE_FILE (marker: $marker)" +} + +# Restore a fenced compose segment before service-scoped start (start needs the entry). +unfence_compose_in_env() { + local compose_name=$1 + local yml_segment="${compose_name}.yml" + local env_file="$RPC_ENV_FILE" + local marker + marker=$(compose_fence_marker_path "$compose_name") + + [[ -f "$marker" ]] || return 0 + + local segment line compose_file + segment=$(cat "$marker") + [[ -n "$segment" ]] || { rm -f "$marker"; return 0; } + + if [[ -f "$env_file" ]]; then + line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1) + compose_file="${line#COMPOSE_FILE=}" + compose_file="${compose_file#\"}" + compose_file="${compose_file%\"}" + case ":$compose_file:" in + *:"$segment":*) ;; + *) + if [[ -n "$compose_file" ]]; then + compose_file="${compose_file}:${segment}" + else + compose_file="$segment" + fi + sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${compose_file}|" "$env_file" + ;; + esac + else + echo "COMPOSE_FILE=${segment}" > "$env_file" + fi + + rm -f "$marker" + echo "Unfenced $yml_segment into COMPOSE_FILE" +} + is_ephemeral_volume_key() { local key=$1 local compose_file=$2