Claude Agent
58adc446e2
backup-node.sh now stops services, fences COMPOSE_FILE during backup, and restores via EXIT trap; consumers skip <1MB .tar.zst artifacts so purged- volume junk cannot shadow real backups. Co-authored-by: Cursor <cursoragent@cursor.com>
208 lines
8.1 KiB
Bash
Executable File
208 lines
8.1 KiB
Bash
Executable File
#!/bin/bash
|
|
dir="$(dirname "$0")"
|
|
source "$dir/volume-utils.sh"
|
|
|
|
# Pull out the --no-slowdisk flag (position-independent); keep the positional args
|
|
# ($1 = compose name, $2 = optional remote source) intact for the rest of the script.
|
|
no_slowdisk_flag=0
|
|
_pos=()
|
|
for _a in "$@"; do
|
|
case "$_a" in
|
|
--no-slowdisk) no_slowdisk_flag=1 ;;
|
|
*) _pos+=("$_a") ;;
|
|
esac
|
|
done
|
|
set -- "${_pos[@]}"
|
|
|
|
# Static-file offload gate. Hosts with a real dedicated extra disk mounted at /slowdisk set
|
|
# SLOWDISK=True in their .env (Python-templated boolean, capitalized); that enables the
|
|
# offload. The --no-slowdisk flag disables it even when SLOWDISK=True (e.g. extra disk full).
|
|
[ -f "$dir/.env" ] && source "$dir/.env"
|
|
SLOWDISK="${SLOWDISK:-}"
|
|
[ "$no_slowdisk_flag" = 1 ] && SLOWDISK=False
|
|
SLOWDISK_FLOOR_GB="${SLOWDISK_FLOOR_GB:-150}"
|
|
|
|
remote_source="$2"
|
|
if [[ -n "$remote_source" ]] && is_local_backup_url "$remote_source"; then
|
|
echo "Source URL points to this server, using local /backup instead of $remote_source"
|
|
remote_source=""
|
|
fi
|
|
|
|
# Path to the backup directory
|
|
backup_dir="/backup"
|
|
|
|
# Path to the volume directory
|
|
volume_dir="/var/lib/docker/volumes"
|
|
|
|
if [ ! -d "$volume_dir" ]; then
|
|
echo "Error: /var/lib/docker/volumes directory does not exist"
|
|
exit 1
|
|
fi
|
|
|
|
# Pre-create static-file -> /slowdisk symlinks from a backup's ".txt" manifest, so the
|
|
# immutable "ancient"/freezer dirs land on the (SSD) /slowdisk during extraction while the
|
|
# hot/dynamic state stays on the primary disk. tar then extracts THROUGH the symlinks via
|
|
# --keep-directory-symlink (it keeps the dir-symlinks instead of clobbering them).
|
|
# Target naming matches delete-volumes.sh / delete_slowdisk_targets_for_key cleanup.
|
|
# GATED on SLOWDISK (see top): offload runs only when SLOWDISK is the Python-templated boolean
|
|
# "True" (set in the host .env on dedicated-extra-disk hosts) and the --no-slowdisk flag was
|
|
# not passed. Case matters — the value comes through as capitalized "True"/"False". Safe
|
|
# fallbacks (normal extract): SLOWDISK not True, /slowdisk missing, no manifest, no static paths,
|
|
# or slowdisk free space below SLOWDISK_FLOOR_GB after projected static payload.
|
|
slowdisk_free_gb() {
|
|
local free
|
|
free=$(df -BG /slowdisk 2>/dev/null | awk 'NR==2 {gsub(/G/,"",$4); print $4}')
|
|
[ -n "$free" ] && [[ "$free" =~ ^[0-9]+$ ]] || return 1
|
|
echo "$free"
|
|
}
|
|
|
|
# Best cheap size signal for incoming static payload: backup archive bytes (local stat or
|
|
# remote Content-Length). Archive is compressed (.tar.zst); treat size as a LOWER bound and
|
|
# assume ~2x when estimating uncompressed footprint on /slowdisk.
|
|
backup_archive_bytes() {
|
|
local newest_file=$1
|
|
if [[ -n "$remote_source" ]]; then
|
|
local cached="$backup_dir/$(basename "$newest_file")"
|
|
if [[ -f "$cached" && ! -e "${cached}.aria2" ]]; then
|
|
stat -c%s "$cached" 2>/dev/null && return 0
|
|
fi
|
|
curl --ipv4 -fsSI "${remote_source}${newest_file}" 2>/dev/null \
|
|
| awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
|
|
else
|
|
stat -c%s "$newest_file" 2>/dev/null
|
|
fi
|
|
}
|
|
|
|
# Pre-extract floor gate: return 0 if offload may proceed, 1 to fall back to primary extract.
|
|
slowdisk_floor_allows_offload() {
|
|
local newest_file=$1 archive_bytes need_gb free_gb floor_gb
|
|
floor_gb="${SLOWDISK_FLOOR_GB:-150}"
|
|
free_gb=$(slowdisk_free_gb) || return 0
|
|
archive_bytes=$(backup_archive_bytes "$newest_file")
|
|
if [[ -z "$archive_bytes" || ! "$archive_bytes" =~ ^[0-9]+$ ]]; then
|
|
return 0
|
|
fi
|
|
# ~2x uncompressed factor (archive is compressed zstd tar)
|
|
need_gb=$(( (archive_bytes * 2 + 1024*1024*1024 - 1) / (1024*1024*1024) ))
|
|
if (( free_gb - need_gb < floor_gb )); then
|
|
echo "slowdisk floor gate: skipping offload (free ${free_gb} GB, projected need ${need_gb} GB, floor ${floor_gb} GB) — extracting to primary"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
warn_slowdisk_floor_breach() {
|
|
local free_gb floor_gb
|
|
floor_gb="${SLOWDISK_FLOOR_GB:-150}"
|
|
free_gb=$(slowdisk_free_gb) || return 0
|
|
if (( free_gb < floor_gb )); then
|
|
echo "WARNING: slowdisk floor breach after static offload (free ${free_gb} GB < floor ${floor_gb} GB) — OS headroom compromised"
|
|
fi
|
|
}
|
|
|
|
prep_static_offload() {
|
|
local key=$1 meta=$2 data_dir=$3 newest_file=$4 rel target
|
|
case "$SLOWDISK" in
|
|
True|true) ;; # offload enabled (Python "True"; also accept manual lowercase "true")
|
|
*) echo " static offload disabled (SLOWDISK=$SLOWDISK) — normal extract"; return 0 ;;
|
|
esac
|
|
[ -d /slowdisk ] || { echo " /slowdisk absent — no static offload"; return 0; }
|
|
[ -f "$meta" ] || { echo " no manifest ($meta) — no static offload"; return 0; }
|
|
if ! slowdisk_floor_allows_offload "$newest_file"; then
|
|
return 0
|
|
fi
|
|
static_offload_used=1
|
|
# manifest data lines (after the 3-line header) are "<size> <relpath>"
|
|
while IFS= read -r rel; do
|
|
[ -z "$rel" ] && continue
|
|
rel="${rel#/}"
|
|
case "$rel" in *..*) echo " skip unsafe static path '$rel'"; continue;; esac
|
|
target="/slowdisk/rpc_${key}__data_${rel//\//_}"
|
|
echo " offload static '$rel' -> $target"
|
|
mkdir -p "$target" "$data_dir/$(dirname "$rel")" || { echo " WARN: mkdir failed for '$rel', skipping"; continue; }
|
|
ln -sfn "$target" "$data_dir/$rel"
|
|
done < <(awk 'NR>3 && NF>=2 {print $NF}' "$meta")
|
|
}
|
|
|
|
# Read the JSON input and extract the list of keys
|
|
keys=$(get_persistent_volume_keys "$dir/$1.yml" | grep -E '^[0-9a-z]')
|
|
|
|
echo "$keys"
|
|
|
|
while IFS= read -r key; do
|
|
[ -z "$key" ] && continue
|
|
data_dir="$volume_dir/rpc_$key/_data"
|
|
declare newest_file
|
|
|
|
if [[ -n "$remote_source" ]]; then
|
|
newest_file=$($dir/list-backups.sh "$remote_source" | select_newest_remote_backup_from_list "$remote_source" "rpc_$key")
|
|
else
|
|
newest_file=$(select_newest_local_backup "$backup_dir" "rpc_$key")
|
|
fi
|
|
|
|
if [ -z "$newest_file" ]; then
|
|
echo "Error: No backup found for volume 'rpc_$key'"
|
|
exit 1
|
|
fi
|
|
meta_file="${newest_file%.tar.zst}.txt"
|
|
echo "=== restoring rpc_$key <- $newest_file ==="
|
|
static_offload_used=0
|
|
|
|
# 1) wipe live data AND any /slowdisk static targets for this key (no leak on re-restore)
|
|
delete_slowdisk_targets_for_key "$key"
|
|
[ -d "$data_dir" ] && rm -rf "$data_dir"/*
|
|
mkdir -p "$data_dir"
|
|
|
|
# 2) obtain the manifest (fetch the sidecar .txt for remote restores) and, unless this is
|
|
# a reth node (reth dropped whole-dir static-file symlinks), pre-create the offload.
|
|
local_meta="$meta_file"
|
|
if [[ -n "$remote_source" ]]; then
|
|
local_meta="$backup_dir/$(basename "$meta_file")"
|
|
[ -d "$backup_dir" ] || local_meta="/tmp/$(basename "$meta_file")"
|
|
if [ ! -f "$local_meta" ]; then
|
|
curl --ipv4 -fsS "${remote_source}${meta_file}" -o "$local_meta" 2>/dev/null || local_meta=""
|
|
fi
|
|
fi
|
|
if [[ "$1" == *reth* ]]; then
|
|
echo " reth node: static-file symlink offload disabled (reth broke whole-dir symlinks)"
|
|
elif [ -n "$local_meta" ]; then
|
|
prep_static_offload "$key" "$local_meta" "$data_dir" "$newest_file"
|
|
fi
|
|
|
|
# 3) extract THROUGH the pre-created symlinks (keep them, don't clobber)
|
|
if [[ -n "$remote_source" ]]; then
|
|
if [ ! -d "$backup_dir" ]; then
|
|
echo "No /backup cache: streaming + extracting $newest_file directly"
|
|
curl --ipv4 -# "${remote_source}${newest_file}" | zstd -d | tar -xf - --keep-directory-symlink -C /
|
|
if [ $? -ne 0 ]; then
|
|
echo "Error processing $newest_file" >&2
|
|
exit 1
|
|
fi
|
|
else
|
|
backup_file="$backup_dir/$(basename "$newest_file")"
|
|
if [ ! -e "$backup_file" ] || [ -e "${backup_file}.aria2" ]; then
|
|
aria2c -c -Z -x8 -j8 -s8 -d "$backup_dir" "${remote_source}${newest_file}"
|
|
fi
|
|
tar -I zstd -xf "$backup_file" --keep-directory-symlink -C /
|
|
if [ $? -ne 0 ]; then
|
|
echo "Error processing $newest_file" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
else
|
|
tar -I zstd -xf "$newest_file" --keep-directory-symlink -C /
|
|
if [ $? -ne 0 ]; then
|
|
echo "Error processing $newest_file" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [ "$static_offload_used" = 1 ]; then
|
|
warn_slowdisk_floor_breach
|
|
fi
|
|
echo "Backup '$newest_file' restored"
|
|
done <<< "$keys"
|
|
|
|
"$dir/delete-node-keys.sh" "$1"
|
|
|
|
echo "node $1 restored."
|