restore-volumes: add /slowdisk absolute floor gate for static offload
Skip static-file offload when projected /slowdisk free space after restore would fall below SLOWDISK_FLOOR_GB (default 150 GB); warn after extract if the floor is breached. Restore never aborts — falls back to primary extract. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -20,6 +20,7 @@ set -- "${_pos[@]}"
|
||||
[ -f "$dir/.env" ] && source "$dir/.env"
|
||||
SLOWDISK="${SLOWDISK:-}"
|
||||
[ "$no_slowdisk_flag" = 1 ] && SLOWDISK=False
|
||||
SLOWDISK_FLOOR_GB="${SLOWDISK_FLOOR_GB:-150}"
|
||||
|
||||
remote_source="$2"
|
||||
if [[ -n "$remote_source" ]] && is_local_backup_url "$remote_source"; then
|
||||
@@ -46,15 +47,71 @@ fi
|
||||
# GATED on SLOWDISK (see top): offload runs only when SLOWDISK is the Python-templated boolean
|
||||
# "True" (set in the host .env on dedicated-extra-disk hosts) and the --no-slowdisk flag was
|
||||
# not passed. Case matters — the value comes through as capitalized "True"/"False". Safe
|
||||
# fallbacks (normal extract): SLOWDISK not True, /slowdisk missing, no manifest, no static paths.
|
||||
# fallbacks (normal extract): SLOWDISK not True, /slowdisk missing, no manifest, no static paths,
|
||||
# or slowdisk free space below SLOWDISK_FLOOR_GB after projected static payload.
|
||||
slowdisk_free_gb() {
|
||||
local free
|
||||
free=$(df -BG /slowdisk 2>/dev/null | awk 'NR==2 {gsub(/G/,"",$4); print $4}')
|
||||
[ -n "$free" ] && [[ "$free" =~ ^[0-9]+$ ]] || return 1
|
||||
echo "$free"
|
||||
}
|
||||
|
||||
# Best cheap size signal for incoming static payload: backup archive bytes (local stat or
|
||||
# remote Content-Length). Archive is compressed (.tar.zst); treat size as a LOWER bound and
|
||||
# assume ~2x when estimating uncompressed footprint on /slowdisk.
|
||||
backup_archive_bytes() {
|
||||
local newest_file=$1
|
||||
if [[ -n "$remote_source" ]]; then
|
||||
local cached="$backup_dir/$(basename "$newest_file")"
|
||||
if [[ -f "$cached" && ! -e "${cached}.aria2" ]]; then
|
||||
stat -c%s "$cached" 2>/dev/null && return 0
|
||||
fi
|
||||
curl --ipv4 -fsSI "${remote_source}${newest_file}" 2>/dev/null \
|
||||
| awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
|
||||
else
|
||||
stat -c%s "$newest_file" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# Pre-extract floor gate: return 0 if offload may proceed, 1 to fall back to primary extract.
|
||||
slowdisk_floor_allows_offload() {
|
||||
local newest_file=$1 archive_bytes need_gb free_gb floor_gb
|
||||
floor_gb="${SLOWDISK_FLOOR_GB:-150}"
|
||||
free_gb=$(slowdisk_free_gb) || return 0
|
||||
archive_bytes=$(backup_archive_bytes "$newest_file")
|
||||
if [[ -z "$archive_bytes" || ! "$archive_bytes" =~ ^[0-9]+$ ]]; then
|
||||
return 0
|
||||
fi
|
||||
# ~2x uncompressed factor (archive is compressed zstd tar)
|
||||
need_gb=$(( (archive_bytes * 2 + 1024*1024*1024 - 1) / (1024*1024*1024) ))
|
||||
if (( free_gb - need_gb < floor_gb )); then
|
||||
echo "slowdisk floor gate: skipping offload (free ${free_gb} GB, projected need ${need_gb} GB, floor ${floor_gb} GB) — extracting to primary"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
warn_slowdisk_floor_breach() {
|
||||
local free_gb floor_gb
|
||||
floor_gb="${SLOWDISK_FLOOR_GB:-150}"
|
||||
free_gb=$(slowdisk_free_gb) || return 0
|
||||
if (( free_gb < floor_gb )); then
|
||||
echo "WARNING: slowdisk floor breach after static offload (free ${free_gb} GB < floor ${floor_gb} GB) — OS headroom compromised"
|
||||
fi
|
||||
}
|
||||
|
||||
prep_static_offload() {
|
||||
local key=$1 meta=$2 data_dir=$3 rel target
|
||||
local key=$1 meta=$2 data_dir=$3 newest_file=$4 rel target
|
||||
case "$SLOWDISK" in
|
||||
True|true) ;; # offload enabled (Python "True"; also accept manual lowercase "true")
|
||||
*) echo " static offload disabled (SLOWDISK=$SLOWDISK) — normal extract"; return 0 ;;
|
||||
esac
|
||||
[ -d /slowdisk ] || { echo " /slowdisk absent — no static offload"; return 0; }
|
||||
[ -f "$meta" ] || { echo " no manifest ($meta) — no static offload"; return 0; }
|
||||
if ! slowdisk_floor_allows_offload "$newest_file"; then
|
||||
return 0
|
||||
fi
|
||||
static_offload_used=1
|
||||
# manifest data lines (after the 3-line header) are "<size> <relpath>"
|
||||
while IFS= read -r rel; do
|
||||
[ -z "$rel" ] && continue
|
||||
@@ -89,6 +146,7 @@ while IFS= read -r key; do
|
||||
fi
|
||||
meta_file="${newest_file%.tar.zst}.txt"
|
||||
echo "=== restoring rpc_$key <- $newest_file ==="
|
||||
static_offload_used=0
|
||||
|
||||
# 1) wipe live data AND any /slowdisk static targets for this key (no leak on re-restore)
|
||||
delete_slowdisk_targets_for_key "$key"
|
||||
@@ -108,7 +166,7 @@ while IFS= read -r key; do
|
||||
if [[ "$1" == *reth* ]]; then
|
||||
echo " reth node: static-file symlink offload disabled (reth broke whole-dir symlinks)"
|
||||
elif [ -n "$local_meta" ]; then
|
||||
prep_static_offload "$key" "$local_meta" "$data_dir"
|
||||
prep_static_offload "$key" "$local_meta" "$data_dir" "$newest_file"
|
||||
fi
|
||||
|
||||
# 3) extract THROUGH the pre-created symlinks (keep them, don't clobber)
|
||||
@@ -138,6 +196,9 @@ while IFS= read -r key; do
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [ "$static_offload_used" = 1 ]; then
|
||||
warn_slowdisk_floor_breach
|
||||
fi
|
||||
echo "Backup '$newest_file' restored"
|
||||
done <<< "$keys"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user