Claude Agent
baf26b234f
find -type d skipped static dirs that are already offloaded (a symlink to /slowdisk is type l, not d) — so show-static-file-size.sh reported zero static for an offloaded node (e.g. bob on de-35), and backup-node.sh would drop them from the manifest (breaking re-offload on the next restore). Match dirs AND symlinks now (root-level entries too). show-static-file-size.sh also tags each static dir with its location: [OFFLOADED -> /slowdisk/...], [on-disk], or [BROKEN SYMLINK ...]. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
124 lines
5.0 KiB
Bash
Executable File
124 lines
5.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
BASEPATH="$(dirname "$0")"
|
|
source "$BASEPATH/volume-utils.sh"
|
|
backup_dir="/backup"
|
|
|
|
remote_target="$2"
|
|
if [[ -n "$remote_target" ]] && is_local_backup_url "$remote_target"; then
|
|
echo "Target URL points to this server, using local /backup instead of $remote_target"
|
|
remote_target=""
|
|
fi
|
|
|
|
if [[ -n "$remote_target" ]]; then
|
|
echo "upload backup via webdav to $remote_target"
|
|
else
|
|
if [ ! -d "$backup_dir" ]; then
|
|
echo "Error: /backup directory does not exist"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Function to generate metadata for a single volume
|
|
generate_volume_metadata() {
|
|
local volume_key=$1
|
|
local source_folder=$2
|
|
local metadata_file=$3
|
|
|
|
prefix="/var/lib/docker/volumes/rpc_$volume_key"
|
|
static_file_list="$BASEPATH/static-file-path-list.txt"
|
|
|
|
# Initialize metadata file
|
|
echo "Static file paths and sizes for volume: rpc_$volume_key" > "$metadata_file"
|
|
echo "Generated: $(date)" >> "$metadata_file"
|
|
echo "" >> "$metadata_file"
|
|
|
|
# Check each static file path
|
|
if [[ -f "$static_file_list" ]]; then
|
|
while IFS= read -r path; do
|
|
[[ -z "$path" ]] && continue
|
|
# Match rule (see static-file-path-list.txt): an entry with NO slash is root-level
|
|
# only (so `snapshots` won't catch postgres pg_logical/snapshots); an entry WITH a
|
|
# slash is a path-suffix, so any prefix matches (network-prefixed nitro freezers,
|
|
# nested l2geth geth/geth/...). Record the CONCRETE relative path so restore-volumes
|
|
# can recreate the exact static-file -> /slowdisk symlink.
|
|
# Match real dirs AND symlinks: an already-OFFLOADED static dir is a symlink to
|
|
# /slowdisk, which find -type d alone would skip (then it would be dropped from the
|
|
# manifest and the next restore would not re-offload it).
|
|
local matches=() m rel
|
|
if [[ "$path" == */* ]]; then
|
|
while IFS= read -r m; do matches+=("$m"); done < <(find "$prefix/_data" \( -type d -o -type l \) -path "*/$path" 2>/dev/null)
|
|
else
|
|
m="$prefix/_data/$path"; { [ -d "$m" ] || [ -L "$m" ]; } && matches+=("$m")
|
|
fi
|
|
for m in "${matches[@]}"; do
|
|
rel="${m#"$prefix/_data/"}"
|
|
size=$(du -sL "$m" 2>/dev/null | awk '{print $1}')
|
|
size_formatted=$(echo "$(( ${size:-0} * 1024 ))" | numfmt --to=iec --suffix=B --format="%.2f")
|
|
echo "$size_formatted $rel" >> "$metadata_file"
|
|
done
|
|
done < "$static_file_list"
|
|
fi
|
|
}
|
|
|
|
# Read the JSON input and extract the list of keys
|
|
keys=$(get_persistent_volume_keys "/root/rpc/$1.yml")
|
|
|
|
# Iterate over the list of keys
|
|
for key in $keys; do
|
|
echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data"
|
|
|
|
source_folder="/var/lib/docker/volumes/rpc_$key/_data"
|
|
folder_size=$(du -shL "$source_folder" | awk '{
|
|
size = $1
|
|
sub(/[Kk]$/, "", size) # Remove 'K' suffix if present
|
|
sub(/[Mm]$/, "", size) # Remove 'M' suffix if present
|
|
sub(/[Gg]$/, "", size) # Remove 'G' suffix if present
|
|
sub(/[Tt]$/, "", size) # Remove 'T' suffix if present
|
|
if ($1 ~ /[Kk]$/) {
|
|
size *= 0.001 # Convert kilobytes to gigabytes
|
|
} else if ($1 ~ /[Mm]$/) {
|
|
size *= 0.001 # Convert megabytes to gigabytes
|
|
} else if ($1 ~ /[Tt]$/) {
|
|
size *= 1000 # convert terabytes to gigabytes
|
|
}
|
|
print size
|
|
}')
|
|
|
|
folder_size_gb=$(printf "%.0f" "$folder_size")
|
|
|
|
timestamp=$(date +'%Y-%m-%d-%H-%M-%S')
|
|
target_file="rpc_$key-${timestamp}-${folder_size_gb}G.tar.zst"
|
|
metadata_file_name="rpc_$key-${timestamp}-${folder_size_gb}G.txt"
|
|
|
|
#echo "$target_file"
|
|
|
|
if [[ -n "$remote_target" ]]; then
|
|
# Upload volume archive
|
|
tar -cf - --dereference "$source_folder" | pv -pterb -s $(du -sb "$source_folder" | awk '{print $1}') | zstd | curl -X PUT --upload-file - "$remote_target/null/uploading-$target_file"
|
|
curl -X MOVE -H "Destination: /null/$target_file" "$remote_target/null/uploading-$target_file"
|
|
|
|
# Generate and upload metadata file
|
|
echo "Generating metadata for volume: rpc_$key"
|
|
temp_metadata="/tmp/$metadata_file_name"
|
|
generate_volume_metadata "$key" "$source_folder" "$temp_metadata"
|
|
curl -X PUT --upload-file "$temp_metadata" "$remote_target/null/$metadata_file_name"
|
|
rm -f "$temp_metadata"
|
|
else
|
|
# Create volume archive
|
|
tar -cf - --dereference "$source_folder" | pv -pterb -s $(du -sb "$source_folder" | awk '{print $1}') | zstd -o "/backup/uploading-$target_file"
|
|
mv "/backup/uploading-$target_file" "/backup/$target_file"
|
|
|
|
# Generate metadata file
|
|
echo "Generating metadata for volume: rpc_$key"
|
|
generate_volume_metadata "$key" "$source_folder" "/backup/$metadata_file_name"
|
|
fi
|
|
done
|
|
|
|
# Run show-size.sh to display overall summary
|
|
echo ""
|
|
echo "=== Overall Size Summary ==="
|
|
if [[ -f "$BASEPATH/show-size.sh" ]]; then
|
|
"$BASEPATH/show-size.sh" "$1" 2>&1
|
|
fi
|