From fccf2792622acd3397ef5ab957a86d42c3174989 Mon Sep 17 00:00:00 2001 From: rob Date: Sat, 11 Jul 2026 01:50:08 +0000 Subject: [PATCH] slowdisk offload: fix EXDEV - extract statics via tar --transform, not through symlinks tar cannot create files through a directory symlink whose target is on another device: open() fails with 'Invalid cross-device link' even under --keep-directory-symlink (reproduced, GNU tar 1.34 and 1.35). This killed the katana->uk-4 provision (place-provision-katana-EU-NorthWest-p1) and means the slowdisk EXTRACT path never worked on hosts with a separate /slowdisk device. Rewrite member paths with --transform so tar writes directly into /slowdisk/rpc___data_; the symlink stays for the container's runtime view. Also aligns restore-volumes' nested-path target naming to the double-underscore convention clone-backup already uses. Co-Authored-By: Claude Fable 5 --- clone-backup.sh | 10 ++++++++-- restore-volumes.sh | 16 ++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/clone-backup.sh b/clone-backup.sh index 8f97fe6a..cefd1c7c 100755 --- a/clone-backup.sh +++ b/clone-backup.sh @@ -224,6 +224,7 @@ setup_slowdisk_structure() { return 1 fi + SLOWDISK_TRANSFORMS="" echo "Setting up SLOWDISK structure for $key" echo " Static size: $((static_size_kb / 1024))MB" echo " Static paths: ${#static_paths[@]}" @@ -253,6 +254,11 @@ setup_slowdisk_structure() { local slowdisk_path="${slowdisk_base}/rpc_${key}__data_${path//\//__}" local target_path="${volume_path}/${path}" local target_dir=$(dirname "$target_path") + # tar cannot extract THROUGH a symlink to another device (EXDEV + # "Invalid cross-device link", reproduced GNU tar 1.34+1.35 2026-07-11 - + # killed the katana->uk-4 provision). Collect --transform rewrites so tar + # writes directly into /slowdisk; the symlink remains for runtime. + SLOWDISK_TRANSFORMS+=" --transform='s|^\\(\\./\\)\\?var/lib/docker/volumes/rpc_${key}/_data/${path}|slowdisk/rpc_${key}__data_${path//\//__}|'" # Create parent directories in volume $SSH_CMD "$DEST_HOST" "mkdir -p '$target_dir'" @@ -327,7 +333,7 @@ transfer_backup() { # Use --skip-old-files to avoid overwriting existing symlinks/directories # and --keep-directory-symlink so tar extracts THROUGH the pre-created dir symlinks # instead of trying to mkdir over them (which fails with 'Cannot mkdir: File exists') - tar_extract_opts="-xf - -C / --skip-old-files --keep-directory-symlink" + tar_extract_opts="-xf - -C / --skip-old-files --keep-directory-symlink${SLOWDISK_TRANSFORMS}" echo "SLOWDISK structure ready, will extract respecting symlinks" else echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" @@ -513,7 +519,7 @@ transfer_backup_ssh() { if setup_slowdisk_structure "$key" "$static_size_kb" "${static_paths[@]}"; then # and --keep-directory-symlink so tar extracts THROUGH the pre-created dir symlinks # instead of trying to mkdir over them (which fails with 'Cannot mkdir: File exists') - tar_extract_opts="-xf - -C / --skip-old-files --keep-directory-symlink" + tar_extract_opts="-xf - -C / --skip-old-files --keep-directory-symlink${SLOWDISK_TRANSFORMS}" echo "SLOWDISK structure ready, will extract respecting symlinks" else echo "Warning: Failed to setup SLOWDISK structure, falling back to normal extraction" diff --git a/restore-volumes.sh b/restore-volumes.sh index 90eb2147..739d862c 100755 --- a/restore-volumes.sh +++ b/restore-volumes.sh @@ -137,10 +137,17 @@ prep_static_offload() { [ -z "$rel" ] && continue rel="${rel#/}" case "$rel" in *..*) echo " skip unsafe static path '$rel'"; continue;; esac - target="/slowdisk/rpc_${key}__data_${rel//\//_}" + target="/slowdisk/rpc_${key}__data_${rel//\//__}" echo " offload static '$rel' -> $target" mkdir -p "$target" "$data_dir/$(dirname "$rel")" || { echo " WARN: mkdir failed for '$rel', skipping"; continue; } ln -sfn "$target" "$data_dir/$rel" + # tar CANNOT extract through a symlink whose target is on another device - + # open() fails with EXDEV "Invalid cross-device link" even with + # --keep-directory-symlink (reproduced GNU tar 1.34 + 1.35, 2026-07-11; + # killed the katana->uk-4 provision). Rewrite the member paths so tar + # writes DIRECTLY into the /slowdisk target; the symlink stays for the + # container's runtime view. + static_transform_args+=(--transform "s|^\\(\\./\\)\\?var/lib/docker/volumes/rpc_${key}/_data/${rel}|slowdisk/rpc_${key}__data_${rel//\//__}|") done < <(awk 'NR>3 && NF>=2 {print $NF}' "$meta") } @@ -167,6 +174,7 @@ while IFS= read -r key; do meta_file="${newest_file%.tar.zst}.txt" echo "=== restoring rpc_$key <- $newest_file ===" static_offload_used=0 + static_transform_args=() # 1) wipe live data AND any /slowdisk static targets for this key (no leak on re-restore) delete_slowdisk_targets_for_key "$key" @@ -193,7 +201,7 @@ while IFS= read -r key; do if [[ -n "$remote_source" ]]; then if [ ! -d "$backup_dir" ]; then echo "No /backup cache: streaming + extracting $newest_file directly" - curl --ipv4 -# "${remote_source}${newest_file}" | zstd -d | tar -xf - --keep-directory-symlink -C / + curl --ipv4 -# "${remote_source}${newest_file}" | zstd -d | tar -xf - --keep-directory-symlink "${static_transform_args[@]}" -C / if [ $? -ne 0 ]; then echo "Error processing $newest_file" >&2 exit 1 @@ -203,14 +211,14 @@ while IFS= read -r key; do if [ ! -e "$backup_file" ] || [ -e "${backup_file}.aria2" ]; then aria2c -c -Z -x8 -j8 -s8 -d "$backup_dir" "${remote_source}${newest_file}" fi - tar -I zstd -xf "$backup_file" --keep-directory-symlink -C / + tar -I zstd -xf "$backup_file" --keep-directory-symlink "${static_transform_args[@]}" -C / if [ $? -ne 0 ]; then echo "Error processing $newest_file" >&2 exit 1 fi fi else - tar -I zstd -xf "$newest_file" --keep-directory-symlink -C / + tar -I zstd -xf "$newest_file" --keep-directory-symlink "${static_transform_args[@]}" -C / if [ $? -ne 0 ]; then echo "Error processing $newest_file" >&2 exit 1