better clone speed
This commit is contained in:
goldsquid
270
clone-node.sh
270
clone-node.sh
@@ -1,21 +1,271 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Fixed version that handles missing netstat
|
||||
|
||||
if [[ -n $2 ]]; then
|
||||
echo "clone volumes via ssh to $2"
|
||||
DEST_HOST="$2.stakesquid.eu"
|
||||
echo "Setting up optimized transfer to $DEST_HOST"
|
||||
else
|
||||
echo "Error: No destination provided"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Read the JSON input and extract the list of keys
|
||||
keys=$(cat /root/rpc/$1.yml | yaml2json - | jq '.volumes' | jq -r 'keys[]')
|
||||
# Configuration
|
||||
BASE_PORT=9000
|
||||
|
||||
# Iterate over the list of keys
|
||||
for key in $keys; do
|
||||
echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data"
|
||||
# Setup SSH multiplexing
|
||||
setup_ssh_multiplex() {
|
||||
echo "Setting up SSH control connection..."
|
||||
ssh -nNf -o ControlMaster=yes \
|
||||
-o ControlPath=/tmp/ssh-mux-%h-%p-%r \
|
||||
-o ControlPersist=600 \
|
||||
-o Compression=no \
|
||||
"$DEST_HOST" 2>/dev/null
|
||||
|
||||
source_folder="/var/lib/docker/volumes/rpc_$key/_data"
|
||||
if [[ -n $2 ]]; then
|
||||
tar -cf - --dereference "$source_folder" | pv -pterb -s $(du -sb "$source_folder" | awk '{print $1}') | zstd | ssh -o Compression=no -c=chacha20-poly1305@openssh.com "$2.stakesquid.eu" "zstd -d | tar -xf - -C /"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "SSH control connection established"
|
||||
export SSH_CMD="ssh -o ControlPath=/tmp/ssh-mux-%h-%p-%r"
|
||||
else
|
||||
echo "Failed to setup SSH multiplexing, using direct SSH"
|
||||
export SSH_CMD="ssh"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Check if port is listening using various methods
|
||||
check_port_listening() {
|
||||
local port=$1
|
||||
|
||||
# Try different methods to check if port is listening
|
||||
$SSH_CMD "$DEST_HOST" "
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -tln | grep -q ':$port '
|
||||
elif command -v netstat >/dev/null 2>&1; then
|
||||
netstat -tln | grep -q ':$port '
|
||||
elif command -v lsof >/dev/null 2>&1; then
|
||||
lsof -i :$port >/dev/null 2>&1
|
||||
else
|
||||
# If no tools available, just try to connect to the port
|
||||
timeout 1 bash -c 'cat < /dev/null > /dev/tcp/localhost/$port' 2>/dev/null
|
||||
fi
|
||||
"
|
||||
return $?
|
||||
}
|
||||
|
||||
# Transfer using screen method with better error handling
|
||||
transfer_volume() {
|
||||
local key=$1
|
||||
local source_folder="/var/lib/docker/volumes/rpc_${key}/_data"
|
||||
|
||||
if [[ ! -d "$source_folder" ]]; then
|
||||
echo "Warning: $source_folder does not exist, skipping"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local folder_size=$(du -sb "$source_folder" 2>/dev/null | awk '{print $1}')
|
||||
local port=$BASE_PORT
|
||||
|
||||
echo "Transferring volume $key (size: $((folder_size / 1048576))MB)"
|
||||
|
||||
# Kill any existing listener on this port
|
||||
$SSH_CMD "$DEST_HOST" "
|
||||
# Try to kill any existing nc on this port
|
||||
lsof -i :$port 2>/dev/null | grep LISTEN | awk '{print \$2}' | xargs -r kill 2>/dev/null
|
||||
# Kill any existing screen session for this transfer
|
||||
screen -S transfer_${key} -X quit 2>/dev/null
|
||||
# Clean up old files
|
||||
rm -f /tmp/transfer_${key}.* 2>/dev/null
|
||||
"
|
||||
|
||||
# Check if screen is available
|
||||
if $SSH_CMD "$DEST_HOST" "which screen" >/dev/null 2>&1; then
|
||||
echo "Starting screen listener on port $port..."
|
||||
|
||||
# Start listener in screen session with proper escaping
|
||||
$SSH_CMD "$DEST_HOST" "
|
||||
screen -dmS transfer_${key} bash -c '
|
||||
nc -l -p $port | zstd -d | tar -xf - -C / 2>/tmp/transfer_${key}.err
|
||||
echo \$? > /tmp/transfer_${key}.done
|
||||
'
|
||||
"
|
||||
|
||||
# Give it time to start
|
||||
sleep 2
|
||||
|
||||
# Check if screen session is running
|
||||
if ! $SSH_CMD "$DEST_HOST" "screen -list | grep -q transfer_${key}"; then
|
||||
echo "Error: Screen session failed to start"
|
||||
return 1
|
||||
fi
|
||||
|
||||
else
|
||||
echo "Screen not available, using nohup method..."
|
||||
|
||||
# Use nohup with proper backgrounding
|
||||
$SSH_CMD "$DEST_HOST" "
|
||||
nohup bash -c '
|
||||
nc -l -p $port | zstd -d | tar -xf - -C / 2>/tmp/transfer_${key}.err
|
||||
echo \$? > /tmp/transfer_${key}.done
|
||||
' > /tmp/transfer_${key}.log 2>&1 < /dev/null &
|
||||
echo \$! > /tmp/transfer_${key}.pid
|
||||
"
|
||||
|
||||
sleep 2
|
||||
|
||||
# Verify process is running
|
||||
if ! $SSH_CMD "$DEST_HOST" "[[ -f /tmp/transfer_${key}.pid ]] && kill -0 \$(cat /tmp/transfer_${key}.pid) 2>/dev/null"; then
|
||||
echo "Error: Listener process failed to start"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Optional: Check if port is listening (may fail if tools aren't available)
|
||||
echo "Checking if port $port is ready..."
|
||||
if check_port_listening $port; then
|
||||
echo "Port $port is listening, starting transfer..."
|
||||
else
|
||||
echo "Cannot verify port status, proceeding with transfer anyway..."
|
||||
fi
|
||||
|
||||
# Send the data
|
||||
echo "Sending data to ${DEST_HOST}:${port}..."
|
||||
tar -cf - --dereference "$source_folder" 2>/dev/null | \
|
||||
pv -pterb -s "$folder_size" -N "$key" | \
|
||||
zstd -3 -T0 | \
|
||||
nc -w 60 "$DEST_HOST" "$port"
|
||||
|
||||
local transfer_status=$?
|
||||
|
||||
if [[ $transfer_status -eq 0 ]]; then
|
||||
echo "Transfer complete, waiting for extraction to finish..."
|
||||
|
||||
# Wait for done flag with timeout
|
||||
local attempts=0
|
||||
local max_attempts=60 # Wait up to 2 minutes
|
||||
|
||||
while [[ $attempts -lt $max_attempts ]]; do
|
||||
if $SSH_CMD "$DEST_HOST" "[[ -f /tmp/transfer_${key}.done ]]" 2>/dev/null; then
|
||||
local remote_status=$($SSH_CMD "$DEST_HOST" "cat /tmp/transfer_${key}.done 2>/dev/null || echo 1")
|
||||
|
||||
if [[ "$remote_status" == "0" ]]; then
|
||||
echo "✓ Volume $key transferred and extracted successfully"
|
||||
|
||||
# Cleanup
|
||||
$SSH_CMD "$DEST_HOST" "
|
||||
rm -f /tmp/transfer_${key}.* 2>/dev/null
|
||||
screen -S transfer_${key} -X quit 2>/dev/null
|
||||
[[ -f /tmp/transfer_${key}.pid ]] && kill \$(cat /tmp/transfer_${key}.pid) 2>/dev/null
|
||||
"
|
||||
return 0
|
||||
else
|
||||
echo "✗ Extraction failed with status $remote_status"
|
||||
echo "Error log:"
|
||||
$SSH_CMD "$DEST_HOST" "cat /tmp/transfer_${key}.err 2>/dev/null || echo 'No error log'"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show progress
|
||||
if [[ $((attempts % 5)) -eq 0 ]]; then
|
||||
echo "Still waiting for extraction to complete... ($attempts/$max_attempts)"
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
attempts=$((attempts + 1))
|
||||
done
|
||||
|
||||
echo "⚠ Timeout waiting for extraction to complete"
|
||||
return 1
|
||||
else
|
||||
echo "✗ Transfer failed with status $transfer_status"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Fallback: Direct SSH pipe
|
||||
transfer_volume_ssh() {
|
||||
local key=$1
|
||||
local source_folder="/var/lib/docker/volumes/rpc_${key}/_data"
|
||||
|
||||
if [[ ! -d "$source_folder" ]]; then
|
||||
echo "Warning: $source_folder does not exist, skipping"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local folder_size=$(du -sb "$source_folder" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
echo "Using direct SSH transfer for $key (size: $((folder_size / 1048576))MB)"
|
||||
|
||||
tar -cf - --dereference "$source_folder" 2>/dev/null | \
|
||||
pv -pterb -s "$folder_size" -N "$key" | \
|
||||
zstd -3 -T0 | \
|
||||
$SSH_CMD -c chacha20-poly1305@openssh.com "$DEST_HOST" \
|
||||
"zstd -d | tar -xf - -C /"
|
||||
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "✓ Volume $key transferred successfully"
|
||||
return 0
|
||||
else
|
||||
echo "✗ Transfer failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
setup_ssh_multiplex
|
||||
|
||||
ssh "$DEST_HOST" "
|
||||
sudo sysctl -w net.core.rmem_max=67108864
|
||||
sudo sysctl -w net.core.wmem_max=67108864
|
||||
sudo sysctl -w net.ipv4.tcp_rmem='4096 87380 67108864'
|
||||
sudo sysctl -w net.ipv4.tcp_wmem='4096 87380 67108864'
|
||||
sudo sysctl -w net.ipv4.tcp_congestion_control=bbr
|
||||
sudo sysctl -w net.core.default_qdisc=fq
|
||||
"
|
||||
|
||||
echo "Reading volume configuration from $1.yml..."
|
||||
keys=$(cat /root/rpc/$1.yml | yaml2json - | jq '.volumes' | jq -r 'keys[]')
|
||||
|
||||
if [[ -z "$keys" ]]; then
|
||||
echo "Error: No volumes found in configuration"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
volume_count=$(echo "$keys" | wc -l)
|
||||
echo "Found $volume_count volumes to transfer"
|
||||
echo "----------------------------------------"
|
||||
|
||||
success_count=0
|
||||
failed_volumes=""
|
||||
|
||||
for key in $keys; do
|
||||
# Try nc method first
|
||||
transfer_volume "$key"
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "NC transfer failed, trying direct SSH..."
|
||||
transfer_volume_ssh "$key"
|
||||
|
||||
if [[ $? -eq 0 ]]; then
|
||||
success_count=$((success_count + 1))
|
||||
else
|
||||
failed_volumes="$failed_volumes $key"
|
||||
fi
|
||||
else
|
||||
success_count=$((success_count + 1))
|
||||
fi
|
||||
|
||||
echo "----------------------------------------"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Transfer Summary:"
|
||||
echo " Successful: $success_count/$volume_count"
|
||||
[[ -n "$failed_volumes" ]] && echo " Failed:$failed_volumes"
|
||||
|
||||
$SSH_CMD -O exit "$DEST_HOST" 2>/dev/null
|
||||
|
||||
[[ $success_count -eq $volume_count ]] && exit 0 || exit 1
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user