diff --git a/clone-node.sh b/clone-node.sh index c001fbec..bcf8f687 100644 --- a/clone-node.sh +++ b/clone-node.sh @@ -1,21 +1,271 @@ #!/bin/bash +# Fixed version that handles missing netstat + if [[ -n $2 ]]; then - echo "clone volumes via ssh to $2" + DEST_HOST="$2.stakesquid.eu" + echo "Setting up optimized transfer to $DEST_HOST" else echo "Error: No destination provided" exit 1 fi -# Read the JSON input and extract the list of keys -keys=$(cat /root/rpc/$1.yml | yaml2json - | jq '.volumes' | jq -r 'keys[]') +# Configuration +BASE_PORT=9000 -# Iterate over the list of keys -for key in $keys; do - echo "Executing command with key: /var/lib/docker/volumes/rpc_$key/_data" - - source_folder="/var/lib/docker/volumes/rpc_$key/_data" - if [[ -n $2 ]]; then - tar -cf - --dereference "$source_folder" | pv -pterb -s $(du -sb "$source_folder" | awk '{print $1}') | zstd | ssh -o Compression=no -c=chacha20-poly1305@openssh.com "$2.stakesquid.eu" "zstd -d | tar -xf - -C /" +# Setup SSH multiplexing +setup_ssh_multiplex() { + echo "Setting up SSH control connection..." + ssh -nNf -o ControlMaster=yes \ + -o ControlPath=/tmp/ssh-mux-%h-%p-%r \ + -o ControlPersist=600 \ + -o Compression=no \ + "$DEST_HOST" 2>/dev/null + + if [[ $? -eq 0 ]]; then + echo "SSH control connection established" + export SSH_CMD="ssh -o ControlPath=/tmp/ssh-mux-%h-%p-%r" + else + echo "Failed to setup SSH multiplexing, using direct SSH" + export SSH_CMD="ssh" fi -done +} + +# Check if port is listening using various methods +check_port_listening() { + local port=$1 + + # Try different methods to check if port is listening + $SSH_CMD "$DEST_HOST" " + if command -v ss >/dev/null 2>&1; then + ss -tln | grep -q ':$port ' + elif command -v netstat >/dev/null 2>&1; then + netstat -tln | grep -q ':$port ' + elif command -v lsof >/dev/null 2>&1; then + lsof -i :$port >/dev/null 2>&1 + else + # If no tools available, just try to connect to the port + timeout 1 bash -c 'cat < /dev/null > /dev/tcp/localhost/$port' 2>/dev/null + fi + " + return $? +} + +# Transfer using screen method with better error handling +transfer_volume() { + local key=$1 + local source_folder="/var/lib/docker/volumes/rpc_${key}/_data" + + if [[ ! -d "$source_folder" ]]; then + echo "Warning: $source_folder does not exist, skipping" + return 1 + fi + + local folder_size=$(du -sb "$source_folder" 2>/dev/null | awk '{print $1}') + local port=$BASE_PORT + + echo "Transferring volume $key (size: $((folder_size / 1048576))MB)" + + # Kill any existing listener on this port + $SSH_CMD "$DEST_HOST" " + # Try to kill any existing nc on this port + lsof -i :$port 2>/dev/null | grep LISTEN | awk '{print \$2}' | xargs -r kill 2>/dev/null + # Kill any existing screen session for this transfer + screen -S transfer_${key} -X quit 2>/dev/null + # Clean up old files + rm -f /tmp/transfer_${key}.* 2>/dev/null + " + + # Check if screen is available + if $SSH_CMD "$DEST_HOST" "which screen" >/dev/null 2>&1; then + echo "Starting screen listener on port $port..." + + # Start listener in screen session with proper escaping + $SSH_CMD "$DEST_HOST" " + screen -dmS transfer_${key} bash -c ' + nc -l -p $port | zstd -d | tar -xf - -C / 2>/tmp/transfer_${key}.err + echo \$? > /tmp/transfer_${key}.done + ' + " + + # Give it time to start + sleep 2 + + # Check if screen session is running + if ! $SSH_CMD "$DEST_HOST" "screen -list | grep -q transfer_${key}"; then + echo "Error: Screen session failed to start" + return 1 + fi + + else + echo "Screen not available, using nohup method..." + + # Use nohup with proper backgrounding + $SSH_CMD "$DEST_HOST" " + nohup bash -c ' + nc -l -p $port | zstd -d | tar -xf - -C / 2>/tmp/transfer_${key}.err + echo \$? > /tmp/transfer_${key}.done + ' > /tmp/transfer_${key}.log 2>&1 < /dev/null & + echo \$! > /tmp/transfer_${key}.pid + " + + sleep 2 + + # Verify process is running + if ! $SSH_CMD "$DEST_HOST" "[[ -f /tmp/transfer_${key}.pid ]] && kill -0 \$(cat /tmp/transfer_${key}.pid) 2>/dev/null"; then + echo "Error: Listener process failed to start" + return 1 + fi + fi + + # Optional: Check if port is listening (may fail if tools aren't available) + echo "Checking if port $port is ready..." + if check_port_listening $port; then + echo "Port $port is listening, starting transfer..." + else + echo "Cannot verify port status, proceeding with transfer anyway..." + fi + + # Send the data + echo "Sending data to ${DEST_HOST}:${port}..." + tar -cf - --dereference "$source_folder" 2>/dev/null | \ + pv -pterb -s "$folder_size" -N "$key" | \ + zstd -3 -T0 | \ + nc -w 60 "$DEST_HOST" "$port" + + local transfer_status=$? + + if [[ $transfer_status -eq 0 ]]; then + echo "Transfer complete, waiting for extraction to finish..." + + # Wait for done flag with timeout + local attempts=0 + local max_attempts=60 # Wait up to 2 minutes + + while [[ $attempts -lt $max_attempts ]]; do + if $SSH_CMD "$DEST_HOST" "[[ -f /tmp/transfer_${key}.done ]]" 2>/dev/null; then + local remote_status=$($SSH_CMD "$DEST_HOST" "cat /tmp/transfer_${key}.done 2>/dev/null || echo 1") + + if [[ "$remote_status" == "0" ]]; then + echo "✓ Volume $key transferred and extracted successfully" + + # Cleanup + $SSH_CMD "$DEST_HOST" " + rm -f /tmp/transfer_${key}.* 2>/dev/null + screen -S transfer_${key} -X quit 2>/dev/null + [[ -f /tmp/transfer_${key}.pid ]] && kill \$(cat /tmp/transfer_${key}.pid) 2>/dev/null + " + return 0 + else + echo "✗ Extraction failed with status $remote_status" + echo "Error log:" + $SSH_CMD "$DEST_HOST" "cat /tmp/transfer_${key}.err 2>/dev/null || echo 'No error log'" + return 1 + fi + fi + + # Show progress + if [[ $((attempts % 5)) -eq 0 ]]; then + echo "Still waiting for extraction to complete... ($attempts/$max_attempts)" + fi + + sleep 2 + attempts=$((attempts + 1)) + done + + echo "⚠ Timeout waiting for extraction to complete" + return 1 + else + echo "✗ Transfer failed with status $transfer_status" + return 1 + fi +} + +# Fallback: Direct SSH pipe +transfer_volume_ssh() { + local key=$1 + local source_folder="/var/lib/docker/volumes/rpc_${key}/_data" + + if [[ ! -d "$source_folder" ]]; then + echo "Warning: $source_folder does not exist, skipping" + return 1 + fi + + local folder_size=$(du -sb "$source_folder" 2>/dev/null | awk '{print $1}') + + echo "Using direct SSH transfer for $key (size: $((folder_size / 1048576))MB)" + + tar -cf - --dereference "$source_folder" 2>/dev/null | \ + pv -pterb -s "$folder_size" -N "$key" | \ + zstd -3 -T0 | \ + $SSH_CMD -c chacha20-poly1305@openssh.com "$DEST_HOST" \ + "zstd -d | tar -xf - -C /" + + if [[ $? -eq 0 ]]; then + echo "✓ Volume $key transferred successfully" + return 0 + else + echo "✗ Transfer failed" + return 1 + fi +} + +# Main execution +main() { + setup_ssh_multiplex + + ssh "$DEST_HOST" " + sudo sysctl -w net.core.rmem_max=67108864 + sudo sysctl -w net.core.wmem_max=67108864 + sudo sysctl -w net.ipv4.tcp_rmem='4096 87380 67108864' + sudo sysctl -w net.ipv4.tcp_wmem='4096 87380 67108864' + sudo sysctl -w net.ipv4.tcp_congestion_control=bbr + sudo sysctl -w net.core.default_qdisc=fq + " + + echo "Reading volume configuration from $1.yml..." + keys=$(cat /root/rpc/$1.yml | yaml2json - | jq '.volumes' | jq -r 'keys[]') + + if [[ -z "$keys" ]]; then + echo "Error: No volumes found in configuration" + exit 1 + fi + + volume_count=$(echo "$keys" | wc -l) + echo "Found $volume_count volumes to transfer" + echo "----------------------------------------" + + success_count=0 + failed_volumes="" + + for key in $keys; do + # Try nc method first + transfer_volume "$key" + + if [[ $? -ne 0 ]]; then + echo "NC transfer failed, trying direct SSH..." + transfer_volume_ssh "$key" + + if [[ $? -eq 0 ]]; then + success_count=$((success_count + 1)) + else + failed_volumes="$failed_volumes $key" + fi + else + success_count=$((success_count + 1)) + fi + + echo "----------------------------------------" + done + + echo "" + echo "Transfer Summary:" + echo " Successful: $success_count/$volume_count" + [[ -n "$failed_volumes" ]] && echo " Failed:$failed_volumes" + + $SSH_CMD -O exit "$DEST_HOST" 2>/dev/null + + [[ $success_count -eq $volume_count ]] && exit 0 || exit 1 +} + +main "$@" \ No newline at end of file