From 575b7133bbb2f689d762bd558801bd1af41c3a37 Mon Sep 17 00:00:00 2001 From: goldsquid Date: Sun, 21 Dec 2025 18:37:14 +0700 Subject: [PATCH] allow peers backups --- .gitignore | 1 + backup-peers.sh | 260 +++++++++++++++++++++++++++++++++++++++++++ list-peer-backups.sh | 90 +++++++++++++++ restore-peers.sh | 162 +++++++++++++++++++++++++++ 4 files changed, 513 insertions(+) create mode 100755 backup-peers.sh create mode 100755 list-peer-backups.sh create mode 100755 restore-peers.sh diff --git a/.gitignore b/.gitignore index 4c49bd78..62a7315a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .env +peer-backups/ diff --git a/backup-peers.sh b/backup-peers.sh new file mode 100755 index 00000000..8e0a24cd --- /dev/null +++ b/backup-peers.sh @@ -0,0 +1,260 @@ +#!/bin/bash + +# Script to backup peers from all running nodes +# Can be run as a cronjob to periodically backup peer lists +# Usage: ./backup-peers.sh [backup-directory] + +BASEPATH="$(dirname "$0")" +source $BASEPATH/.env + +# Default backup directory +BACKUP_DIR="${1:-$BASEPATH/peer-backups}" + +# Create backup directory if it doesn't exist +mkdir -p "$BACKUP_DIR" + +# Timestamp for this backup run +TIMESTAMP=$(date +%Y%m%d_%H%M%S) + +# Blacklist for compose files (same as show-status.sh) +blacklist=( + "drpc.yml" "drpc-free.yml" "drpc-home.yml" # dshackles + "arbitrum-one-mainnet-arbnode-archive-trace.yml" # always behind and no reference rpc + "ethereum-beacon-mainnet-lighthouse-pruned-blobs" # can't handle beacon rest api yet + "rpc.yml" "monitoring.yml" "ftp.yml" "backup-http.yml" "base.yml" # no rpcs +) + +# Path blacklist (read from file if it exists) +path_blacklist=() +if [ -f "$BASEPATH/path-blacklist.txt" ]; then + while IFS= read -r line; do + if [ -n "$line" ]; then + path_blacklist+=("$line") + fi + done < "$BASEPATH/path-blacklist.txt" +fi + +# Protocol and domain settings +if [ -n "$NO_SSL" ]; then + PROTO="http" + DOMAIN="${DOMAIN:-0.0.0.0}" +else + PROTO="https" +fi + +# Function to extract RPC paths from a compose file +extract_rpc_paths() { + local compose_file="$1" + local full_path="$BASEPATH/${compose_file}" + + if [ ! -f "$full_path" ]; then + return 1 + fi + + # Extract paths using grep (same method as peer-count.sh) + pathlist=$(cat "$full_path" | grep -oP "stripprefix\.prefixes.*?/\K[^\"]+" 2>/dev/null) + + if [ -z "$pathlist" ]; then + return 1 + fi + + echo "$pathlist" +} + +# Function to check if a path should be included +should_include_path() { + local path="$1" + + for word in "${path_blacklist[@]}"; do + if echo "$path" | grep -qE "$word"; then + return 1 + fi + done + + return 0 +} + +# Function to backup peers from a single RPC endpoint +backup_peers_from_path() { + local compose_file="$1" + local path="$2" + local compose_name="${compose_file%.yml}" + + # Sanitize compose name and path for filename + local safe_compose_name=$(echo "$compose_name" | sed 's/[^a-zA-Z0-9_-]/_/g') + local safe_path=$(echo "$path" | sed 's|[^a-zA-Z0-9_-]|_|g') + + local RPC_URL="${PROTO}://${DOMAIN}${path}" + + # Try admin_peers first (returns detailed peer info) + response=$(curl --ipv4 -L -s -X POST "$RPC_URL" \ + -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"admin_peers","params":[],"id":1}' \ + --max-time 10 2>/dev/null) + + # Check if we got a valid response + if echo "$response" | jq -e '.result' > /dev/null 2>&1; then + peer_count=$(echo "$response" | jq -r '.result | length') + + if [ "$peer_count" -gt 0 ]; then + # Extract enodes + enodes=$(echo "$response" | jq -r '.result[].enode' 2>/dev/null | grep -v '^$' | grep -v '^null$') + + if [ -n "$enodes" ]; then + # Create backup file + local backup_file="$BACKUP_DIR/${safe_compose_name}__${safe_path}__${TIMESTAMP}.json" + + # Create JSON structure with metadata + { + echo "{" + echo " \"compose_file\": \"$compose_file\"," + echo " \"rpc_path\": \"$path\"," + echo " \"rpc_url\": \"$RPC_URL\"," + echo " \"timestamp\": \"$TIMESTAMP\"," + echo " \"peer_count\": $peer_count," + echo " \"peers\": [" + + # Write enodes as JSON array + first=true + while IFS= read -r enode; do + if [ -z "$enode" ] || [ "$enode" = "null" ]; then + continue + fi + + if [ "$first" = true ]; then + first=false + else + echo "," + fi + + # Escape the enode string for JSON + escaped_enode=$(echo "$enode" | sed 's/\\/\\\\/g' | sed 's/"/\\"/g') + echo -n " \"$escaped_enode\"" + done <<< "$enodes" + + echo "" + echo " ]" + echo "}" + } > "$backup_file" + + # Also create a simple text file with just enodes (one per line) for easy playback + local backup_txt_file="$BACKUP_DIR/${safe_compose_name}__${safe_path}__${TIMESTAMP}.txt" + echo "$enodes" > "$backup_txt_file" + + echo "✓ Backed up $peer_count peer(s) from $compose_file ($path) to $(basename "$backup_file")" + return 0 + fi + else + echo "⚠ No peers found for $compose_file ($path)" + return 1 + fi + else + # Check if this is a method not found error (consensus client or admin API disabled) + error_code=$(echo "$response" | jq -r '.error.code // empty' 2>/dev/null) + + if [ -n "$error_code" ] && [ "$error_code" != "null" ]; then + # This is likely a consensus client endpoint, skip it silently + return 1 + fi + + # Try net_peerCount as fallback (but we can't get enodes from this) + response=$(curl --ipv4 -L -s -X POST "$RPC_URL" \ + -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' \ + --max-time 10 2>/dev/null) + + if echo "$response" | jq -e '.result' > /dev/null 2>&1; then + peer_count=$(echo "$response" | jq -r '.result' | xargs printf "%d") + if [ "$peer_count" -gt 0 ]; then + echo "⚠ $compose_file ($path) has $peer_count peer(s) but admin_peers not available (cannot backup enodes)" + fi + fi + + return 1 + fi +} + +# Main execution +if [ -z "$COMPOSE_FILE" ]; then + echo "Error: COMPOSE_FILE not found in $BASEPATH/.env" >&2 + exit 1 +fi + +# Split COMPOSE_FILE by colon +IFS=':' read -ra parts <<< "$COMPOSE_FILE" + +total_backed_up=0 +total_failed=0 +total_skipped=0 + +echo "Starting peer backup at $(date)" +echo "Backup directory: $BACKUP_DIR" +echo "" + +# Process each compose file +for part in "${parts[@]}"; do + # Remove .yml extension if present for processing + compose_file="${part%.yml}.yml" + + # Check blacklist + include=true + for word in "${blacklist[@]}"; do + if echo "$compose_file" | grep -qE "$word"; then + include=false + break + fi + done + + if [ "$include" = false ]; then + total_skipped=$((total_skipped + 1)) + continue + fi + + # Extract RPC paths from compose file + paths=$(extract_rpc_paths "$compose_file") + + if [ -z "$paths" ]; then + total_skipped=$((total_skipped + 1)) + continue + fi + + # Process each path + path_found=false + for path in $paths; do + # Check path blacklist + if should_include_path "$path"; then + path_found=true + if backup_peers_from_path "$compose_file" "$path"; then + total_backed_up=$((total_backed_up + 1)) + else + total_failed=$((total_failed + 1)) + fi + fi + done + + if [ "$path_found" = false ]; then + total_skipped=$((total_skipped + 1)) + fi +done + +echo "" +echo "==========================================" +echo "Backup Summary" +echo "==========================================" +echo "Total nodes backed up: $total_backed_up" +echo "Total nodes failed: $total_failed" +echo "Total nodes skipped: $total_skipped" +echo "Backup directory: $BACKUP_DIR" +echo "Completed at $(date)" +echo "" + +# Optional: Clean up old backups (keep last 30 days) +if [ -n "$CLEANUP_OLD_BACKUPS" ] && [ "$CLEANUP_OLD_BACKUPS" = "true" ]; then + echo "Cleaning up backups older than 30 days..." + find "$BACKUP_DIR" -name "*.json" -type f -mtime +30 -delete + find "$BACKUP_DIR" -name "*.txt" -type f -mtime +30 -delete + echo "Cleanup complete" +fi + +exit 0 + diff --git a/list-peer-backups.sh b/list-peer-backups.sh new file mode 100755 index 00000000..15ceb9e9 --- /dev/null +++ b/list-peer-backups.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Script to list available peer backups +# Usage: ./list-peer-backups.sh [backup-directory] [filter] + +BASEPATH="$(dirname "$0")" +BACKUP_DIR="${1:-$BASEPATH/peer-backups}" +FILTER="${2:-}" + +if [ ! -d "$BACKUP_DIR" ]; then + echo "Backup directory does not exist: $BACKUP_DIR" + echo "Run backup-peers.sh first to create backups" + exit 1 +fi + +# Count files +json_count=$(find "$BACKUP_DIR" -name "*.json" -type f | wc -l | tr -d ' ') +txt_count=$(find "$BACKUP_DIR" -name "*.txt" -type f | wc -l | tr -d ' ') + +if [ "$json_count" -eq 0 ] && [ "$txt_count" -eq 0 ]; then + echo "No backups found in $BACKUP_DIR" + exit 0 +fi + +echo "Peer backups in: $BACKUP_DIR" +echo "Total backups: $json_count JSON, $txt_count TXT" +echo "" + +# List JSON backups with details +if [ "$json_count" -gt 0 ]; then + echo "JSON Backups (with metadata):" + echo "==========================================" + + for backup_file in $(find "$BACKUP_DIR" -name "*.json" -type f | sort -r); do + if [ -n "$FILTER" ] && ! echo "$backup_file" | grep -qi "$FILTER"; then + continue + fi + + if command -v jq &> /dev/null; then + compose_file=$(jq -r '.compose_file // "unknown"' "$backup_file" 2>/dev/null) + rpc_path=$(jq -r '.rpc_path // "unknown"' "$backup_file" 2>/dev/null) + timestamp=$(jq -r '.timestamp // "unknown"' "$backup_file" 2>/dev/null) + peer_count=$(jq -r '.peer_count // 0' "$backup_file" 2>/dev/null) + rpc_url=$(jq -r '.rpc_url // "unknown"' "$backup_file" 2>/dev/null) + + file_size=$(du -h "$backup_file" | cut -f1) + file_date=$(stat -f "%Sm" -t "%Y-%m-%d %H:%M:%S" "$backup_file" 2>/dev/null || stat -c "%y" "$backup_file" 2>/dev/null | cut -d'.' -f1) + + echo "File: $(basename "$backup_file")" + echo " Compose: $compose_file" + echo " Path: $rpc_path" + echo " URL: $rpc_url" + echo " Peers: $peer_count" + echo " Timestamp: $timestamp" + echo " Size: $file_size" + echo " Date: $file_date" + echo "" + else + # Fallback if jq is not available + file_size=$(du -h "$backup_file" | cut -f1) + file_date=$(stat -f "%Sm" -t "%Y-%m-%d %H:%M:%S" "$backup_file" 2>/dev/null || stat -c "%y" "$backup_file" 2>/dev/null | cut -d'.' -f1) + echo "$(basename "$backup_file") - $file_size - $file_date" + fi + done +fi + +# Show usage examples +echo "" +echo "To restore a backup, use:" +echo " ./restore-peers.sh [rpc-url]" +echo "" +echo "Examples:" +if [ "$json_count" -gt 0 ]; then + latest_json=$(find "$BACKUP_DIR" -name "*.json" -type f | sort -r | head -1) + if [ -n "$latest_json" ]; then + echo " ./restore-peers.sh $latest_json" + fi +fi +if [ "$txt_count" -gt 0 ]; then + latest_txt=$(find "$BACKUP_DIR" -name "*.txt" -type f | sort -r | head -1) + if [ -n "$latest_txt" ]; then + if command -v jq &> /dev/null && [ -n "$latest_json" ]; then + rpc_url=$(jq -r '.rpc_url // ""' "$latest_json" 2>/dev/null) + if [ -n "$rpc_url" ]; then + echo " ./restore-peers.sh $latest_txt $rpc_url" + fi + fi + fi +fi + diff --git a/restore-peers.sh b/restore-peers.sh new file mode 100755 index 00000000..89ae1cd6 --- /dev/null +++ b/restore-peers.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# Script to restore peers from a backup file +# Usage: ./restore-peers.sh [rpc-url] +# backup-file: Path to backup JSON or TXT file +# rpc-url: Optional. If not provided, will extract from backup file + +BASEPATH="$(dirname "$0")" +source $BASEPATH/.env + +if [ $# -lt 1 ]; then + echo "Usage: $0 [rpc-url]" + echo "" + echo " backup-file: Path to backup JSON or TXT file" + echo " rpc-url: Optional. Target RPC URL to restore peers to" + echo " If not provided, will use rpc_url from backup file" + echo "" + echo "Examples:" + echo " $0 peer-backups/ethereum-mainnet__ethereum-mainnet-archive__20240101_120000.json" + echo " $0 peer-backups/ethereum-mainnet__ethereum-mainnet-archive__20240101_120000.txt https://domain.com/ethereum-mainnet-archive" + exit 1 +fi + +BACKUP_FILE="$1" +TARGET_URL="$2" + +if [ ! -f "$BACKUP_FILE" ]; then + echo "Error: Backup file not found: $BACKUP_FILE" >&2 + exit 1 +fi + +# Determine file type and extract enodes +if [[ "$BACKUP_FILE" == *.json ]]; then + # JSON backup file + if ! command -v jq &> /dev/null; then + echo "Error: jq is required to parse JSON backup files" >&2 + exit 1 + fi + + # Extract RPC URL from backup if not provided + if [ -z "$TARGET_URL" ]; then + TARGET_URL=$(jq -r '.rpc_url // empty' "$BACKUP_FILE" 2>/dev/null) + if [ -z "$TARGET_URL" ] || [ "$TARGET_URL" = "null" ]; then + echo "Error: Could not extract rpc_url from backup file and no target URL provided" >&2 + exit 1 + fi + fi + + # Extract enodes + enodes=$(jq -r '.peers[]?' "$BACKUP_FILE" 2>/dev/null) + + if [ -z "$enodes" ]; then + echo "Error: No peers found in backup file" >&2 + exit 1 + fi + + peer_count=$(jq -r '.peer_count // 0' "$BACKUP_FILE" 2>/dev/null) + compose_file=$(jq -r '.compose_file // "unknown"' "$BACKUP_FILE" 2>/dev/null) + rpc_path=$(jq -r '.rpc_path // "unknown"' "$BACKUP_FILE" 2>/dev/null) + timestamp=$(jq -r '.timestamp // "unknown"' "$BACKUP_FILE" 2>/dev/null) + + echo "Restoring peers from backup:" + echo " Compose file: $compose_file" + echo " RPC path: $rpc_path" + echo " Timestamp: $timestamp" + echo " Peer count: $peer_count" + echo " Target URL: $TARGET_URL" + echo "" + +elif [[ "$BACKUP_FILE" == *.txt ]]; then + # TXT backup file (one enode per line) + enodes=$(grep -v '^$' "$BACKUP_FILE" | grep -v '^null$') + + if [ -z "$enodes" ]; then + echo "Error: No peers found in backup file" >&2 + exit 1 + fi + + peer_count=$(echo "$enodes" | wc -l | tr -d ' ') + + if [ -z "$TARGET_URL" ]; then + echo "Error: Target RPC URL required for TXT backup files" >&2 + echo "Usage: $0 " >&2 + exit 1 + fi + + echo "Restoring peers from backup:" + echo " Backup file: $BACKUP_FILE" + echo " Peer count: $peer_count" + echo " Target URL: $TARGET_URL" + echo "" + +else + echo "Error: Unsupported backup file format. Expected .json or .txt" >&2 + exit 1 +fi + +# Confirm before proceeding +read -p "Do you want to restore $peer_count peer(s) to $TARGET_URL? (y/N) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Restore cancelled" + exit 0 +fi + +# Restore peers +success_count=0 +failed_count=0 +skipped_count=0 + +echo "" +echo "Restoring peers..." + +while IFS= read -r enode; do + if [ -z "$enode" ] || [ "$enode" = "null" ]; then + skipped_count=$((skipped_count + 1)) + continue + fi + + # Check if peer is reachable (optional, can be slow) + if [ -f "$BASEPATH/check-enode.sh" ]; then + if ! "$BASEPATH/check-enode.sh" "$enode" --target "$TARGET_URL" > /dev/null 2>&1; then + echo "⚠ Skipping unreachable peer: ${enode:0:50}..." + skipped_count=$((skipped_count + 1)) + continue + fi + fi + + echo -n "Adding peer: ${enode:0:50}... " + + result=$(curl --ipv4 -X POST -H "Content-Type: application/json" \ + --silent --max-time 10 \ + --data "{\"jsonrpc\":\"2.0\",\"method\":\"admin_addPeer\",\"params\":[\"${enode}\"],\"id\":1}" \ + "$TARGET_URL" 2>/dev/null | jq -r '.result // .error.message // "unknown error"' 2>/dev/null) + + if [ "$result" = "true" ] || [ "$result" = "null" ]; then + echo "✓ Success" + success_count=$((success_count + 1)) + else + echo "✗ Failed: $result" + failed_count=$((failed_count + 1)) + fi + + # Small delay to avoid overwhelming the node + sleep 0.1 +done <<< "$enodes" + +echo "" +echo "==========================================" +echo "Restore Summary" +echo "==========================================" +echo "Successful: $success_count/$peer_count" +echo "Failed: $failed_count/$peer_count" +echo "Skipped: $skipped_count/$peer_count" +echo "" + +if [ $failed_count -gt 0 ]; then + exit 1 +fi + +exit 0 +