allow peers backups

.gitignore

@@ -1 +1,2 @@
 .env
+peer-backups/

backup-peers.sh

@@ -0,0 +1,260 @@
+#!/bin/bash
+
+# Script to backup peers from all running nodes
+# Can be run as a cronjob to periodically backup peer lists
+# Usage: ./backup-peers.sh [backup-directory]
+
+BASEPATH="$(dirname "$0")"
+source $BASEPATH/.env
+
+# Default backup directory
+BACKUP_DIR="${1:-$BASEPATH/peer-backups}"
+
+# Create backup directory if it doesn't exist
+mkdir -p "$BACKUP_DIR"
+
+# Timestamp for this backup run
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+# Blacklist for compose files (same as show-status.sh)
+blacklist=(
+    "drpc.yml" "drpc-free.yml" "drpc-home.yml" # dshackles
+    "arbitrum-one-mainnet-arbnode-archive-trace.yml" # always behind and no reference rpc
+    "ethereum-beacon-mainnet-lighthouse-pruned-blobs" # can't handle beacon rest api yet
+    "rpc.yml" "monitoring.yml" "ftp.yml" "backup-http.yml" "base.yml" # no rpcs
+)
+
+# Path blacklist (read from file if it exists)
+path_blacklist=()
+if [ -f "$BASEPATH/path-blacklist.txt" ]; then
+    while IFS= read -r line; do
+        if [ -n "$line" ]; then
+            path_blacklist+=("$line")
+        fi
+    done < "$BASEPATH/path-blacklist.txt"
+fi
+
+# Protocol and domain settings
+if [ -n "$NO_SSL" ]; then
+    PROTO="http"
+    DOMAIN="${DOMAIN:-0.0.0.0}"
+else
+    PROTO="https"
+fi
+
+# Function to extract RPC paths from a compose file
+extract_rpc_paths() {
+    local compose_file="$1"
+    local full_path="$BASEPATH/${compose_file}"
+    
+    if [ ! -f "$full_path" ]; then
+        return 1
+    fi
+    
+    # Extract paths using grep (same method as peer-count.sh)
+    pathlist=$(cat "$full_path" | grep -oP "stripprefix\.prefixes.*?/\K[^\"]+" 2>/dev/null)
+    
+    if [ -z "$pathlist" ]; then
+        return 1
+    fi
+    
+    echo "$pathlist"
+}
+
+# Function to check if a path should be included
+should_include_path() {
+    local path="$1"
+    
+    for word in "${path_blacklist[@]}"; do
+        if echo "$path" | grep -qE "$word"; then
+            return 1
+        fi
+    done
+    
+    return 0
+}
+
+# Function to backup peers from a single RPC endpoint
+backup_peers_from_path() {
+    local compose_file="$1"
+    local path="$2"
+    local compose_name="${compose_file%.yml}"
+    
+    # Sanitize compose name and path for filename
+    local safe_compose_name=$(echo "$compose_name" | sed 's/[^a-zA-Z0-9_-]/_/g')
+    local safe_path=$(echo "$path" | sed 's|[^a-zA-Z0-9_-]|_|g')
+    
+    local RPC_URL="${PROTO}://${DOMAIN}${path}"
+    
+    # Try admin_peers first (returns detailed peer info)
+    response=$(curl --ipv4 -L -s -X POST "$RPC_URL" \
+        -H "Content-Type: application/json" \
+        --data '{"jsonrpc":"2.0","method":"admin_peers","params":[],"id":1}' \
+        --max-time 10 2>/dev/null)
+    
+    # Check if we got a valid response
+    if echo "$response" | jq -e '.result' > /dev/null 2>&1; then
+        peer_count=$(echo "$response" | jq -r '.result | length')
+        
+        if [ "$peer_count" -gt 0 ]; then
+            # Extract enodes
+            enodes=$(echo "$response" | jq -r '.result[].enode' 2>/dev/null | grep -v '^$' | grep -v '^null$')
+            
+            if [ -n "$enodes" ]; then
+                # Create backup file
+                local backup_file="$BACKUP_DIR/${safe_compose_name}__${safe_path}__${TIMESTAMP}.json"
+                
+                # Create JSON structure with metadata
+                {
+                    echo "{"
+                    echo "  \"compose_file\": \"$compose_file\","
+                    echo "  \"rpc_path\": \"$path\","
+                    echo "  \"rpc_url\": \"$RPC_URL\","
+                    echo "  \"timestamp\": \"$TIMESTAMP\","
+                    echo "  \"peer_count\": $peer_count,"
+                    echo "  \"peers\": ["
+                    
+                    # Write enodes as JSON array
+                    first=true
+                    while IFS= read -r enode; do
+                        if [ -z "$enode" ] || [ "$enode" = "null" ]; then
+                            continue
+                        fi
+                        
+                        if [ "$first" = true ]; then
+                            first=false
+                        else
+                            echo ","
+                        fi
+                        
+                        # Escape the enode string for JSON
+                        escaped_enode=$(echo "$enode" | sed 's/\\/\\\\/g' | sed 's/"/\\"/g')
+                        echo -n "    \"$escaped_enode\""
+                    done <<< "$enodes"
+                    
+                    echo ""
+                    echo "  ]"
+                    echo "}"
+                } > "$backup_file"
+                
+                # Also create a simple text file with just enodes (one per line) for easy playback
+                local backup_txt_file="$BACKUP_DIR/${safe_compose_name}__${safe_path}__${TIMESTAMP}.txt"
+                echo "$enodes" > "$backup_txt_file"
+                
+                echo "✓ Backed up $peer_count peer(s) from $compose_file ($path) to $(basename "$backup_file")"
+                return 0
+            fi
+        else
+            echo "⚠ No peers found for $compose_file ($path)"
+            return 1
+        fi
+    else
+        # Check if this is a method not found error (consensus client or admin API disabled)
+        error_code=$(echo "$response" | jq -r '.error.code // empty' 2>/dev/null)
+        
+        if [ -n "$error_code" ] && [ "$error_code" != "null" ]; then
+            # This is likely a consensus client endpoint, skip it silently
+            return 1
+        fi
+        
+        # Try net_peerCount as fallback (but we can't get enodes from this)
+        response=$(curl --ipv4 -L -s -X POST "$RPC_URL" \
+            -H "Content-Type: application/json" \
+            --data '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' \
+            --max-time 10 2>/dev/null)
+        
+        if echo "$response" | jq -e '.result' > /dev/null 2>&1; then
+            peer_count=$(echo "$response" | jq -r '.result' | xargs printf "%d")
+            if [ "$peer_count" -gt 0 ]; then
+                echo "⚠ $compose_file ($path) has $peer_count peer(s) but admin_peers not available (cannot backup enodes)"
+            fi
+        fi
+        
+        return 1
+    fi
+}
+
+# Main execution
+if [ -z "$COMPOSE_FILE" ]; then
+    echo "Error: COMPOSE_FILE not found in $BASEPATH/.env" >&2
+    exit 1
+fi
+
+# Split COMPOSE_FILE by colon
+IFS=':' read -ra parts <<< "$COMPOSE_FILE"
+
+total_backed_up=0
+total_failed=0
+total_skipped=0
+
+echo "Starting peer backup at $(date)"
+echo "Backup directory: $BACKUP_DIR"
+echo ""
+
+# Process each compose file
+for part in "${parts[@]}"; do
+    # Remove .yml extension if present for processing
+    compose_file="${part%.yml}.yml"
+    
+    # Check blacklist
+    include=true
+    for word in "${blacklist[@]}"; do
+        if echo "$compose_file" | grep -qE "$word"; then
+            include=false
+            break
+        fi
+    done
+    
+    if [ "$include" = false ]; then
+        total_skipped=$((total_skipped + 1))
+        continue
+    fi
+    
+    # Extract RPC paths from compose file
+    paths=$(extract_rpc_paths "$compose_file")
+    
+    if [ -z "$paths" ]; then
+        total_skipped=$((total_skipped + 1))
+        continue
+    fi
+    
+    # Process each path
+    path_found=false
+    for path in $paths; do
+        # Check path blacklist
+        if should_include_path "$path"; then
+            path_found=true
+            if backup_peers_from_path "$compose_file" "$path"; then
+                total_backed_up=$((total_backed_up + 1))
+            else
+                total_failed=$((total_failed + 1))
+            fi
+        fi
+    done
+    
+    if [ "$path_found" = false ]; then
+        total_skipped=$((total_skipped + 1))
+    fi
+done
+
+echo ""
+echo "=========================================="
+echo "Backup Summary"
+echo "=========================================="
+echo "Total nodes backed up: $total_backed_up"
+echo "Total nodes failed: $total_failed"
+echo "Total nodes skipped: $total_skipped"
+echo "Backup directory: $BACKUP_DIR"
+echo "Completed at $(date)"
+echo ""
+
+# Optional: Clean up old backups (keep last 30 days)
+if [ -n "$CLEANUP_OLD_BACKUPS" ] && [ "$CLEANUP_OLD_BACKUPS" = "true" ]; then
+    echo "Cleaning up backups older than 30 days..."
+    find "$BACKUP_DIR" -name "*.json" -type f -mtime +30 -delete
+    find "$BACKUP_DIR" -name "*.txt" -type f -mtime +30 -delete
+    echo "Cleanup complete"
+fi
+
+exit 0
+

list-peer-backups.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script to list available peer backups
+# Usage: ./list-peer-backups.sh [backup-directory] [filter]
+
+BASEPATH="$(dirname "$0")"
+BACKUP_DIR="${1:-$BASEPATH/peer-backups}"
+FILTER="${2:-}"
+
+if [ ! -d "$BACKUP_DIR" ]; then
+    echo "Backup directory does not exist: $BACKUP_DIR"
+    echo "Run backup-peers.sh first to create backups"
+    exit 1
+fi
+
+# Count files
+json_count=$(find "$BACKUP_DIR" -name "*.json" -type f | wc -l | tr -d ' ')
+txt_count=$(find "$BACKUP_DIR" -name "*.txt" -type f | wc -l | tr -d ' ')
+
+if [ "$json_count" -eq 0 ] && [ "$txt_count" -eq 0 ]; then
+    echo "No backups found in $BACKUP_DIR"
+    exit 0
+fi
+
+echo "Peer backups in: $BACKUP_DIR"
+echo "Total backups: $json_count JSON, $txt_count TXT"
+echo ""
+
+# List JSON backups with details
+if [ "$json_count" -gt 0 ]; then
+    echo "JSON Backups (with metadata):"
+    echo "=========================================="
+    
+    for backup_file in $(find "$BACKUP_DIR" -name "*.json" -type f | sort -r); do
+        if [ -n "$FILTER" ] && ! echo "$backup_file" | grep -qi "$FILTER"; then
+            continue
+        fi
+        
+        if command -v jq &> /dev/null; then
+            compose_file=$(jq -r '.compose_file // "unknown"' "$backup_file" 2>/dev/null)
+            rpc_path=$(jq -r '.rpc_path // "unknown"' "$backup_file" 2>/dev/null)
+            timestamp=$(jq -r '.timestamp // "unknown"' "$backup_file" 2>/dev/null)
+            peer_count=$(jq -r '.peer_count // 0' "$backup_file" 2>/dev/null)
+            rpc_url=$(jq -r '.rpc_url // "unknown"' "$backup_file" 2>/dev/null)
+            
+            file_size=$(du -h "$backup_file" | cut -f1)
+            file_date=$(stat -f "%Sm" -t "%Y-%m-%d %H:%M:%S" "$backup_file" 2>/dev/null || stat -c "%y" "$backup_file" 2>/dev/null | cut -d'.' -f1)
+            
+            echo "File: $(basename "$backup_file")"
+            echo "  Compose: $compose_file"
+            echo "  Path: $rpc_path"
+            echo "  URL: $rpc_url"
+            echo "  Peers: $peer_count"
+            echo "  Timestamp: $timestamp"
+            echo "  Size: $file_size"
+            echo "  Date: $file_date"
+            echo ""
+        else
+            # Fallback if jq is not available
+            file_size=$(du -h "$backup_file" | cut -f1)
+            file_date=$(stat -f "%Sm" -t "%Y-%m-%d %H:%M:%S" "$backup_file" 2>/dev/null || stat -c "%y" "$backup_file" 2>/dev/null | cut -d'.' -f1)
+            echo "$(basename "$backup_file") - $file_size - $file_date"
+        fi
+    done
+fi
+
+# Show usage examples
+echo ""
+echo "To restore a backup, use:"
+echo "  ./restore-peers.sh <backup-file> [rpc-url]"
+echo ""
+echo "Examples:"
+if [ "$json_count" -gt 0 ]; then
+    latest_json=$(find "$BACKUP_DIR" -name "*.json" -type f | sort -r | head -1)
+    if [ -n "$latest_json" ]; then
+        echo "  ./restore-peers.sh $latest_json"
+    fi
+fi
+if [ "$txt_count" -gt 0 ]; then
+    latest_txt=$(find "$BACKUP_DIR" -name "*.txt" -type f | sort -r | head -1)
+    if [ -n "$latest_txt" ]; then
+        if command -v jq &> /dev/null && [ -n "$latest_json" ]; then
+            rpc_url=$(jq -r '.rpc_url // ""' "$latest_json" 2>/dev/null)
+            if [ -n "$rpc_url" ]; then
+                echo "  ./restore-peers.sh $latest_txt $rpc_url"
+            fi
+        fi
+    fi
+fi
+

restore-peers.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+
+# Script to restore peers from a backup file
+# Usage: ./restore-peers.sh <backup-file> [rpc-url]
+#   backup-file: Path to backup JSON or TXT file
+#   rpc-url: Optional. If not provided, will extract from backup file
+
+BASEPATH="$(dirname "$0")"
+source $BASEPATH/.env
+
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 <backup-file> [rpc-url]"
+    echo ""
+    echo "  backup-file: Path to backup JSON or TXT file"
+    echo "  rpc-url: Optional. Target RPC URL to restore peers to"
+    echo "           If not provided, will use rpc_url from backup file"
+    echo ""
+    echo "Examples:"
+    echo "  $0 peer-backups/ethereum-mainnet__ethereum-mainnet-archive__20240101_120000.json"
+    echo "  $0 peer-backups/ethereum-mainnet__ethereum-mainnet-archive__20240101_120000.txt https://domain.com/ethereum-mainnet-archive"
+    exit 1
+fi
+
+BACKUP_FILE="$1"
+TARGET_URL="$2"
+
+if [ ! -f "$BACKUP_FILE" ]; then
+    echo "Error: Backup file not found: $BACKUP_FILE" >&2
+    exit 1
+fi
+
+# Determine file type and extract enodes
+if [[ "$BACKUP_FILE" == *.json ]]; then
+    # JSON backup file
+    if ! command -v jq &> /dev/null; then
+        echo "Error: jq is required to parse JSON backup files" >&2
+        exit 1
+    fi
+    
+    # Extract RPC URL from backup if not provided
+    if [ -z "$TARGET_URL" ]; then
+        TARGET_URL=$(jq -r '.rpc_url // empty' "$BACKUP_FILE" 2>/dev/null)
+        if [ -z "$TARGET_URL" ] || [ "$TARGET_URL" = "null" ]; then
+            echo "Error: Could not extract rpc_url from backup file and no target URL provided" >&2
+            exit 1
+        fi
+    fi
+    
+    # Extract enodes
+    enodes=$(jq -r '.peers[]?' "$BACKUP_FILE" 2>/dev/null)
+    
+    if [ -z "$enodes" ]; then
+        echo "Error: No peers found in backup file" >&2
+        exit 1
+    fi
+    
+    peer_count=$(jq -r '.peer_count // 0' "$BACKUP_FILE" 2>/dev/null)
+    compose_file=$(jq -r '.compose_file // "unknown"' "$BACKUP_FILE" 2>/dev/null)
+    rpc_path=$(jq -r '.rpc_path // "unknown"' "$BACKUP_FILE" 2>/dev/null)
+    timestamp=$(jq -r '.timestamp // "unknown"' "$BACKUP_FILE" 2>/dev/null)
+    
+    echo "Restoring peers from backup:"
+    echo "  Compose file: $compose_file"
+    echo "  RPC path: $rpc_path"
+    echo "  Timestamp: $timestamp"
+    echo "  Peer count: $peer_count"
+    echo "  Target URL: $TARGET_URL"
+    echo ""
+    
+elif [[ "$BACKUP_FILE" == *.txt ]]; then
+    # TXT backup file (one enode per line)
+    enodes=$(grep -v '^$' "$BACKUP_FILE" | grep -v '^null$')
+    
+    if [ -z "$enodes" ]; then
+        echo "Error: No peers found in backup file" >&2
+        exit 1
+    fi
+    
+    peer_count=$(echo "$enodes" | wc -l | tr -d ' ')
+    
+    if [ -z "$TARGET_URL" ]; then
+        echo "Error: Target RPC URL required for TXT backup files" >&2
+        echo "Usage: $0 <backup-file> <rpc-url>" >&2
+        exit 1
+    fi
+    
+    echo "Restoring peers from backup:"
+    echo "  Backup file: $BACKUP_FILE"
+    echo "  Peer count: $peer_count"
+    echo "  Target URL: $TARGET_URL"
+    echo ""
+    
+else
+    echo "Error: Unsupported backup file format. Expected .json or .txt" >&2
+    exit 1
+fi
+
+# Confirm before proceeding
+read -p "Do you want to restore $peer_count peer(s) to $TARGET_URL? (y/N) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Restore cancelled"
+    exit 0
+fi
+
+# Restore peers
+success_count=0
+failed_count=0
+skipped_count=0
+
+echo ""
+echo "Restoring peers..."
+
+while IFS= read -r enode; do
+    if [ -z "$enode" ] || [ "$enode" = "null" ]; then
+        skipped_count=$((skipped_count + 1))
+        continue
+    fi
+    
+    # Check if peer is reachable (optional, can be slow)
+    if [ -f "$BASEPATH/check-enode.sh" ]; then
+        if ! "$BASEPATH/check-enode.sh" "$enode" --target "$TARGET_URL" > /dev/null 2>&1; then
+            echo "⚠ Skipping unreachable peer: ${enode:0:50}..."
+            skipped_count=$((skipped_count + 1))
+            continue
+        fi
+    fi
+    
+    echo -n "Adding peer: ${enode:0:50}... "
+    
+    result=$(curl --ipv4 -X POST -H "Content-Type: application/json" \
+        --silent --max-time 10 \
+        --data "{\"jsonrpc\":\"2.0\",\"method\":\"admin_addPeer\",\"params\":[\"${enode}\"],\"id\":1}" \
+        "$TARGET_URL" 2>/dev/null | jq -r '.result // .error.message // "unknown error"' 2>/dev/null)
+    
+    if [ "$result" = "true" ] || [ "$result" = "null" ]; then
+        echo "✓ Success"
+        success_count=$((success_count + 1))
+    else
+        echo "✗ Failed: $result"
+        failed_count=$((failed_count + 1))
+    fi
+    
+    # Small delay to avoid overwhelming the node
+    sleep 0.1
+done <<< "$enodes"
+
+echo ""
+echo "=========================================="
+echo "Restore Summary"
+echo "=========================================="
+echo "Successful: $success_count/$peer_count"
+echo "Failed: $failed_count/$peer_count"
+echo "Skipped: $skipped_count/$peer_count"
+echo ""
+
+if [ $failed_count -gt 0 ]; then
+    exit 1
+fi
+
+exit 0
+