From 291996069789d06795594acb9ebf9879e24d354d Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 19 Jul 2026 05:25:27 +0000 Subject: [PATCH] Peer persistence hooks: capture before stop, reinject after start Clients (esp. discovery-disabled OP-stack EL) lose peers across recreate. Generic peer-capture/peer-reinject hooks + force-recreate wiring restore the peerset from local peer-state via in-network admin_addPeer. Co-authored-by: Cursor --- .gitignore | 4 ++ force-recreate.sh | 25 ++++++++- peer-capture.sh | 120 ++++++++++++++++++++++++++++++++++++++++ peer-reinject.sh | 138 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100755 peer-capture.sh create mode 100755 peer-reinject.sh diff --git a/.gitignore b/.gitignore index fb321375..d733dc44 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ peer-backups/ snapshots/ *.bak *.swp + +# Task C local peer capture state +peer-state/ + diff --git a/force-recreate.sh b/force-recreate.sh index 40a500fc..c64058ab 100755 --- a/force-recreate.sh +++ b/force-recreate.sh @@ -1,8 +1,31 @@ #!/bin/bash +# force-recreate.sh — recreate one node's compose services (keeps volumes). +# Runs peer-capture (.before-stop) before and peer-reinject (.after-start) after +# so EL peer sets survive the restart (lisk-class --disable-discovery fix). if [ -z "$1" ] || [ ! -f "/root/rpc/$1.yml" ]; then echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist." exit 1 fi -docker compose up -d --force-recreate $(cat /root/rpc/$1.yml | yaml2json - | jq '.services' | jq -r 'keys[]' | tr '\n' ' ') +COMPOSE_PATH="$1" +BASEPATH="/root/rpc" +cd "$BASEPATH" || exit 1 + +run_hook() { + local kind="$1" # before-stop | after-start + local generic="$2" + if [ -f "${BASEPATH}/${COMPOSE_PATH}.${kind}" ]; then + /bin/bash "${BASEPATH}/${COMPOSE_PATH}.${kind}" || \ + echo "WARN: ${COMPOSE_PATH}.${kind} exited $?" >&2 + elif [ -x "${BASEPATH}/${generic}" ]; then + "${BASEPATH}/${generic}" "$COMPOSE_PATH" || \ + echo "WARN: ${generic} exited $?" >&2 + fi +} + +run_hook before-stop peer-capture.sh + +docker compose up -d --force-recreate $(cat "/root/rpc/${COMPOSE_PATH}.yml" | yaml2json - | jq '.services' | jq -r 'keys[]' | tr '\n' ' ') + +run_hook after-start peer-reinject.sh diff --git a/peer-capture.sh b/peer-capture.sh new file mode 100755 index 00000000..bd627d9d --- /dev/null +++ b/peer-capture.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# peer-capture.sh — generic .before-stop hook: snapshot admin peers for reinject. +# In-network vantage (rpc_chains curl): public gateway rejects admin; some hosts +# firewall host→bridge-IP. Client-aware peer list → /root/rpc/peer-state/.json +# +# Usage: peer-capture.sh +# compose-path: e.g. op/reth/lisk-mainnet-op-reth-pruned-trace +set -euo pipefail + +COMPOSE_PATH="${1:-}" +if [ -z "$COMPOSE_PATH" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +BASEPATH="$(cd "$(dirname "$0")" && pwd)" +STATE_DIR="$BASEPATH/peer-state" +mkdir -p "$STATE_DIR" +SAFE="$(echo "$COMPOSE_PATH" | sed 's|[^a-zA-Z0-9._-]|_|g')" +OUT="$STATE_DIR/${SAFE}.json" +CURL_IMAGE="${PEER_CURL_IMAGE:-curlimages/curl:8.5.0}" +LOG="${PEER_HOOK_LOG:-/var/log/peer-hooks.log}" + +log() { echo "$(date '+%F %T') peer-capture[$COMPOSE_PATH]: $*" | tee -a "$LOG" >&2; } + +# Documented skips +case "$COMPOSE_PATH" in + *nitro*|*zksync*|*zk-sync*) + log "skip (nitro/zksync)" + exit 0 + ;; +esac + +BASE="$(basename "$COMPOSE_PATH" | sed 's/--.*//')" +# Find execution container (same heuristics as characterize-node exec_container) +EL="" +while read -r n; do + [ -z "$n" ] && continue + case "$n" in + rpc-*) ;; + *) continue ;; + esac + case "$n" in + *-node-*|*-relay-*) continue ;; + esac + core="$(echo "$n" | sed -E 's/^rpc-(.+)-[0-9]+$/\1/')" + if [ "$BASE" = "$core" ] || [[ "$BASE" == "$core"-* ]]; then + EL="$n" + break + fi +done < <(docker ps --format '{{.Names}}') + +if [ -z "$EL" ]; then + log "no execution container for $BASE" + exit 0 +fi + +IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$EL" 2>/dev/null || true)" +if [ -z "$IP" ]; then + log "no IP for $EL" + exit 0 +fi + +docker rm -f peer-hook-probe >/dev/null 2>&1 || true +docker image inspect "$CURL_IMAGE" >/dev/null 2>&1 || docker pull -q "$CURL_IMAGE" >/dev/null +docker run -d --name peer-hook-probe --network rpc_chains "$CURL_IMAGE" sleep 90 >/dev/null + +rpc() { + local method="$1" + docker exec peer-hook-probe curl -s --max-time 8 -X POST \ + -H 'Content-Type: application/json' \ + --data "{\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":[],\"id\":1}" \ + "http://${IP}:8545" 2>/dev/null || true +} + +INFO="$(rpc admin_nodeInfo)" +CLIENT_NAME="$(echo "$INFO" | jq -r '.result.name // empty' 2>/dev/null || true)" +PEERS_JSON="$(rpc admin_peers)" +PEER_COUNT="$(echo "$PEERS_JSON" | jq -r '.result | length' 2>/dev/null || echo 0)" + +# Detect client type (connect-peers.sh logic) +NAME_LC="$(echo "$CLIENT_NAME" | tr '[:upper:]' '[:lower:]')" +CLIENT_TYPE="unknown" +case "$NAME_LC" in + *geth*) CLIENT_TYPE="geth" ;; + *reth*) CLIENT_TYPE="reth" ;; + *erigon*) CLIENT_TYPE="erigon" ;; + *nethermind*) CLIENT_TYPE="nethermind" ;; + *besu*) CLIENT_TYPE="besu" ;; +esac + +TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "$PEERS_JSON" | jq -c --arg path "$COMPOSE_PATH" --arg ts "$TS" \ + --arg el "$EL" --arg cn "$CLIENT_NAME" --arg ct "$CLIENT_TYPE" \ + --argjson pc "${PEER_COUNT:-0}" ' +{ + node_path: $path, + captured_at: $ts, + container: $el, + client_name: $cn, + client_type: $ct, + peer_count: $pc, + peers: [(.result // [])[] | select(.enode != null) | { + id: .enode, + client: (.name // ""), + layer: "execution" + }] +}' > "$OUT.tmp" 2>/dev/null || { + log "admin_peers failed; writing empty seed" + cat > "$OUT.tmp" < "${OUT%.json}.enodes" 2>/dev/null || true + +docker rm -f peer-hook-probe >/dev/null 2>&1 || true +log "captured ${PEER_COUNT:-0} peer(s) → $OUT" +exit 0 diff --git a/peer-reinject.sh b/peer-reinject.sh new file mode 100755 index 00000000..9c62dd1e --- /dev/null +++ b/peer-reinject.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# peer-reinject.sh — generic .after-start hook: re-seed peers from peer-capture state. +# Uses admin_addStaticPeer for geth, admin_addPeer otherwise (connect-peers.sh). +# In-network vantage (rpc_chains). Idempotent; best-effort (never fails the restart). +# +# Usage: peer-reinject.sh +set -euo pipefail + +COMPOSE_PATH="${1:-}" +if [ -z "$COMPOSE_PATH" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +BASEPATH="$(cd "$(dirname "$0")" && pwd)" +STATE_DIR="$BASEPATH/peer-state" +SAFE="$(echo "$COMPOSE_PATH" | sed 's|[^a-zA-Z0-9._-]|_|g')" +STATE="$STATE_DIR/${SAFE}.json" +CURL_IMAGE="${PEER_CURL_IMAGE:-curlimages/curl:8.5.0}" +LOG="${PEER_HOOK_LOG:-/var/log/peer-hooks.log}" + +log() { echo "$(date '+%F %T') peer-reinject[$COMPOSE_PATH]: $*" | tee -a "$LOG" >&2; } + +case "$COMPOSE_PATH" in + *nitro*|*zksync*|*zk-sync*) + log "skip (nitro/zksync)" + exit 0 + ;; +esac + +if [ ! -f "$STATE" ]; then + log "no capture state at $STATE — nothing to reinject" + exit 0 +fi + +CLIENT_TYPE="$(jq -r '.client_type // "unknown"' "$STATE" 2>/dev/null || echo unknown)" +mapfile -t ENODES < <(jq -r '.peers[].id // empty' "$STATE" 2>/dev/null | grep -E '^enode://' | head -n 50 || true) +if [ "${#ENODES[@]}" -eq 0 ]; then + log "no enodes in $STATE" + exit 0 +fi + +BASE="$(basename "$COMPOSE_PATH" | sed 's/--.*//')" +EL="" +# wait briefly for container after recreate +for _ in 1 2 3 4 5 6 7 8 9 10; do + while read -r n; do + [ -z "$n" ] && continue + case "$n" in rpc-*) ;; *) continue ;; esac + case "$n" in *-node-*|*-relay-*) continue ;; esac + core="$(echo "$n" | sed -E 's/^rpc-(.+)-[0-9]+$/\1/')" + if [ "$BASE" = "$core" ] || [[ "$BASE" == "$core"-* ]]; then + EL="$n" + break + fi + done < <(docker ps --format '{{.Names}}') + [ -n "$EL" ] && break + sleep 3 +done + +if [ -z "$EL" ]; then + log "no execution container yet — giving up" + exit 0 +fi + +IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$EL" 2>/dev/null || true)" +if [ -z "$IP" ]; then + log "no IP for $EL" + exit 0 +fi + +# wait for RPC +for _ in 1 2 3 4 5 6 7 8 9 10 11 12; do + docker rm -f peer-hook-probe >/dev/null 2>&1 || true + docker image inspect "$CURL_IMAGE" >/dev/null 2>&1 || docker pull -q "$CURL_IMAGE" >/dev/null + docker run -d --name peer-hook-probe --network rpc_chains "$CURL_IMAGE" sleep 180 >/dev/null + if docker exec peer-hook-probe curl -s --max-time 5 -X POST \ + -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","method":"net_version","params":[],"id":1}' \ + "http://${IP}:8545" 2>/dev/null | grep -q result; then + break + fi + docker rm -f peer-hook-probe >/dev/null 2>&1 || true + sleep 5 +done + +add_peer() { + local enode="$1" + local method + if [ "$CLIENT_TYPE" = "geth" ]; then + method="admin_addStaticPeer" + else + method="admin_addPeer" + fi + local payload + payload="$(jq -nc --arg m "$method" --arg e "$enode" \ + '{jsonrpc:"2.0",method:$m,params:[$e],id:1}')" + local resp + resp="$(docker exec peer-hook-probe curl -s --max-time 8 -X POST \ + -H 'Content-Type: application/json' --data "$payload" \ + "http://${IP}:8545" 2>/dev/null || true)" + if echo "$resp" | grep -qi 'method not found'; then + if [ "$method" = "admin_addStaticPeer" ]; then + method="admin_addPeer" + else + method="admin_addStaticPeer" + fi + payload="$(jq -nc --arg m "$method" --arg e "$enode" \ + '{jsonrpc:"2.0",method:$m,params:[$e],id:1}')" + resp="$(docker exec peer-hook-probe curl -s --max-time 8 -X POST \ + -H 'Content-Type: application/json' --data "$payload" \ + "http://${IP}:8545" 2>/dev/null || true)" + fi + if echo "$resp" | grep -q '"result":true'; then + return 0 + fi + return 1 +} + +ADDED=0 +FAIL=0 +for enode in "${ENODES[@]}"; do + if add_peer "$enode"; then + ADDED=$((ADDED + 1)) + else + FAIL=$((FAIL + 1)) + fi +done + +# post count +COUNT="$(docker exec peer-hook-probe curl -s --max-time 8 -X POST \ + -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","method":"admin_peers","params":[],"id":1}' \ + "http://${IP}:8545" 2>/dev/null | jq -r '.result | length' 2>/dev/null || echo '?')" + +docker rm -f peer-hook-probe >/dev/null 2>&1 || true +log "reinjected added=$ADDED fail=$FAIL now_peers=$COUNT (from ${#ENODES[@]} enodes)" +exit 0