Peer persistence hooks: capture before stop, reinject after start

Clients (esp. discovery-disabled OP-stack EL) lose peers across recreate.
Generic peer-capture/peer-reinject hooks + force-recreate wiring restore
the peerset from local peer-state via in-network admin_addPeer.

Co-authored-by: Cursor <cursoragent@cursor.com>

.gitignore

@@ -11,3 +11,7 @@ peer-backups/
 snapshots/
 *.bak
 *.swp
+
+# Task C local peer capture state
+peer-state/
+

force-recreate.sh

@@ -1,8 +1,31 @@
 #!/bin/bash
+# force-recreate.sh — recreate one node's compose services (keeps volumes).
+# Runs peer-capture (.before-stop) before and peer-reinject (.after-start) after
+# so EL peer sets survive the restart (lisk-class --disable-discovery fix).
 
 if [ -z "$1" ] || [ ! -f "/root/rpc/$1.yml" ]; then
     echo "Error: Either no argument provided or /root/rpc/$1.yml does not exist."
     exit 1
 fi
 
-docker compose up -d --force-recreate $(cat /root/rpc/$1.yml | yaml2json - | jq '.services' | jq -r 'keys[]' | tr '\n' ' ')
+COMPOSE_PATH="$1"
+BASEPATH="/root/rpc"
+cd "$BASEPATH" || exit 1
+
+run_hook() {
+    local kind="$1"   # before-stop | after-start
+    local generic="$2"
+    if [ -f "${BASEPATH}/${COMPOSE_PATH}.${kind}" ]; then
+        /bin/bash "${BASEPATH}/${COMPOSE_PATH}.${kind}" || \
+            echo "WARN: ${COMPOSE_PATH}.${kind} exited $?" >&2
+    elif [ -x "${BASEPATH}/${generic}" ]; then
+        "${BASEPATH}/${generic}" "$COMPOSE_PATH" || \
+            echo "WARN: ${generic} exited $?" >&2
+    fi
+}
+
+run_hook before-stop peer-capture.sh
+
+docker compose up -d --force-recreate $(cat "/root/rpc/${COMPOSE_PATH}.yml" | yaml2json - | jq '.services' | jq -r 'keys[]' | tr '\n' ' ')
+
+run_hook after-start peer-reinject.sh

peer-capture.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+# peer-capture.sh — generic .before-stop hook: snapshot admin peers for reinject.
+# In-network vantage (rpc_chains curl): public gateway rejects admin; some hosts
+# firewall host→bridge-IP. Client-aware peer list → /root/rpc/peer-state/<path>.json
+#
+# Usage: peer-capture.sh <compose-path>
+#   compose-path: e.g. op/reth/lisk-mainnet-op-reth-pruned-trace
+set -euo pipefail
+
+COMPOSE_PATH="${1:-}"
+if [ -z "$COMPOSE_PATH" ]; then
+    echo "Usage: $0 <compose-path>" >&2
+    exit 1
+fi
+
+BASEPATH="$(cd "$(dirname "$0")" && pwd)"
+STATE_DIR="$BASEPATH/peer-state"
+mkdir -p "$STATE_DIR"
+SAFE="$(echo "$COMPOSE_PATH" | sed 's|[^a-zA-Z0-9._-]|_|g')"
+OUT="$STATE_DIR/${SAFE}.json"
+CURL_IMAGE="${PEER_CURL_IMAGE:-curlimages/curl:8.5.0}"
+LOG="${PEER_HOOK_LOG:-/var/log/peer-hooks.log}"
+
+log() { echo "$(date '+%F %T') peer-capture[$COMPOSE_PATH]: $*" | tee -a "$LOG" >&2; }
+
+# Documented skips
+case "$COMPOSE_PATH" in
+    *nitro*|*zksync*|*zk-sync*)
+        log "skip (nitro/zksync)"
+        exit 0
+        ;;
+esac
+
+BASE="$(basename "$COMPOSE_PATH" | sed 's/--.*//')"
+# Find execution container (same heuristics as characterize-node exec_container)
+EL=""
+while read -r n; do
+    [ -z "$n" ] && continue
+    case "$n" in
+        rpc-*) ;;
+        *) continue ;;
+    esac
+    case "$n" in
+        *-node-*|*-relay-*) continue ;;
+    esac
+    core="$(echo "$n" | sed -E 's/^rpc-(.+)-[0-9]+$/\1/')"
+    if [ "$BASE" = "$core" ] || [[ "$BASE" == "$core"-* ]]; then
+        EL="$n"
+        break
+    fi
+done < <(docker ps --format '{{.Names}}')
+
+if [ -z "$EL" ]; then
+    log "no execution container for $BASE"
+    exit 0
+fi
+
+IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$EL" 2>/dev/null || true)"
+if [ -z "$IP" ]; then
+    log "no IP for $EL"
+    exit 0
+fi
+
+docker rm -f peer-hook-probe >/dev/null 2>&1 || true
+docker image inspect "$CURL_IMAGE" >/dev/null 2>&1 || docker pull -q "$CURL_IMAGE" >/dev/null
+docker run -d --name peer-hook-probe --network rpc_chains "$CURL_IMAGE" sleep 90 >/dev/null
+
+rpc() {
+    local method="$1"
+    docker exec peer-hook-probe curl -s --max-time 8 -X POST \
+        -H 'Content-Type: application/json' \
+        --data "{\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":[],\"id\":1}" \
+        "http://${IP}:8545" 2>/dev/null || true
+}
+
+INFO="$(rpc admin_nodeInfo)"
+CLIENT_NAME="$(echo "$INFO" | jq -r '.result.name // empty' 2>/dev/null || true)"
+PEERS_JSON="$(rpc admin_peers)"
+PEER_COUNT="$(echo "$PEERS_JSON" | jq -r '.result | length' 2>/dev/null || echo 0)"
+
+# Detect client type (connect-peers.sh logic)
+NAME_LC="$(echo "$CLIENT_NAME" | tr '[:upper:]' '[:lower:]')"
+CLIENT_TYPE="unknown"
+case "$NAME_LC" in
+    *geth*) CLIENT_TYPE="geth" ;;
+    *reth*) CLIENT_TYPE="reth" ;;
+    *erigon*) CLIENT_TYPE="erigon" ;;
+    *nethermind*) CLIENT_TYPE="nethermind" ;;
+    *besu*) CLIENT_TYPE="besu" ;;
+esac
+
+TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+echo "$PEERS_JSON" | jq -c --arg path "$COMPOSE_PATH" --arg ts "$TS" \
+    --arg el "$EL" --arg cn "$CLIENT_NAME" --arg ct "$CLIENT_TYPE" \
+    --argjson pc "${PEER_COUNT:-0}" '
+{
+  node_path: $path,
+  captured_at: $ts,
+  container: $el,
+  client_name: $cn,
+  client_type: $ct,
+  peer_count: $pc,
+  peers: [(.result // [])[] | select(.enode != null) | {
+    id: .enode,
+    client: (.name // ""),
+    layer: "execution"
+  }]
+}' > "$OUT.tmp" 2>/dev/null || {
+    log "admin_peers failed; writing empty seed"
+    cat > "$OUT.tmp" <<EOF
+{"node_path":"$COMPOSE_PATH","captured_at":"$TS","container":"$EL","client_name":"$CLIENT_NAME","client_type":"$CLIENT_TYPE","peer_count":0,"peers":[]}
+EOF
+}
+mv -f "$OUT.tmp" "$OUT"
+# also keep a plain enode list for easy playback
+jq -r '.peers[].id' "$OUT" > "${OUT%.json}.enodes" 2>/dev/null || true
+
+docker rm -f peer-hook-probe >/dev/null 2>&1 || true
+log "captured ${PEER_COUNT:-0} peer(s) → $OUT"
+exit 0

peer-reinject.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+# peer-reinject.sh — generic .after-start hook: re-seed peers from peer-capture state.
+# Uses admin_addStaticPeer for geth, admin_addPeer otherwise (connect-peers.sh).
+# In-network vantage (rpc_chains). Idempotent; best-effort (never fails the restart).
+#
+# Usage: peer-reinject.sh <compose-path>
+set -euo pipefail
+
+COMPOSE_PATH="${1:-}"
+if [ -z "$COMPOSE_PATH" ]; then
+    echo "Usage: $0 <compose-path>" >&2
+    exit 1
+fi
+
+BASEPATH="$(cd "$(dirname "$0")" && pwd)"
+STATE_DIR="$BASEPATH/peer-state"
+SAFE="$(echo "$COMPOSE_PATH" | sed 's|[^a-zA-Z0-9._-]|_|g')"
+STATE="$STATE_DIR/${SAFE}.json"
+CURL_IMAGE="${PEER_CURL_IMAGE:-curlimages/curl:8.5.0}"
+LOG="${PEER_HOOK_LOG:-/var/log/peer-hooks.log}"
+
+log() { echo "$(date '+%F %T') peer-reinject[$COMPOSE_PATH]: $*" | tee -a "$LOG" >&2; }
+
+case "$COMPOSE_PATH" in
+    *nitro*|*zksync*|*zk-sync*)
+        log "skip (nitro/zksync)"
+        exit 0
+        ;;
+esac
+
+if [ ! -f "$STATE" ]; then
+    log "no capture state at $STATE — nothing to reinject"
+    exit 0
+fi
+
+CLIENT_TYPE="$(jq -r '.client_type // "unknown"' "$STATE" 2>/dev/null || echo unknown)"
+mapfile -t ENODES < <(jq -r '.peers[].id // empty' "$STATE" 2>/dev/null | grep -E '^enode://' | head -n 50 || true)
+if [ "${#ENODES[@]}" -eq 0 ]; then
+    log "no enodes in $STATE"
+    exit 0
+fi
+
+BASE="$(basename "$COMPOSE_PATH" | sed 's/--.*//')"
+EL=""
+# wait briefly for container after recreate
+for _ in 1 2 3 4 5 6 7 8 9 10; do
+    while read -r n; do
+        [ -z "$n" ] && continue
+        case "$n" in rpc-*) ;; *) continue ;; esac
+        case "$n" in *-node-*|*-relay-*) continue ;; esac
+        core="$(echo "$n" | sed -E 's/^rpc-(.+)-[0-9]+$/\1/')"
+        if [ "$BASE" = "$core" ] || [[ "$BASE" == "$core"-* ]]; then
+            EL="$n"
+            break
+        fi
+    done < <(docker ps --format '{{.Names}}')
+    [ -n "$EL" ] && break
+    sleep 3
+done
+
+if [ -z "$EL" ]; then
+    log "no execution container yet — giving up"
+    exit 0
+fi
+
+IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$EL" 2>/dev/null || true)"
+if [ -z "$IP" ]; then
+    log "no IP for $EL"
+    exit 0
+fi
+
+# wait for RPC
+for _ in 1 2 3 4 5 6 7 8 9 10 11 12; do
+    docker rm -f peer-hook-probe >/dev/null 2>&1 || true
+    docker image inspect "$CURL_IMAGE" >/dev/null 2>&1 || docker pull -q "$CURL_IMAGE" >/dev/null
+    docker run -d --name peer-hook-probe --network rpc_chains "$CURL_IMAGE" sleep 180 >/dev/null
+    if docker exec peer-hook-probe curl -s --max-time 5 -X POST \
+        -H 'Content-Type: application/json' \
+        --data '{"jsonrpc":"2.0","method":"net_version","params":[],"id":1}' \
+        "http://${IP}:8545" 2>/dev/null | grep -q result; then
+        break
+    fi
+    docker rm -f peer-hook-probe >/dev/null 2>&1 || true
+    sleep 5
+done
+
+add_peer() {
+    local enode="$1"
+    local method
+    if [ "$CLIENT_TYPE" = "geth" ]; then
+        method="admin_addStaticPeer"
+    else
+        method="admin_addPeer"
+    fi
+    local payload
+    payload="$(jq -nc --arg m "$method" --arg e "$enode" \
+        '{jsonrpc:"2.0",method:$m,params:[$e],id:1}')"
+    local resp
+    resp="$(docker exec peer-hook-probe curl -s --max-time 8 -X POST \
+        -H 'Content-Type: application/json' --data "$payload" \
+        "http://${IP}:8545" 2>/dev/null || true)"
+    if echo "$resp" | grep -qi 'method not found'; then
+        if [ "$method" = "admin_addStaticPeer" ]; then
+            method="admin_addPeer"
+        else
+            method="admin_addStaticPeer"
+        fi
+        payload="$(jq -nc --arg m "$method" --arg e "$enode" \
+            '{jsonrpc:"2.0",method:$m,params:[$e],id:1}')"
+        resp="$(docker exec peer-hook-probe curl -s --max-time 8 -X POST \
+            -H 'Content-Type: application/json' --data "$payload" \
+            "http://${IP}:8545" 2>/dev/null || true)"
+    fi
+    if echo "$resp" | grep -q '"result":true'; then
+        return 0
+    fi
+    return 1
+}
+
+ADDED=0
+FAIL=0
+for enode in "${ENODES[@]}"; do
+    if add_peer "$enode"; then
+        ADDED=$((ADDED + 1))
+    else
+        FAIL=$((FAIL + 1))
+    fi
+done
+
+# post count
+COUNT="$(docker exec peer-hook-probe curl -s --max-time 8 -X POST \
+    -H 'Content-Type: application/json' \
+    --data '{"jsonrpc":"2.0","method":"admin_peers","params":[],"id":1}' \
+    "http://${IP}:8545" 2>/dev/null | jq -r '.result | length' 2>/dev/null || echo '?')"
+
+docker rm -f peer-hook-probe >/dev/null 2>&1 || true
+log "reinjected added=$ADDED fail=$FAIL now_peers=$COUNT (from ${#ENODES[@]} enodes)"
+exit 0