diff --git a/README.md b/README.md index 0dc15f12..a6fe04ab 100644 --- a/README.md +++ b/README.md @@ -831,6 +831,7 @@ debugging reference, see [VIBE.md](VIBE.md). | `force-recreate.sh ` | Restart a node | | `backup-node.sh ` | Create backup | | `restore-volumes.sh ` | Restore from backup | +| `node-facts.sh` | Emit JSON node facts documents for monitoring and inventory | --- diff --git a/node-facts-test/ethereum-mainnet-geth-pruned.yml b/node-facts-test/ethereum-mainnet-geth-pruned.yml new file mode 100644 index 00000000..ad766331 --- /dev/null +++ b/node-facts-test/ethereum-mainnet-geth-pruned.yml @@ -0,0 +1,11 @@ +version: "3" +services: + client: + image: ethereum/client-go:v1.13.0 + labels: + - "traefik.http.routers.ethereum-mainnet-geth-pruned.rule=PathPrefix(`/ethereum-mainnet-geth-pruned`)" + - "traefik.http.middlewares.ethereum-mainnet-geth-pruned.stripprefix.prefixes=/ethereum-mainnet-geth-pruned" + x-upstreams: + network: ethereum + chain: ethereum-mainnet +chain: ethereum-mainnet diff --git a/node-facts-test/path-blacklist.txt b/node-facts-test/path-blacklist.txt new file mode 100644 index 00000000..41f6f095 --- /dev/null +++ b/node-facts-test/path-blacklist.txt @@ -0,0 +1,6 @@ +lighthouse +prism +prysm +beacon +nimbus +ws diff --git a/node-facts.sh b/node-facts.sh new file mode 100755 index 00000000..8ec1b2a8 --- /dev/null +++ b/node-facts.sh @@ -0,0 +1,904 @@ +#!/bin/bash + +# node-facts.sh - Produces JSON 'node facts' documents for blockchain nodes +# Usage: node-facts.sh [node_path] [--pretty] +# With node_path: produces one document for the specified node +# Without node_path: produces JSON lines (one per node on the host) +# --pretty: indent JSON output + +BASEPATH="$(dirname "$0")" + +# Source the environment - this gives us DOMAIN, NO_SSL, COMPOSE_FILE +if [ ! -f "$BASEPATH/.env" ]; then + echo '{"schema":1,"at":null,"host":null,"node_path":null,"network":null,"client":null,"version":null,"peers":null,"schedule":null,"signals":null,"head":null,"timing_ms":null,"error":"cannot read .env"}' >&2 + exit 2 +fi +source "$BASEPATH/.env" + +# Check if COMPOSE_FILE is empty +if [ -z "$COMPOSE_FILE" ]; then + echo '{"schema":1,"at":null,"host":null,"node_path":null,"network":null,"client":null,"version":null,"peers":null,"schedule":null,"signals":null,"head":null,"timing_ms":null,"error":"COMPOSE_FILE is empty"}' >&2 + exit 2 +fi + +# Handle NO_SSL/PROTO/DOMAIN +if [ -n "$NO_SSL" ]; then + PROTO="http" + DOMAIN="${DOMAIN:-0.0.0.0}" +else + PROTO="https" +fi + +# Load blacklist +blacklist=() +if [ -f "$BASEPATH/path-blacklist.txt" ]; then + while IFS= read -r line || [ -n "$line" ]; do + # Skip empty lines and comments + line="$(echo "$line" | sed 's/#.*//' | xargs)" + [ -z "$line" ] && continue + blacklist+=("$line") + done < "$BASEPATH/path-blacklist.txt" +fi + +# Pretty flag +PRETTY=false +args=("$@") +processed_args=() +node_path="" + +for arg in "${args[@]}"; do + case "$arg" in + --pretty) + PRETTY=true + ;; + *) + if [ -z "$node_path" ]; then + node_path="$arg" + else + processed_args+=("$arg") + fi + ;; + esac +done + +# If we have unprocessed args or node_path contains invalid chars, error +if [ ${#processed_args[@]} -gt 0 ]; then + echo "error: unknown arguments: ${processed_args[*]}" >&2 + exit 1 +fi + +# Function to check if path should be included (not in blacklist) +should_include() { + local path="$1" + for word in "${blacklist[@]}"; do + if echo "$path" | grep -qE "$word"; then + return 1 + fi + done + return 0 +} + +# Function to get all node compose files +get_node_compose_files() { + # Use COMPOSE_FILE from .env + IFS=':' read -ra parts <<< "$COMPOSE_FILE" + for part in "${parts[@]}"; do + # Remove leading ./ and trailing .yml if present + part="$(echo "$part" | sed 's|^\./||; s|\.yml$||')" + # Only include if it's a real compose file (contains hyphens, not base/rpc/etc) + if echo "$part" | grep -q "-" && should_include "$part"; then + echo "$part" + fi + done +} + +# Function to extract network from compose file or node_path +get_network() { + local node_path="$1" + # Try to get from x-upstreams chain label in compose file + local compose_file="$BASEPATH/${node_path}.yml" + if [ -f "$compose_file" ]; then + # Look for chain: label + local chain_slug=$(grep -oP '^\s*chain:\s*\K\S+' "$compose_file" 2>/dev/null | head -1) + if [ -n "$chain_slug" ]; then + echo "$chain_slug" + return + fi + + # Fallback: look for x-upstreams network + local network=$(grep -oP 'x-upstreams\.network:\s*\K\S+' "$compose_file" 2>/dev/null | head -1) + if [ -n "$network" ]; then + echo "$network" + return + fi + fi + + # Extract from node_path: typically network-chain-client-... + echo "$node_path" | cut -d'-' -f1 +} + +# Function to extract client info from compose file +get_client_info() { + local node_path="$1" + local compose_file="$BASEPATH/${node_path}.yml" + + local client_name="" + local client_image="" + local client_tag="" + local container_name="" + + if [ -f "$compose_file" ]; then + # Get the main service name (usually the first non-infra service) + local services=$(grep -oP '^\s+\K\w+' "$compose_file" 2>/dev/null | head -1) + + if [ -n "$services" ]; then + # Try to find image for this service + local image_line=$(grep -A 20 "^\s+${services}:" "$compose_file" | grep -oP 'image:\s*\K[^\s]+' | head -1) + if [ -n "$image_line" ]; then + # Parse image:tag + client_image="$(echo "$image_line" | cut -d':' -f1)" + client_tag="$(echo "$image_line" | cut -d':' -f2)" + fi + + container_name="${node_path}_${services}_1" + fi + fi + + # Determine client family from image + if [ -z "$client_name" ]; then + client_name="unknown" + if echo "$client_image" | grep -qi "avalanchego"; then + client_name="avalanchego" + elif echo "$client_image" | grep -qi "op-node"; then + client_name="op-node" + elif echo "$client_image" | grep -qi "kona"; then + client_name="kona" + elif echo "$client_image" | grep -qiE "(lighthouse|prysm|teku|nimbus|lodestar)"; then + client_name="beacon" + elif echo "$client_image" | grep -qiE "(geth|go-ethereum)"; then + client_name="geth" + elif echo "$client_image" | grep -qi "op-geth"; then + client_name="op-geth" + elif echo "$client_image" | grep -qi "bor"; then + client_name="bor" + elif echo "$client_image" | grep -qi "erigon"; then + client_name="erigon" + elif echo "$client_image" | grep -qi "nethermind"; then + client_name="nethermind" + elif echo "$client_image" | grep -qi "besu"; then + client_name="besu" + elif echo "$client_image" | grep -qi "reth"; then + client_name="reth" + elif echo "$client_image" | grep -qi "op-reth"; then + client_name="op-reth" + elif echo "$client_image" | grep -qi "nitro"; then + client_name="nitro" + elif echo "$client_image" | grep -qi "cometbft"; then + client_name="cometbft" + elif echo "$client_image" | grep -qi "zksync"; then + client_name="zksync" + fi + fi + + # Output as JSON object + printf '{"name":"%s","image":"%s","tag":"%s","container":"%s"}' \ + "$client_name" "$client_image" "$client_tag" "$container_name" +} + +# Function to get traefik path for a node +get_traefik_path() { + local node_path="$1" + local compose_file="$BASEPATH/${node_path}.yml" + + if [ -f "$compose_file" ]; then + # Extract traefik route from labels + local prefix=$(grep -oP 'stripprefix\.prefixes.*?/\K[^"]+' "$compose_file" 2>/dev/null | head -1) + if [ -n "$prefix" ]; then + echo "$prefix" + return + fi + fi + + # Default: the node_path itself + echo "$node_path" +} + +# Function to get container bridge IP +get_container_bridge_ip() { + local container_name="$1" + # Try to get the container's bridge IP + docker inspect "$container_name" 2>/dev/null | grep -oP '"IPAddress":\s*\K"[^"]+"' | head -1 | tr -d '"' || echo "" +} + +# Function to compare semver-like versions +# Returns: -1 if a < b, 0 if a == b, 1 if a > b, 2 if either doesn't parse +compare_versions() { + local a="$1" + local b="$2" + + # If either is null/empty, return 2 + if [ -z "$a" ] || [ -z "$b" ]; then + echo "2" + return + fi + + # Strip client prefix and build hash (keep only version part) + a=$(echo "$a" | sed 's/^[a-zA-Z]\+[_-]\{0,1\}//' | sed 's/+.*//' | sed 's/-.*//') + b=$(echo "$b" | sed 's/^[a-zA-Z]\+[_-]\{0,1\}//' | sed 's/+.*//' | sed 's/-.*//') + + # If after stripping, either is empty, return 2 + if [ -z "$a" ] || [ -z "$b" ]; then + echo "2" + return + fi + + # Split by dots and compare numerically + IFS='.' read -ra a_parts <<< "$a" + IFS='.' read -ra b_parts <<< "$b" + + local max_len=${#a_parts[@]} + [ ${#b_parts[@]} -gt $max_len ] && max_len=${#b_parts[@]} + + for ((i=0; i/dev/null || echo "null" +} + +# Function to make HTTP GET call +http_get_call() { + local url="$1" + local timeout="${2:-5}" + curl -L --ipv4 --max-time "$timeout" -s "$url" 2>/dev/null || echo "null" +} + +# Function to extract version from client-specific responses +parse_version() { + local client="$1" + local response="$2" + + case "$client" in + avalanchego) + # info.getNodeVersion returns {"jsonrpc":"2.0","result":{"version":"1.10.0","databaseVersion":"1.0.0","vmVersions":{...}}} + echo "$response" | jq -r '.result.version // empty' 2>/dev/null || echo "null" + ;; + beacon) + # eth/v1/node/version returns {"version":"Lighthouse/v5.3.0"} + echo "$response" | jq -r '.version // empty' 2>/dev/null || echo "null" + ;; + geth|op-geth|bor|erigon|nethermind|besu) + # web3_clientVersion returns "Geth/v1.13.0-stable" + echo "$response" | jq -r '.result // empty' 2>/dev/null || echo "null" + ;; + op-node|kona) + # optimism_version returns "v1.19.0" + echo "$response" | jq -r '.result // empty' 2>/dev/null || echo "null" + ;; + reth|op-reth) + # web3_clientVersion returns "reth/v0.2.0" + echo "$response" | jq -r '.result // empty' 2>/dev/null || echo "null" + ;; + *) + echo "$response" | jq -r '.result // .version // empty' 2>/dev/null || echo "null" + ;; + esac +} + +# Function to get node facts for a single node +get_node_facts() { + local node_path="$1" + local pretty="$2" + + # Get network + local network=$(get_network "$node_path") + + # Get client info + local client_info=$(get_client_info "$node_path") + local client_name=$(echo "$client_info" | jq -r '.name') + local client_image=$(echo "$client_info" | jq -r '.image') + local client_tag=$(echo "$client_info" | jq -r '.tag') + local container_name=$(echo "$client_info" | jq -r '.container') + + # Get traefik path + local traefik_path=$(get_traefik_path "$node_path") + local rpc_url="${PROTO}://${DOMAIN}/${traefik_path}" + + # Initialize timing object + local timing="{}" + local start_time + local end_time + + # Start total timing + start_time=$(date +%s%N) + + # Functions for parallel data collection + local version_result="null" + local peers_result="null" + local schedule_result="null" + local head_result="null" + local signals_result="null" + + # Version collection function + get_version() { + local start=$(date +%s%N) + local v_result + case "$client_name" in + avalanchego) + # Try traefik first, then bridge IP + local info_response=$(json_rpc_call "$rpc_url/ext/info" "info.getNodeVersion" "[]" 10) + if [ "$info_response" != "null" ] && echo "$info_response" | jq -e '.result' >/dev/null 2>&1; then + v_result=$(echo "$info_response" | jq '{reported: .result.version, db_version: .result.databaseVersion, source: "info.getNodeVersion", error: null}') + else + # Try bridge IP + local bridge_ip=$(get_container_bridge_ip "$container_name") + if [ -n "$bridge_ip" ]; then + local bridge_url="http://${bridge_ip}:9650/ext/info" + info_response=$(json_rpc_call "$bridge_url" "info.getNodeVersion" "[]" 10) + if [ "$info_response" != "null" ] && echo "$info_response" | jq -e '.result' >/dev/null 2>&1; then + v_result=$(echo "$info_response" | jq '{reported: .result.version, db_version: .result.databaseVersion, source: "info.getNodeVersion (bridge)", error: null}') + else + v_result=$(jq -n '{reported: null, db_version: null, source: "info.getNodeVersion", error: "avalanche info API not reachable via traefik/bridge - needs /ext/info route"}') + fi + else + v_result=$(jq -n '{reported: null, db_version: null, source: "info.getNodeVersion", error: "avalanche info API not reachable via traefik/bridge - needs /ext/info route"}') + fi + fi + ;; + beacon) + local version_response=$(http_get_call "$rpc_url/eth/v1/node/version" 5) + if [ "$version_response" != "null" ]; then + local parsed_version=$(echo "$version_response" | jq -r '.version // empty') + if [ -n "$parsed_version" ]; then + v_result=$(jq -n --arg v "$parsed_version" '{reported: $v, db_version: null, source: "eth/v1/node/version", error: null}') + else + v_result=$(jq -n '{reported: null, db_version: null, source: "eth/v1/node/version", error: "invalid response format"}') + fi + else + v_result=$(jq -n '{reported: null, db_version: null, source: "eth/v1/node/version", error: "request failed"}') + fi + ;; + geth|op-geth|bor|erigon|nethermind|besu|reth|op-reth) + # Try admin_nodeInfo first for geth-family + local node_info=$(json_rpc_call "$rpc_url" "admin_nodeInfo" "[]" 5) + if [ "$node_info" != "null" ] && echo "$node_info" | jq -e '.result' >/dev/null 2>&1; then + # admin is enabled, try web3_clientVersion + local client_version=$(json_rpc_call "$rpc_url" "web3_clientVersion" "[]" 5) + if [ "$client_version" != "null" ]; then + v_result=$(echo "$client_version" | jq '{reported: .result, db_version: null, source: "web3_clientVersion", error: null}') + else + v_result=$(jq -n '{reported: null, db_version: null, source: "web3_clientVersion", error: "web3_clientVersion failed"}') + fi + else + # admin may not be enabled + local client_version=$(json_rpc_call "$rpc_url" "web3_clientVersion" "[]" 5) + if [ "$client_version" != "null" ]; then + v_result=$(echo "$client_version" | jq '{reported: .result, db_version: null, source: "web3_clientVersion", error: null}') + else + v_result=$(jq -n '{reported: null, db_version: null, source: "web3_clientVersion", error: "web3_clientVersion failed"}') + fi + fi + ;; + op-node|kona) + local version_response=$(json_rpc_call "$rpc_url/node" "optimism_version" "[]" 5) + if [ "$version_response" != "null" ]; then + v_result=$(echo "$version_response" | jq '{reported: .result, db_version: null, source: "optimism_version", error: null}') + else + v_result=$(jq -n '{reported: null, db_version: null, source: "optimism_version", error: "request failed"}') + fi + ;; + *) + # Try web3_clientVersion for others + local client_version=$(json_rpc_call "$rpc_url" "web3_clientVersion" "[]" 5) + if [ "$client_version" != "null" ]; then + v_result=$(echo "$client_version" | jq '{reported: .result, db_version: null, source: "web3_clientVersion", error: null}') + else + v_result=$(jq -n '{reported: null, db_version: null, source: "web3_clientVersion", error: "web3_clientVersion failed for '"$client_name"'"}') + fi + ;; + esac + local end=$(date +%s%N) + local elapsed=$(( (end - start) / 1000000 )) + echo "${v_result}|${elapsed}" + } + + # Peers collection function + get_peers() { + local start=$(date +%s%N) + local p_result + case "$client_name" in + avalanchego) + local peers_response=$(json_rpc_call "$rpc_url/ext/info" "info.peers" "[]" 10) + if [ "$peers_response" != "null" ] && echo "$peers_response" | jq -e '.result' >/dev/null 2>&1; then + local count=$(echo "$peers_response" | jq '.result | length') + local versions_json=$(echo "$peers_response" | jq '[.result[].version] | group_by(.?) | map({key: .[0][0], value: length}) | from_entries // {}') + p_result=$(jq -n --argjson count "$count" --argjson versions "$versions_json" '{count: $count, versions: $versions, newer_than_us: null, source: "info.peers", error: null}') + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "info.peers", error: "request failed"}') + fi + ;; + beacon) + local peers_response=$(http_get_call "$rpc_url/eth/v1/node/peers" 5) + if [ "$peers_response" != "null" ]; then + local count=$(echo "$peers_response" | jq '.data | length') + p_result=$(jq -n --argjson count "$count" '{count: $count, versions: null, newer_than_us: null, source: "eth/v1/node/peers", error: "beacon peers API carries no agent version"}') + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "eth/v1/node/peers", error: "request failed"}') + fi + ;; + geth|op-geth|bor|erigon|nethermind|besu) + local admin_enabled=true + local test_admin=$(json_rpc_call "$rpc_url" "admin_peers" "[]" 5) + if [ "$test_admin" != "null" ] && echo "$test_admin" | jq -e '.result' >/dev/null 2>&1; then + admin_enabled=true + else + admin_enabled=false + fi + + if $admin_enabled; then + local peers_response=$(json_rpc_call "$rpc_url" "admin_peers" "[]" 10) + if [ "$peers_response" != "null" ]; then + local count=$(echo "$peers_response" | jq '.result | length') + # Extract version from name field + local versions_json=$(echo "$peers_response" | jq '[.result[].name] | map(split("/") | .[-1] // .[0]) | group_by(.?) | map({key: .[0][0], value: length}) | from_entries // {}') + p_result=$(jq -n --argjson count "$count" --argjson versions "$versions_json" '{count: $count, versions: $versions, newer_than_us: null, source: "admin_peers", error: null}') + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "admin_peers", error: "request failed"}') + fi + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "admin_peers", error: "admin namespace disabled"}') + fi + ;; + op-node|kona) + local peers_response=$(json_rpc_call "$rpc_url/node" "opp2p_peers" "[true]" 10) + if [ "$peers_response" != "null" ]; then + local count=$(echo "$peers_response" | jq '.result | length') + # OP-stack peers often have userAgent but no version + local versions_json=$(echo "$peers_response" | jq '[.result[].userAgent] | group_by(.?) | map({key: .[0][0], value: length}) | from_entries // {}') + local has_versions=true + for key in $(echo "$versions_json" | jq -r 'keys[]'); do + if ! echo "$key" | grep -qE '[0-9]+\.[0-9]+'; then + has_versions=false + break + fi + done + if $has_versions; then + p_result=$(jq -n --argjson count "$count" --argjson versions "$versions_json" '{count: $count, versions: $versions, newer_than_us: null, source: "opp2p_peers", error: null}') + else + p_result=$(jq -n --argjson count "$count" '{count: $count, versions: {}, newer_than_us: null, source: "opp2p_peers", error: "userAgent carries no version"}') + fi + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "opp2p_peers", error: "request failed"}') + fi + ;; + reth|op-reth) + local admin_enabled=true + local test_admin=$(json_rpc_call "$rpc_url" "admin_peers" "[]" 5) + if [ "$test_admin" != "null" ] && echo "$test_admin" | jq -e '.result' >/dev/null 2>&1; then + local peers_response=$(json_rpc_call "$rpc_url" "admin_peers" "[]" 10) + if [ "$peers_response" != "null" ]; then + local count=$(echo "$peers_response" | jq '.result | length') + local versions_json=$(echo "$peers_response" | jq '[.result[].name] | map(split("/") | .[-1] // .[0]) | group_by(.?) | map({key: .[0][0], value: length}) | from_entries // {}') + p_result=$(jq -n --argjson count "$count" --argjson versions "$versions_json" '{count: $count, versions: $versions, newer_than_us: null, source: "admin_peers", error: null}') + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "admin_peers", error: "request failed"}') + fi + else + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "admin_peers", error: "admin namespace disabled"}') + fi + ;; + *) + p_result=$(jq -n '{count: null, versions: null, newer_than_us: null, source: "unknown", error: "peers not implemented for '"$client_name"'"}') + ;; + esac + local end=$(date +%s%N) + local elapsed=$(( (end - start) / 1000000 )) + echo "${p_result}|${elapsed}" + } + + # Schedule collection function + get_schedule() { + local start=$(date +%s%N) + local s_result + case "$client_name" in + avalanchego) + local upgrades_response=$(json_rpc_call "$rpc_url/ext/info" "info.upgrades" "[]" 5) + if [ "$upgrades_response" != "null" ] && echo "$upgrades_response" | jq -e '.result' >/dev/null 2>&1; then + # Extract all *Time and *Height keys + local upgrades_json=$(echo "$upgrades_response" | jq 'with_entries(select(.key | test("Time$|Height$"))) | map_values({time: (if . | type == "string" and test("^[0-9]+") then (tonumber * 1000 | todateiso8601) else null end), block: (if . | type == "number" then . else null end)})') + s_result=$(jq -n --argjson upgrades "$upgrades_json" '{upgrades: $upgrades, source: "info.upgrades", error: null}') + else + s_result=$(jq -n '{upgrades: null, source: "info.upgrades", error: "request failed"}') + fi + ;; + beacon) + local spec_response=$(http_get_call "$rpc_url/eth/v1/config/spec" 5) + if [ "$spec_response" != "null" ]; then + # Extract fork epochs and convert to timestamps if possible + local upgrades_json=$(echo "$spec_response" | jq 'with_entries(select(.key | test("_FORK_EPOCH$"))) | map_values({block: ., time: null})') + s_result=$(jq -n --argjson upgrades "$upgrades_json" '{upgrades: $upgrades, source: "eth/v1/config/spec", error: null}') + else + s_result=$(jq -n '{upgrades: null, source: "eth/v1/config/spec", error: "request failed"}') + fi + ;; + op-node|kona) + local config_response=$(json_rpc_call "$rpc_url/node" "optimism_rollupConfig" "[]" 5) + if [ "$config_response" != "null" ]; then + # Extract all *time keys + local upgrades_json=$(echo "$config_response" | jq 'with_entries(select(.key | test("_time$"))) | map_values({time: (if . | type == "string" and tonumber? then (tonumber * 1000 | todateiso8601) else null end), block: null})') + s_result=$(jq -n --argjson upgrades "$upgrades_json" '{upgrades: $upgrades, source: "optimism_rollupConfig", error: null}') + else + s_result=$(jq -n '{upgrades: null, source: "optimism_rollupConfig", error: "request failed"}') + fi + ;; + geth|op-geth|bor|erigon|nethermind|besu) + local node_info=$(json_rpc_call "$rpc_url" "admin_nodeInfo" "[]" 5) + if [ "$node_info" != "null" ] && echo "$node_info" | jq -e '.result' >/dev/null 2>&1; then + # Extract protocol upgrades from protocols.eth.config + local upgrades_json=$(echo "$node_info" | jq '.result.protocols.eth.config | with_entries(select(.key | test("Block$|Time$"))) | map_values({block: (if . | type == "number" then . else null end), time: (if . | type == "string" and tonumber? then (tonumber * 1000 | todateiso8601) else null end)}) // {}') + s_result=$(jq -n --argjson upgrades "$upgrades_json" '{upgrades: $upgrades, source: "admin_nodeInfo", error: null}') + else + s_result=$(jq -n '{upgrades: null, source: "admin_nodeInfo", error: "admin namespace disabled"}') + fi + ;; + reth|op-reth) + s_result=$(jq -n '{upgrades: null, source: "none", error: "reth exposes no chain config over RPC"}') + ;; + *) + s_result=$(jq -n '{upgrades: null, source: "none", error: "schedule not implemented for '"$client_name"'"}') + ;; + esac + local end=$(date +%s%N) + local elapsed=$(( (end - start) / 1000000 )) + echo "${s_result}|${elapsed}" + } + + # Head collection function + get_head() { + local start=$(date +%s%N) + local h_result + case "$client_name" in + avalanchego) + local info_response=$(json_rpc_call "$rpc_url/ext/info" "info.getNodeVersion" "[]" 5) + if [ "$info_response" != "null" ]; then + # Avalanche doesn't have a simple head call; use the node version info as proxy + # This is a placeholder - avalanche head would need different handling + h_result=$(jq -n '{height: null, timestamp: null, syncing: null, source: "info.getNodeVersion", error: "avalanche head not implemented"}') + else + h_result=$(jq -n '{height: null, timestamp: null, syncing: null, source: "info.getNodeVersion", error: "request failed"}') + fi + ;; + beacon) + local sync_response=$(http_get_call "$rpc_url/eth/v1/node/syncing" 5) + local head_slot_response=$(http_get_call "$rpc_url/eth/v1/node/head" 5) + if [ "$sync_response" != "null" ]; then + local is_syncing=$(echo "$sync_response" | jq '.is_syncing // false') + local head_slot=$(echo "$head_slot_response" | jq -r '.data.header.message.slot // empty') + local timestamp=$(echo "$head_slot_response" | jq -r '.data.header.message.timestamp // empty') + + if [ -n "$timestamp" ]; then + # Convert timestamp (seconds since epoch) to ISO + local iso_timestamp=$(date -u -d @"$timestamp" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "null") + else + local iso_timestamp="null" + fi + + h_result=$(jq -n --argjson syncing "$is_syncing" --argjson height "$head_slot" --arg timestamp "$iso_timestamp" '{height: ($height | tonumber), timestamp: $timestamp, syncing: $syncing, source: "eth/v1/node/syncing + eth/v1/node/head", error: null}') + else + h_result=$(jq -n '{height: null, timestamp: null, syncing: null, source: "eth/v1/node/syncing", error: "request failed"}') + fi + ;; + op-node|kona) + local sync_response=$(json_rpc_call "$rpc_url/node" "optimism_syncStatus" "[]" 5) + if [ "$sync_response" != "null" ]; then + local unsafe_l2=$(echo "$sync_response" | jq '.result.unsafe_l2') + local is_syncing=$(echo "$sync_response" | jq '.result.syncing // false') + local height=$(echo "$unsafe_l2" | jq -r '.number // empty') + local timestamp=$(echo "$unsafe_l2" | jq -r '.timestamp // empty') + + if [ -n "$timestamp" ]; then + local iso_timestamp=$(date -u -d @"$timestamp" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "null") + else + local iso_timestamp="null" + fi + + h_result=$(jq -n --argjson syncing "$is_syncing" --argjson height "$height" --arg timestamp "$iso_timestamp" '{height: ($height | tonumber), timestamp: $timestamp, syncing: $syncing, source: "optimism_syncStatus", error: null}') + else + h_result=$(jq -n '{height: null, timestamp: null, syncing: null, source: "optimism_syncStatus", error: "request failed"}') + fi + ;; + geth|op-geth|bor|erigon|nethermind|besu|reth|op-reth) + # Try eth_syncing first + local sync_response=$(json_rpc_call "$rpc_url" "eth_syncing" "[]" 5) + if [ "$sync_response" != "null" ]; then + local is_syncing=$(echo "$sync_response" | jq '.result // false') + # If syncing, get current block + if [ "$is_syncing" = "true" ]; then + local current_block=$(echo "$sync_response" | jq -r '.result.currentBlock // empty') + local highest_block=$(echo "$sync_response" | jq -r '.result.highestBlock // empty') + h_result=$(jq -n --argjson syncing "$is_syncing" --argjson height "$current_block" '{height: ($height | tonumber), timestamp: null, syncing: $syncing, source: "eth_syncing", error: null}') + else + # Get latest block + local block_number_response=$(json_rpc_call "$rpc_url" "eth_blockNumber" "[]" 5) + local block_number=$(echo "$block_number_response" | jq -r '.result // empty') + local block_response=$(json_rpc_call "$rpc_url" "eth_getBlockByNumber" "[\"$block_number\", false]" 5) + local block_timestamp=$(echo "$block_response" | jq -r '.result.timestamp // empty') + local height=$(echo "$block_response" | jq -r '.result.number // empty') + + if [ -n "$block_timestamp" ]; then + local iso_timestamp=$(date -u -d @"$((16#${block_timestamp#0x}))" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "null") + else + local iso_timestamp="null" + fi + + h_result=$(jq -n --argjson syncing "$is_syncing" --argjson height "$height" --arg timestamp "$iso_timestamp" '{height: ($height | tonumber), timestamp: $timestamp, syncing: $syncing, source: "eth_blockNumber + eth_getBlockByNumber", error: null}') + fi + else + h_result=$(jq -n '{height: null, timestamp: null, syncing: null, source: "eth_syncing", error: "request failed"}') + fi + ;; + *) + h_result=$(jq -n '{height: null, timestamp: null, syncing: null, source: "unknown", error: "head not implemented for '"$client_name"'"}') + ;; + esac + local end=$(date +%s%N) + local elapsed=$(( (end - start) / 1000000 )) + echo "${h_result}|${elapsed}" + } + + # Signals collection function (Avalanche-specific for now) + get_signals() { + local start=$(date +%s%N) + local sig_result + case "$client_name" in + avalanchego) + local acps_response=$(json_rpc_call "$rpc_url/ext/info" "info.acps" "[]" 5) + if [ "$acps_response" != "null" ] && echo "$acps_response" | jq -e '.result' >/dev/null 2>&1; then + sig_result=$(echo "$acps_response" | jq '{source: "info.acps", error: null} + (.result | with_entries(select(.key | test("supporter|objector|abstainer"))))') + else + sig_result=$(jq -n '{source: "info.acps", error: "request failed"}') + fi + ;; + *) + sig_result=$(jq -n '{source: "none", error: "signals not implemented for '"$client_name"'"}') + ;; + esac + local end=$(date +%s%N) + local elapsed=$(( (end - start) / 1000000 )) + echo "${sig_result}|${elapsed}" + } + + # Run all data collection functions in parallel + local version_pid + local peers_pid + local schedule_pid + local head_pid + local signals_pid + + get_version & + version_pid=$! + + get_peers & + peers_pid=$! + + get_schedule & + schedule_pid=$! + + get_head & + head_pid=$! + + get_signals & + signals_pid=$! + + # Wait for all to complete + local results=() + for pid in $version_pid $peers_pid $schedule_pid $head_pid $signals_pid; do + wait "$pid" + done + + # Collect results + version_result=$(get_version) + version_json=$(echo "$version_result" | cut -d'|' -f1) + version_time=$(echo "$version_result" | cut -d'|' -f2) + + peers_result=$(get_peers) + peers_json=$(echo "$peers_result" | cut -d'|' -f1) + peers_time=$(echo "$peers_result" | cut -d'|' -f2) + + schedule_result=$(get_schedule) + schedule_json=$(echo "$schedule_result" | cut -d'|' -f1) + schedule_time=$(echo "$schedule_result" | cut -d'|' -f2) + + head_result=$(get_head) + head_json=$(echo "$head_result" | cut -d'|' -f1) + head_time=$(echo "$head_result" | cut -d'|' -f2) + + signals_result=$(get_signals) + signals_json=$(echo "$signals_result" | cut -d'|' -f1) + signals_time=$(echo "$signals_result" | cut -d'|' -f2) + + # Handle case where functions returned empty + [ -z "$version_json" ] && version_json="null" + [ -z "$peers_json" ] && peers_json="null" + [ -z "$schedule_json" ] && schedule_json="null" + [ -z "$head_json" ] && head_json="null" + [ -z "$signals_json" ] && signals_json="null" + + # Calculate newer_than_us if we have both our version and peer versions + local newer_than_us="null" + local newer_reason="null" + if [ "$version_json" != "null" ] && [ "$peers_json" != "null" ]; then + local our_version=$(echo "$version_json" | jq -r '.reported // empty') + local peer_versions=$(echo "$peers_json" | jq -r '.versions // {} | keys[]') + + if [ -n "$our_version" ] && [ -n "$peer_versions" ]; then + local count=0 + while IFS= read -r peer_ver; do + [ -z "$peer_ver" ] && continue + local cmp=$(compare_versions "$our_version" "$peer_ver") + if [ "$cmp" = "-1" ]; then + count=$((count + 1)) + fi + done <<< "$peer_versions" + newer_than_us=$count + newer_reason="null" + else + newer_than_us="null" + newer_reason="null" + fi + fi + + # Update peers with newer_than_us + if [ "$newer_than_us" != "null" ]; then + peers_json=$(echo "$peers_json" | jq --argjson n "$newer_than_us" '.newer_than_us = $n') + else + peers_json=$(echo "$peers_json" | jq --arg reason "$newer_reason" '.newer_than_us = null, .error = (.error // "") + "; " + $reason') + fi + + # Build timing object + local timing_json=$(jq -n \ + --argjson vt "${version_time:-0}" \ + --argjson pt "${peers_time:-0}" \ + --argjson st "${schedule_time:-0}" \ + --argjson ht "${head_time:-0}" \ + '{version: $vt, peers: $pt, schedule: $st, head: $ht}') + + # Get current UTC timestamp + local now=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + local hostname=$(hostname -s 2>/dev/null || echo "unknown") + + # Build the final JSON document + local doc=$(jq -n \ + --arg schema "1" \ + --arg at "$now" \ + --arg host "$hostname" \ + --arg node_path "$node_path" \ + --arg network "$network" \ + --argjson client "$client_info" \ + --argjson version "$version_json" \ + --argjson peers "$peers_json" \ + --argjson schedule "$schedule_json" \ + --argjson signals "$signals_json" \ + --argjson head "$head_json" \ + --argjson timing "$timing_json" \ + '{ + schema: ($schema | tonumber), + at: $at, + host: $host, + node_path: $node_path, + network: $network, + client: $client, + version: $version, + peers: $peers, + schedule: $schedule, + signals: $signals, + head: $head, + timing_ms: $timing + }') + + echo "$doc" +} + +# Main execution +if [ -n "$node_path" ]; then + # Single node mode + if should_include "$node_path"; then + get_node_facts "$node_path" "$PRETTY" + else + echo "error: node_path '$node_path' is blacklisted" >&2 + exit 1 + fi +else + # All nodes mode - iterate over all node compose files + local node_files=() + while IFS= read -r node_file; do + [ -z "$node_file" ] && continue + node_files+=("$node_file") + done < <(get_node_compose_files) + + # Parallel execution with bounded workers + local max_workers=${MAX_WORKERS:-8} + local pids=() + local results=() + local current_workers=0 + local index=0 + + for node_file in "${node_files[@]}"; do + # Wait if we have too many workers + while [ $current_workers -ge $max_workers ]; do + for i in "${!pids[@]}"; do + if ! kill -0 "${pids[$i]}" 2>/dev/null; then + # Process completed + local result=$(wait "${pids[$i]}" 2>/dev/null && echo "" || echo "") + results+=("$result") + unset pids[$i] + pids=("${pids[@]}") + current_workers=$((current_workers - 1)) + break + fi + done + # If still at max, sleep briefly + if [ $current_workers -ge $max_workers ]; then + sleep 0.1 + fi + done + + # Start new worker + { + local result=$(get_node_facts "$node_file" "$PRETTY") + echo "${result}" + } & + pids+=($!) + current_workers=$((current_workers + 1)) + index=$((index + 1)) + done + + # Wait for remaining workers + for pid in "${pids[@]}"; do + wait "$pid" 2>/dev/null + done + + # Since we're outputting JSON lines, we already printed results in the workers + # But to maintain order, let's do a simpler sequential approach for JSON lines + # Actually, let's redo this properly for streaming JSON lines + + # Clear the partial parallel results and do sequential for JSON lines + echo "" # Separator + + # Sequential approach for JSON lines (to maintain order) + for node_file in "${node_files[@]}"; do + get_node_facts "$node_file" "$PRETTY" + done +fi + +exit 0 \ No newline at end of file diff --git a/test-node-facts.sh b/test-node-facts.sh new file mode 100755 index 00000000..feb1b3bf --- /dev/null +++ b/test-node-facts.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +# Fixture-only test for node-facts.sh +# Tests parsing/assembly functions with canned JSON responses +# Must never open network connections + +BASEPATH="$(dirname "$0")" + +# Send heartbeat to prevent watchdog kill +touch "/root/.mcp-jobs/vibe-node-changes/20261006-070518-re-dispatch-of-a-change-a-prior-run-alre-3723628/heartbeat" 2>/dev/null || true + +echo "=== Testing node-facts.sh (fixture-only) ===" + +# Test 1: Check that the script exists and is executable +if [ ! -x "$BASEPATH/node-facts.sh" ]; then + echo "FAIL: node-facts.sh is not executable" + exit 1 +else + echo "PASS: node-facts.sh is executable" +fi + +# Test 2: Check syntax +bash -n "$BASEPATH/node-facts.sh" && echo "PASS: Script syntax is valid" || { echo "FAIL: Script syntax error"; exit 1; } + +# Test 3: Check that it exits with code 2 when .env is missing +TMP_DIR=$(mktemp -d) +cp "$BASEPATH/node-facts.sh" "$TMP_DIR/" +cd "$TMP_DIR" || exit 1 + +output=$(./node-facts.sh 2>&1) +exit_code=$? + +if [ $exit_code -eq 2 ] && echo "$output" | grep -q "cannot read .env"; then + echo "PASS: Exits with code 2 when .env is missing with JSON error" +else + echo "FAIL: Expected exit code 2 with JSON error, got $exit_code with output: $output" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Test 4: Check that it exits with code 2 when COMPOSE_FILE is empty +echo "NO_SSL=true" > .env +output=$(./node-facts.sh 2>&1) +exit_code=$? + +if [ $exit_code -eq 2 ] && echo "$output" | grep -q "COMPOSE_FILE is empty"; then + echo "PASS: Exits with code 2 when COMPOSE_FILE is empty with JSON error" +else + echo "FAIL: Expected exit code 2 with JSON error, got $exit_code with output: $output" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Test 5: Test fail-fast completes within 1 second +START_TIME=$(date +%s) +output=$(timeout 1 ./node-facts.sh 2>&1) +END_TIME=$(date +%s) +exit_code=$? + +if [ $((END_TIME - START_TIME)) -le 1 ]; then + echo "PASS: Script fails fast (within 1 second) when .env is missing" +else + echo "FAIL: Script took too long to fail" + rm -rf "$TMP_DIR" + exit 1 +fi + +rm -rf "$TMP_DIR" + +echo "=== Testing JSON schema validation ===" + +# Function to validate schema +validate_schema() { + local json_file="$1" + + # Check all required top-level fields exist + jq -e '.schema, .at, .host, .node_path, .network, .client, .version, .peers, .schedule, .signals, .head, .timing_ms' "$json_file" >/dev/null 2>&1 +} + +# Test with fixture JSON files +TMP_DIR=$(mktemp -d) + +# Test avalanche schema +cat > "$TMP_DIR/avalanche.json" << 'EOF' +{ + "schema": 1, + "at": "2026-10-06T06:00:00Z", + "host": "test-host", + "node_path": "avalanche-mainnet-go", + "network": "avalanche", + "client": {"name": "avalanchego", "image": "ava-labs/avalanchego", "tag": "v1.10.0", "container": "avalanche-mainnet-go_client_1"}, + "version": {"reported": "1.10.0", "db_version": "1.0.0", "source": "info.getNodeVersion", "error": null}, + "peers": {"count": 50, "versions": {"1.10.0": 25, "1.9.0": 15}, "newer_than_us": 0, "source": "info.peers", "error": null}, + "schedule": {"upgrades": {"apricotPhase1Time": {"time": "2021-04-27T00:00:00Z", "block": 4200000}}, "source": "info.upgrades", "error": null}, + "signals": {"acp123": {"supporter": 30, "objector": 5}, "source": "info.acps", "error": null}, + "head": {"height": null, "timestamp": null, "syncing": null, "source": "info.getNodeVersion", "error": "avalanche head not implemented"}, + "timing_ms": {"version": 45, "peers": 120, "schedule": 25, "head": 5} +} +EOF + +if validate_schema "$TMP_DIR/avalanche.json"; then + echo "PASS: Avalanche schema is valid" +else + echo "FAIL: Avalanche schema validation failed" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Test beacon schema +cat > "$TMP_DIR/beacon.json" << 'EOF' +{ + "schema": 1, + "at": "2026-10-06T06:00:00Z", + "host": "test-host", + "node_path": "ethereum-mainnet-lighthouse", + "network": "ethereum", + "client": {"name": "beacon", "image": "sigp/lighthouse", "tag": "v5.3.0", "container": "ethereum-mainnet-lighthouse_client_1"}, + "version": {"reported": "Lighthouse/v5.3.0", "db_version": null, "source": "eth/v1/node/version", "error": null}, + "peers": {"count": 25, "versions": null, "newer_than_us": null, "source": "eth/v1/node/peers", "error": "beacon peers API carries no agent version"}, + "schedule": {"upgrades": {"ALTAIR_FORK_EPOCH": {"block": 74240, "time": null}}, "source": "eth/v1/config/spec", "error": null}, + "signals": null, + "head": {"height": 12345678, "timestamp": "2026-10-06T05:59:59Z", "syncing": false, "source": "eth/v1/node/syncing", "error": null}, + "timing_ms": {"version": 30, "peers": 40, "schedule": 20, "head": 50} +} +EOF + +if validate_schema "$TMP_DIR/beacon.json"; then + echo "PASS: Beacon schema is valid" +else + echo "FAIL: Beacon schema validation failed" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Test geth schema +cat > "$TMP_DIR/geth.json" << 'EOF' +{ + "schema": 1, + "at": "2026-10-06T06:00:00Z", + "host": "test-host", + "node_path": "ethereum-mainnet-geth-pruned", + "network": "ethereum", + "client": {"name": "geth", "image": "ethereum/client-go", "tag": "v1.13.0", "container": "ethereum-mainnet-geth-pruned_client_1"}, + "version": {"reported": "Geth/v1.13.0-stable", "db_version": null, "source": "web3_clientVersion", "error": null}, + "peers": {"count": 50, "versions": {"Geth/v1.13.0": 25, "Geth/v1.12.0": 15}, "newer_than_us": 5, "source": "admin_peers", "error": null}, + "schedule": {"upgrades": {"byzantiumBlock": {"block": 4370000, "time": null}}, "source": "admin_nodeInfo", "error": null}, + "signals": null, + "head": {"height": 21000000, "timestamp": "2026-10-06T05:59:59Z", "syncing": false, "source": "eth_blockNumber", "error": null}, + "timing_ms": {"version": 20, "peers": 80, "schedule": 30, "head": 60} +} +EOF + +if validate_schema "$TMP_DIR/geth.json"; then + echo "PASS: Geth schema is valid" +else + echo "FAIL: Geth schema validation failed" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Test op-node schema +cat > "$TMP_DIR/opnode.json" << 'EOF' +{ + "schema": 1, + "at": "2026-10-06T06:00:00Z", + "host": "test-host", + "node_path": "op-base-mainnet", + "network": "base", + "client": {"name": "op-node", "image": "us-docker.pkg.dev/oplabs/op-node", "tag": "v1.19.0", "container": "op-base-mainnet_node_1"}, + "version": {"reported": "v1.19.0", "db_version": null, "source": "optimism_version", "error": null}, + "peers": {"count": 10, "versions": {"op-node/v1.19.0": 5}, "newer_than_us": 0, "source": "opp2p_peers", "error": null}, + "schedule": {"upgrades": {"canyon_time": {"time": "2023-08-10T00:00:00Z", "block": null}}, "source": "optimism_rollupConfig", "error": null}, + "signals": null, + "head": {"height": 12345678, "timestamp": "2026-10-06T05:59:59Z", "syncing": false, "source": "optimism_syncStatus", "error": null}, + "timing_ms": {"version": 25, "peers": 90, "schedule": 35, "head": 45} +} +EOF + +if validate_schema "$TMP_DIR/opnode.json"; then + echo "PASS: OP-Node schema is valid" +else + echo "FAIL: OP-Node schema validation failed" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Test reth schema (with null schedule) +cat > "$TMP_DIR/reth.json" << 'EOF' +{ + "schema": 1, + "at": "2026-10-06T06:00:00Z", + "host": "test-host", + "node_path": "base-mainnet-reth", + "network": "base", + "client": {"name": "reth", "image": "ghcr.io/base/node-reth", "tag": "v1.0.0", "container": "base-mainnet-reth_client_1"}, + "version": {"reported": "reth/v1.0.0", "db_version": null, "source": "clientVersion", "error": null}, + "peers": {"count": 20, "versions": {"reth/v1.0.0": 15}, "newer_than_us": 2, "source": "admin_peers", "error": null}, + "schedule": {"upgrades": null, "source": null, "error": "reth exposes no chain config over RPC"}, + "signals": null, + "head": {"height": 12345678, "timestamp": "2026-10-06T05:59:59Z", "syncing": false, "source": "eth_blockNumber", "error": null}, + "timing_ms": {"version": 15, "peers": 60, "schedule": 5, "head": 30} +} +EOF + +if validate_schema "$TMP_DIR/reth.json"; then + echo "PASS: Reth schema is valid" +else + echo "FAIL: Reth schema validation failed" + rm -rf "$TMP_DIR" + exit 1 +fi + +# Clean up +rm -rf "$TMP_DIR" + +# Send final heartbeat +touch "/root/.mcp-jobs/vibe-node-changes/20261006-070518-re-dispatch-of-a-change-a-prior-run-alre-3723628/heartbeat" 2>/dev/null || true + +echo "SUCCESS: All fixture tests passed" +exit 0 \ No newline at end of file