From fe22be322eb757c6abe095d9b8a517015d7aa6d4 Mon Sep 17 00:00:00 2001 From: Claude Agent Date: Tue, 6 Oct 2026 07:22:51 +0000 Subject: [PATCH] Fix node-facts.sh bugs and add stub-curl integration tests - Fix grep -qE calls to handle variables starting with '-' (add -- separator) - Remove local declarations at top level by wrapping main body in main() function - Replace bounded-worker scheduler with simpler temp file + wait -n pattern - Fix per-node fan-out to capture function output instead of re-calling - Add stub-curl integration tests that catch all three original issues - Add fixture compose files for avalanche, op-node+beacon, geth testing - Update test path-blacklist.txt to include entry starting with - Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- node-facts-test/avalanche-mainnet-go.yml | 11 ++ .../ethereum-mainnet-lighthouse.yml | 11 ++ node-facts-test/op-base-mainnet.yml | 11 ++ node-facts-test/path-blacklist.txt | 1 + node-facts-test/stub-curl/curl | 127 ++++++++++++++ node-facts.sh | 158 +++++++++--------- test-node-facts.sh | 131 +++++++++++++++ 7 files changed, 367 insertions(+), 83 deletions(-) create mode 100644 node-facts-test/avalanche-mainnet-go.yml create mode 100644 node-facts-test/ethereum-mainnet-lighthouse.yml create mode 100644 node-facts-test/op-base-mainnet.yml create mode 100755 node-facts-test/stub-curl/curl diff --git a/node-facts-test/avalanche-mainnet-go.yml b/node-facts-test/avalanche-mainnet-go.yml new file mode 100644 index 00000000..18447969 --- /dev/null +++ b/node-facts-test/avalanche-mainnet-go.yml @@ -0,0 +1,11 @@ +version: "3" +services: + client: + image: ava-labs/avalanchego:v1.10.0 + labels: + - "traefik.http.routers.avalanche-mainnet-go.rule=PathPrefix(`/avalanche-mainnet-go`)" + - "traefik.http.middlewares.avalanche-mainnet-go.stripprefix.prefixes=/avalanche-mainnet-go" + x-upstreams: + network: avalanche + chain: avalanche-mainnet +chain: avalanche-mainnet \ No newline at end of file diff --git a/node-facts-test/ethereum-mainnet-lighthouse.yml b/node-facts-test/ethereum-mainnet-lighthouse.yml new file mode 100644 index 00000000..dc9b9ab6 --- /dev/null +++ b/node-facts-test/ethereum-mainnet-lighthouse.yml @@ -0,0 +1,11 @@ +version: "3" +services: + client: + image: sigp/lighthouse:v5.3.0 + labels: + - "traefik.http.routers.ethereum-mainnet-lighthouse.rule=PathPrefix(`/ethereum-mainnet-lighthouse`)" + - "traefik.http.middlewares.ethereum-mainnet-lighthouse.stripprefix.prefixes=/ethereum-mainnet-lighthouse" + x-upstreams: + network: ethereum + chain: ethereum-mainnet +chain: ethereum-mainnet \ No newline at end of file diff --git a/node-facts-test/op-base-mainnet.yml b/node-facts-test/op-base-mainnet.yml new file mode 100644 index 00000000..90662507 --- /dev/null +++ b/node-facts-test/op-base-mainnet.yml @@ -0,0 +1,11 @@ +version: "3" +services: + node: + image: us-docker.pkg.dev/oplabs/op-node:v1.19.0 + labels: + - "traefik.http.routers.op-base-mainnet.rule=PathPrefix(`/op-base-mainnet`)" + - "traefik.http.middlewares.op-base-mainnet.stripprefix.prefixes=/op-base-mainnet" + x-upstreams: + network: base + chain: op-base-mainnet +chain: op-base-mainnet \ No newline at end of file diff --git a/node-facts-test/path-blacklist.txt b/node-facts-test/path-blacklist.txt index 41f6f095..91898da4 100644 --- a/node-facts-test/path-blacklist.txt +++ b/node-facts-test/path-blacklist.txt @@ -4,3 +4,4 @@ prysm beacon nimbus ws +-node diff --git a/node-facts-test/stub-curl/curl b/node-facts-test/stub-curl/curl new file mode 100755 index 00000000..8c9adb31 --- /dev/null +++ b/node-facts-test/stub-curl/curl @@ -0,0 +1,127 @@ +#!/bin/bash + +# Simple stub curl for node-facts.sh testing +# Returns minimal valid JSON responses + +# Extract the URL (last http/https argument) +URL="" +METHOD="" +for arg in "$@"; do + if [[ "$arg" == http* || "$arg" == https* ]]; then + URL="$arg" + fi + # Look for method in JSON data arguments + if echo "$arg" | grep -q "method"; then + METHOD=$(echo "$arg" | sed -n 's/.*"method":"\([^"]*\)".*/\1/p' | head -1) + [ -n "$METHOD" ] && break + fi +done + +# If method not found in data args, try to extract from the URL +if [ -z "$METHOD" ]; then + # Extract method from URL path for REST endpoints + case "$URL" in + *eth/v1/node/version*) METHOD="eth/v1/node/version" ;; + *eth/v1/node/peers*) METHOD="eth/v1/node/peers" ;; + *eth/v1/config/spec*) METHOD="eth/v1/config/spec" ;; + *eth/v1/node/syncing*) METHOD="eth/v1/node/syncing" ;; + *eth/v1/node/head*) METHOD="eth/v1/node/head" ;; + *optimism_version*) METHOD="optimism_version" ;; + *opp2p_peers*) METHOD="opp2p_peers" ;; + *optimism_rollupConfig*) METHOD="optimism_rollupConfig" ;; + *optimism_syncStatus*) METHOD="optimism_syncStatus" ;; + esac +fi + +# Respond based on URL and method +case "$URL" in + *avalanche-mainnet-go*) + case "$METHOD" in + info.getNodeVersion) + echo '{"jsonrpc":"2.0","result":{"version":"avalanche/1.10.0","databaseVersion":"1.0.0"},"id":1}' + ;; + info.peers) + echo '{"jsonrpc":"2.0","result":[{"version":"avalanche/1.10.0"},{"version":"avalanche/1.9.0"}],"id":1}' + ;; + info.upgrades) + echo '{"jsonrpc":"2.0","result":{"apricotPhase1Time":4200000},"id":1}' + ;; + *) + # For /ext/info without method, return version info + if [[ "$URL" == *"/ext/info"* ]]; then + echo '{"jsonrpc":"2.0","result":{"version":"avalanche/1.10.0","databaseVersion":"1.0.0"},"id":1}' + else + echo '{"jsonrpc":"2.0","error":{"code":-32601,"message":"Method not found"},"id":1}' + fi + ;; + esac + ;; + *ethereum-mainnet-lighthouse* | *beacon* | *lighthouse*) + case "$METHOD" in + eth/v1/node/version*) + echo '{"version":"Lighthouse/v5.3.0"}' + ;; + eth/v1/node/peers*) + echo '{"data":[]}' + ;; + eth/v1/config/spec*) + echo '{"data":{}}' + ;; + eth/v1/node/syncing*) + echo '{"data":{"head_slot":"100","syncing":false}}' + ;; + eth/v1/node/head*) + echo '{"data":{"slot":"100","timestamp":"2026-10-06T12:00:00Z"}}' + ;; + *) + echo '{"message":"Not found"}' + ;; + esac + ;; + *op-base-mainnet* | *op-node* | *kona*) + case "$METHOD" in + optimism_version* | */node*) + echo '{"result":"v1.19.0"}' + ;; + opp2p_peers*) + echo '{"result":[]}' + ;; + optimism_rollupConfig*) + echo '{"result":{}}' + ;; + optimism_syncStatus*) + echo '{"result":{"current_block_number":"100","syncing":false}}' + ;; + *) + echo '{"message":"Not found"}' + ;; + esac + ;; + *ethereum-mainnet-geth* | *geth*) + case "$METHOD" in + web3_clientVersion) + echo '{"jsonrpc":"2.0","result":"Geth/v1.13.0-stable","id":1}' + ;; + admin_peers) + echo '{"jsonrpc":"2.0","result":[{"clientId":"Geth/v1.13.0"},{"clientId":"Geth/v1.12.0"}],"id":1}' + ;; + eth_blockNumber) + echo '{"jsonrpc":"2.0","result":"0x64","id":1}' + ;; + eth_getBlockByNumber) + echo '{"jsonrpc":"2.0","result":{"timestamp":"0x65a6b3d4","number":"0x64"},"id":1}' + ;; + eth_syncing) + echo '{"jsonrpc":"2.0","result":false,"id":1}' + ;; + *) + echo '{"jsonrpc":"2.0","error":{"code":-32601,"message":"Method not found"},"id":1}' + ;; + esac + ;; + *) + echo '{"jsonrpc":"2.0","error":{"code":-32601,"message":"Unknown endpoint"},"id":1}' + ;; +esac + +exit 0 \ No newline at end of file diff --git a/node-facts.sh b/node-facts.sh index 8ec1b2a8..390bd1ac 100755 --- a/node-facts.sh +++ b/node-facts.sh @@ -71,7 +71,7 @@ fi should_include() { local path="$1" for word in "${blacklist[@]}"; do - if echo "$path" | grep -qE "$word"; then + if echo "$path" | grep -qE -- "$word"; then return 1 fi done @@ -86,7 +86,7 @@ get_node_compose_files() { # Remove leading ./ and trailing .yml if present part="$(echo "$part" | sed 's|^\./||; s|\.yml$||')" # Only include if it's a real compose file (contains hyphens, not base/rpc/etc) - if echo "$part" | grep -q "-" && should_include "$part"; then + if echo "$part" | grep -q -- "-" && should_include "$part"; then echo "$part" fi done @@ -133,7 +133,7 @@ get_client_info() { if [ -n "$services" ]; then # Try to find image for this service - local image_line=$(grep -A 20 "^\s+${services}:" "$compose_file" | grep -oP 'image:\s*\K[^\s]+' | head -1) + local image_line=$(grep -A 20 -- "^\s+${services}:" "$compose_file" | grep -oP -- 'image:\s*\K[^\s]+' | head -1) if [ -n "$image_line" ]; then # Parse image:tag client_image="$(echo "$image_line" | cut -d':' -f1)" @@ -698,52 +698,51 @@ get_node_facts() { echo "${sig_result}|${elapsed}" } - # Run all data collection functions in parallel + # Run all data collection functions in parallel and capture output local version_pid local peers_pid local schedule_pid local head_pid local signals_pid + local version_result + local peers_result + local schedule_result + local head_result + local signals_result - get_version & + version_result=$(get_version) & version_pid=$! - get_peers & + peers_result=$(get_peers) & peers_pid=$! - get_schedule & + schedule_result=$(get_schedule) & schedule_pid=$! - get_head & + head_result=$(get_head) & head_pid=$! - get_signals & + signals_result=$(get_signals) & signals_pid=$! # Wait for all to complete - local results=() for pid in $version_pid $peers_pid $schedule_pid $head_pid $signals_pid; do wait "$pid" done - # Collect results - version_result=$(get_version) + # Collect results from captured output version_json=$(echo "$version_result" | cut -d'|' -f1) version_time=$(echo "$version_result" | cut -d'|' -f2) - peers_result=$(get_peers) peers_json=$(echo "$peers_result" | cut -d'|' -f1) peers_time=$(echo "$peers_result" | cut -d'|' -f2) - schedule_result=$(get_schedule) schedule_json=$(echo "$schedule_result" | cut -d'|' -f1) schedule_time=$(echo "$schedule_result" | cut -d'|' -f2) - head_result=$(get_head) head_json=$(echo "$head_result" | cut -d'|' -f1) head_time=$(echo "$head_result" | cut -d'|' -f2) - signals_result=$(get_signals) signals_json=$(echo "$signals_result" | cut -d'|' -f1) signals_time=$(echo "$signals_result" | cut -d'|' -f2) @@ -829,76 +828,69 @@ get_node_facts() { echo "$doc" } -# Main execution -if [ -n "$node_path" ]; then - # Single node mode - if should_include "$node_path"; then - get_node_facts "$node_path" "$PRETTY" +# Main execution function +main() { + if [ -n "$node_path" ]; then + # Single node mode + if should_include "$node_path"; then + get_node_facts "$node_path" "$PRETTY" + else + echo "error: node_path '$node_path' is blacklisted" >&2 + exit 1 + fi else - echo "error: node_path '$node_path' is blacklisted" >&2 - exit 1 - fi -else - # All nodes mode - iterate over all node compose files - local node_files=() - while IFS= read -r node_file; do - [ -z "$node_file" ] && continue - node_files+=("$node_file") - done < <(get_node_compose_files) - - # Parallel execution with bounded workers - local max_workers=${MAX_WORKERS:-8} - local pids=() - local results=() - local current_workers=0 - local index=0 - - for node_file in "${node_files[@]}"; do - # Wait if we have too many workers - while [ $current_workers -ge $max_workers ]; do - for i in "${!pids[@]}"; do - if ! kill -0 "${pids[$i]}" 2>/dev/null; then - # Process completed - local result=$(wait "${pids[$i]}" 2>/dev/null && echo "" || echo "") - results+=("$result") - unset pids[$i] - pids=("${pids[@]}") - current_workers=$((current_workers - 1)) - break - fi - done - # If still at max, sleep briefly - if [ $current_workers -ge $max_workers ]; then - sleep 0.1 + # All nodes mode - iterate over all node compose files + node_files=() + while IFS= read -r node_file; do + [ -z "$node_file" ] && continue + node_files+=("$node_file") + done < <(get_node_compose_files) + + # Create temp directory for output files + temp_dir=$(mktemp -d) + + # Track pids and temp files + pids=() + temp_files=() + max_workers=${MAX_WORKERS:-8} + + for node_file in "${node_files[@]}"; do + temp_file="$temp_dir/$(basename "$node_file" .yml).json" + ( get_node_facts "$node_file" "$PRETTY" > "$temp_file" ) & + pids+=($!) + temp_files+=("$temp_file") + + # Wait if we reached max workers + if [ ${#pids[@]} -ge $max_workers ]; then + wait -n + # Remove the completed pid and corresponding temp file + completed_pid="$?" + for i in "${!pids[@]}"; do + if [ "${pids[$i]}" = "$completed_pid" ]; then + unset pids[$i] + unset temp_files[$i] + break + fi + done + pids=("${pids[@]}") + temp_files=("${temp_files[@]}") fi done - # Start new worker - { - local result=$(get_node_facts "$node_file" "$PRETTY") - echo "${result}" - } & - pids+=($!) - current_workers=$((current_workers + 1)) - index=$((index + 1)) - done - - # Wait for remaining workers - for pid in "${pids[@]}"; do - wait "$pid" 2>/dev/null - done - - # Since we're outputting JSON lines, we already printed results in the workers - # But to maintain order, let's do a simpler sequential approach for JSON lines - # Actually, let's redo this properly for streaming JSON lines - - # Clear the partial parallel results and do sequential for JSON lines - echo "" # Separator - - # Sequential approach for JSON lines (to maintain order) - for node_file in "${node_files[@]}"; do - get_node_facts "$node_file" "$PRETTY" - done -fi + # Wait for all remaining workers + wait + + # Output results in order + for temp_file in "${temp_files[@]}"; do + cat "$temp_file" + done + + # Clean up temp directory + rm -rf "$temp_dir" + fi +} + +# Call main function +main exit 0 \ No newline at end of file diff --git a/test-node-facts.sh b/test-node-facts.sh index feb1b3bf..cff9cc07 100755 --- a/test-node-facts.sh +++ b/test-node-facts.sh @@ -67,6 +67,137 @@ fi rm -rf "$TMP_DIR" +echo "=== Testing stub-curl integration ===" + +# Stub-curl test: test with fixture compose directory and stub curl +STUB_TEST_DIR="$BASEPATH/node-facts-test" +if [ ! -d "$STUB_TEST_DIR" ]; then + echo "SKIP: Stub test directory not found" +else + # Test 6: Test with stub curl - single node mode + TMP_DIR=$(mktemp -d) + cd "$TMP_DIR" || exit 1 + + # Copy test files + cp -r "$STUB_TEST_DIR"/* . + + # Make stub curl executable and put it first in PATH + STUB_CURL_DIR="$TMP_DIR/stub-curl" + chmod +x "$STUB_CURL_DIR/curl" + + # Test single node mode with stub curl + ORIGINAL_PATH="$PATH" + export PATH="$STUB_CURL_DIR:$PATH" + + START_TIME=$(date +%s) + output=$(timeout 60 ./node-facts.sh "avalanche-mainnet-go" 2>&1) + END_TIME=$(date +%s) + exit_code=$? + export PATH="$ORIGINAL_PATH" + + if [ $exit_code -eq 0 ] && [ $((END_TIME - START_TIME)) -lt 10 ]; then + echo "PASS: Stub curl single node mode exits 0 and completes in < 10s" + # Validate it's JSON and has the right schema + if echo "$output" | jq -e '.schema, .node_path, .network, .client' >/dev/null 2>&1; then + echo "PASS: Single node output is valid JSON with required schema fields" + else + echo "FAIL: Single node output is not valid JSON schema" + echo "Output: $output" + rm -rf "$TMP_DIR" + exit 1 + fi + else + echo "FAIL: Stub curl single node mode failed with exit code $exit_code, took $((END_TIME - START_TIME))s" + echo "Output: $output" + rm -rf "$TMP_DIR" + exit 1 + fi + + # Test 7: Test with stub curl - all nodes mode + START_TIME=$(date +%s) + output=$(timeout 60 ./node-facts.sh 2>&1) + END_TIME=$(date +%s) + exit_code=$? + + if [ $exit_code -eq 0 ] && [ $((END_TIME - START_TIME)) -lt 10 ]; then + echo "PASS: Stub curl all-nodes mode exits 0 and completes in < 10s" + + # Count JSON lines - should have at least 3 valid JSON lines (one per node) + json_lines=$(echo "$output" | grep -c '^{.*}$' || echo "0") + if [ "$json_lines" -ge 3 ]; then + echo "PASS: All-nodes mode produced $json_lines JSON lines (>= 3)" + else + echo "FAIL: All-nodes mode produced only $json_lines JSON lines, expected >= 3" + echo "Output: $output" + rm -rf "$TMP_DIR" + exit 1 + fi + + # Validate each JSON line against schema + valid_count=0 + while IFS= read -r line; do + if echo "$line" | jq -e '.schema, .at, .host, .node_path, .network, .client, .version, .peers, .schedule, .signals, .head, .timing_ms' >/dev/null 2>&1; then + valid_count=$((valid_count + 1)) + else + echo "Invalid JSON line: $line" + fi + done <<< "$(echo "$output" | grep '^{.*}$')" + + if [ $valid_count -ge 3 ]; then + echo "PASS: All $valid_count JSON lines have valid schema v1 fields" + else + echo "FAIL: Only $valid_count valid JSON lines out of $json_lines total" + rm -rf "$TMP_DIR" + exit 1 + fi + + # Check for no stderr noise (grep/sed/jq errors) + if echo "$output" | grep -q "grep: invalid argument\|local: can only be used\|sed:\|jq:"; then + echo "FAIL: Found stderr noise in output: $(echo "$output" | grep "grep: invalid argument\|local: can only be used\|sed:\|jq:")" + rm -rf "$TMP_DIR" + exit 1 + else + echo "PASS: No stderr noise (grep/sed/jq errors) in output" + fi + + else + echo "FAIL: Stub curl all-nodes mode failed with exit code $exit_code, took $((END_TIME - START_TIME))s" + echo "Output: $output" + rm -rf "$TMP_DIR" + exit 1 + fi + + # Test 8: Test that path-blacklist with - prefix works + # Add a compose file that should be excluded by the -node blacklist entry + echo "version: '3'" > test-node-blacklist.yml + echo "services:" >> test-node-blacklist.yml + echo " client:" >> test-node-blacklist.yml + echo " image: test/node-client:v1.0.0" >> test-node-blacklist.yml + echo "chain: test-node-blacklist" >> test-node-blacklist.yml + + # Update COMPOSE_FILE to include the blacklisted file + export COMPOSE_FILE="avalanche-mainnet-go.yml:test-node-blacklist.yml" + output=$(timeout 60 ./node-facts.sh 2>&1) + exit_code=$? + + if [ $exit_code -eq 0 ]; then + # Should not include test-node-blacklist + if echo "$output" | grep -q "test-node-blacklist"; then + echo "FAIL: Blacklisted node was included in output" + rm -rf "$TMP_DIR" + exit 1 + else + echo "PASS: Blacklist with - prefix works correctly" + fi + else + echo "FAIL: Blacklist test failed with exit code $exit_code" + rm -rf "$TMP_DIR" + exit 1 + fi + + rm -rf "$TMP_DIR" +fi + echo "=== Testing JSON schema validation ===" # Function to validate schema