diff --git a/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml b/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml index d213750e..eb39035f 100644 --- a/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml +++ b/ethereum/reth/ethereum-mainnet-reth-minimal-trace--nimbus.yml @@ -111,53 +111,10 @@ services: ethereum-mainnet-minimal-pruner: image: debian:latest - entrypoint: [/bin/bash, -c] - command: - - | - mkdir -p "/data/static_files/delete_me" - # Step 1: List base filenames (without .conf/.off) and sort by starting block number - base_files=$$(find "/data/static_files" -maxdepth 1 -type f | sed -E "s/\.(conf|off)$$" | sort -u | awk -F_ '{print $$NF-0, $$0}' | sort -n | cut -d" " -f2-) - # Convert base_files to an array for easier manipulation - base_files_array=($$base_files) - # Step 3: Group files by prefix and block range, keeping only the last two block ranges - declare -A file_groups - # Group files by prefix - for base in "$${base_files_array[@]}"; do - prefix=$$(echo "$$base" | sed -E "s/_([0-9]+)$$"//") # Get everything before the block range - block_range=$$(echo "$$base" | sed -E "s/.*_([0-9]+)$$"//") # Get the block range - file_groups["$$prefix"]+="$$block_range:$$base " - done - # Step 4: Process each group - for prefix in "$${!file_groups[@]}"; do - block_ranges=($${file_groups[$$prefix]}) - num_files=$${#block_ranges[@]} - # Keep the last 2 block ranges - files_to_move=("$${block_ranges[@]:0:num_files-2}") - # Move files for the current group - for file_range in "$${files_to_move[@]}"; do - base="$${file_range#*:}" # Remove block range part, keeping the full filename - # Handle base file (no extension) first - if [[ -f "$$base" ]]; then - size=$$(stat --printf="%s" "$$base") - mv "$$base" "/data/static_files/delete_me" - echo "Moved $$base to backup." - fi - # Handle files with extensions .conf and .off - for ext in .conf .off; do # Removed quotes and empty string from loop list - file="$${base}$${ext}" - if [[ -f "$$file" ]]; then - size=$$(stat --printf="%s" "$$file") - mv "$$file" "/data/static_files/delete_me" - echo "Moved $$file to backup." - fi - done - done - done - freed_bytes=$$(du -cb "/data/static_files/delete_me"/* | tail -1 | awk '{print $$1}') - echo "Moved files to backup. Total freed: $$freed_bytes bytes." - echo "Done." + entrypoint: [/scripts/prune-reth.sh] restart: no volumes: + - ./scripts/prune-reth.sh:/scripts/prune-reth.sh - /root/.local/share/reth:/data volumes: diff --git a/scripts/prune-reth.sh b/scripts/prune-reth.sh new file mode 100644 index 00000000..1ee0891c --- /dev/null +++ b/scripts/prune-reth.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +set -e # Exit immediately if a command exits with a non-zero status. +set -u # Treat unset variables as an error when substituting. + +DATA_DIR="${DATA_DIR:-/data}" +STATIC_FILES_DIR="${STATIC_FILES_DIR:-/data/static_files}" +DELETE_DIR="${DELETE_DIR:-/data/static_files/delete_me}" + +echo "Starting Reth pruning process for static files in $STATIC_FILES_DIR..." + +mkdir -p "$DELETE_DIR" + +# Step 1: List base filenames (without .conf/.off) and sort by starting block number +# Find files, remove extensions, sort uniquely, extract block number for numeric sort, then get original base name +echo "Finding and sorting static files..." +base_files=$(find "$STATIC_FILES_DIR" -maxdepth 1 -type f \( -name '*.conf' -o -name '*.off' -o -name '*[0-9]' \) | \ + sed -E "s/\.(conf|off)$//" | \ + sort -u | \ + awk -F_ '{print $NF+0, $0}' | \ + sort -n | \ + cut -d" " -f2-) + +if [ -z "$base_files" ]; then + echo "No static files found to process." + exit 0 +fi + +# Convert base_files to an array for easier manipulation +readarray -t base_files_array <<< "$base_files" +echo "Found ${#base_files_array[@]} unique base file ranges." + +# Step 3: Group files by prefix and block range, keeping only the last two block ranges +declare -A file_groups + +# Group files by prefix +echo "Grouping files by prefix..." +for base in "${base_files_array[@]}"; do + prefix=$(echo "$base" | sed -E "s/_([0-9]+)$//") # Get everything before the block range + block_range=$(echo "$base" | sed -E "s/.*_([0-9]+)$//") # Get the block range + file_groups["$prefix"]+="$block_range:$base " +done + +# Step 4: Process each group +echo "Processing file groups to identify files for removal..." +moved_count=0 +for prefix in "${!file_groups[@]}"; do + # Read ranges into an array, sorting numerically by block range (the part before ':') + readarray -t block_ranges < <(echo "${file_groups[$prefix]}" | tr ' ' '\n' | sort -t: -k1,1n) + + num_files=${#block_ranges[@]} + echo "Processing group '$prefix' with $num_files ranges." + + # Keep the last 2 block ranges (or fewer if less than 2 exist) + keep_count=2 + if [ "$num_files" -le "$keep_count" ]; then + echo "Keeping all files for group '$prefix' as there are $num_files ranges (<= $keep_count)." + continue + fi + + num_to_move=$((num_files - keep_count)) + echo "Identified $num_to_move ranges to move for group '$prefix'." + + # Get the ranges to move (all except the last 'keep_count') + files_to_move=("${block_ranges[@]:0:$num_to_move}") + + # Move files for the current group + for file_range in "${files_to_move[@]}"; do + base="${file_range#*:}" # Remove block range part, keeping the full filename path + + # Handle files with extensions .conf and .off first + for ext in .conf .off; do + file="${base}${ext}" + if [[ -f "$file" ]]; then + echo "Moving $file to $DELETE_DIR" + mv "$file" "$DELETE_DIR/" + moved_count=$((moved_count + 1)) + fi + done + + # Handle base file (no extension) - check if it exists and is a file + if [[ -f "$base" && ! "$base" =~ \.(conf|off)$ ]]; then + echo "Moving $base to $DELETE_DIR" + mv "$base" "$DELETE_DIR/" + moved_count=$((moved_count + 1)) + fi + done +done + +if [ "$moved_count" -eq 0 ]; then + echo "No files needed moving based on the retention policy." +else + echo "Moved $moved_count files to $DELETE_DIR." + freed_bytes=$(du -cb "$DELETE_DIR"/* | tail -1 | awk '{print $1}') + echo "Total space potentially freed (before deletion): $freed_bytes bytes." +fi + +# Optional: Add command to actually delete files in DELETE_DIR if desired +# echo "Deleting files in $DELETE_DIR..." +# rm -rf "$DELETE_DIR"/* + +echo "Pruning script finished." \ No newline at end of file