ethereum-rpc-docker/volume-utils.sh

#!/bin/bash

# Shared helpers for compose volume operations.
# Ephemeral volumes (init-container config) are excluded from backup/restore/size.

# Minimum on-disk bytes for a backup archive to be considered restorable (near-empty
# zstd headers from purged/missing volumes must not win sort|tail newest selection).
MIN_RESTORABLE_BACKUP_BYTES=$((1024 * 1024))

# Config volumes hold a handful of small files: a legitimate config archive can sit
# far below the 1MB data floor (cronos: 68KB) while a purged-volume stub is a bare
# zstd header (<4KB). A flat 1MB floor made every node with a _config volume in its
# registry entry report "Restorable: 0" despite healthy multi-GB data backups.
min_restorable_bytes_for() {
  case "$(basename "$1")" in
    *_config-*.tar.zst) echo 4096 ;;
    *) echo "$MIN_RESTORABLE_BACKUP_BYTES" ;;
  esac
}

RPC_ENV_FILE="/root/rpc/.env"

backup_archive_size_bytes() {
  local file=$1
  stat -c%s "$file" 2>/dev/null || echo 0
}

is_restorable_backup_archive() {
  local file=$1
  local size
  [[ -f "$file" ]] || return 1
  size=$(backup_archive_size_bytes "$file")
  [[ "$size" =~ ^[0-9]+$ ]] && (( size >= $(min_restorable_bytes_for "$file") ))
}

# Pick the lexicographically newest backup that is not near-empty (<1MB). Prints one
# skip line per rejected artifact; returns empty when none qualify.
select_newest_local_backup() {
  local backup_dir=$1
  local volume_name=$2
  local pattern="${backup_dir}/${volume_name}"-[0-9]*G.tar.zst
  local newest="" f size

  shopt -s nullglob
  local files=( $pattern )
  shopt -u nullglob
  if (( ${#files[@]} == 0 )); then
    return 0
  fi

  IFS=$'\n' files=( $(printf '%s\n' "${files[@]}" | sort) )
  unset IFS

  for f in "${files[@]}"; do
    size=$(backup_archive_size_bytes "$f")
    if [[ ! "$size" =~ ^[0-9]+$ ]] || (( size < $(min_restorable_bytes_for "$f") )); then
      echo "skip near-empty backup artifact: $(basename "$f") (${size:-0} bytes)" >&2
      continue
    fi
    newest="$f"
  done

  if [[ -n "$newest" ]]; then
    echo "$newest"
  fi
}

remote_backup_archive_size_bytes() {
  local remote_source=$1
  local archive_name=$2
  curl --ipv4 -fsSI "${remote_source}${archive_name}" 2>/dev/null \
    | awk -F': ' 'tolower($1)=="content-length" {gsub(/\r/,"",$2); print $2; exit}'
}

# Newest restorable backup filename (not path) from a remote WebDAV listing on stdin.
select_newest_remote_backup_from_list() {
  local remote_source=$1
  local volume_name=$2
  local newest="" f size
  local -a files=()

  while IFS= read -r f; do
    [[ -n "$f" ]] && files+=("$f")
  done

  if (( ${#files[@]} == 0 )); then
    return 0
  fi

  IFS=$'\n' files=( $(printf '%s\n' "${files[@]}" | sort) )
  unset IFS

  for f in "${files[@]}"; do
    [[ "$f" == "${volume_name}-"* ]] || continue
    [[ "$f" == *.tar.zst ]] || continue
    size=$(remote_backup_archive_size_bytes "$remote_source" "$f")
    if [[ -z "$size" || ! "$size" =~ ^[0-9]+$ ]] || (( size < $(min_restorable_bytes_for "$f") )); then
      echo "skip near-empty backup artifact: $f (${size:-unknown} bytes)" >&2
      continue
    fi
    newest="$f"
  done

  if [[ -n "$newest" ]]; then
    echo "$newest"
  fi
}

compose_fence_marker_path() {
  local compose_name=$1
  local env_dir
  env_dir=$(dirname "$RPC_ENV_FILE")
  echo "${env_dir}/.env.fence-$(echo "$compose_name" | tr '/' '_')"
}

# Remove a compose yml segment from COMPOSE_FILE; store it in a sidecar marker for restore.
fence_compose_in_env() {
  local compose_name=$1
  local yml_segment="${compose_name}.yml"
  local env_file="$RPC_ENV_FILE"
  local marker
  marker=$(compose_fence_marker_path "$compose_name")

  [[ -f "$env_file" ]] || { echo "WARN: $env_file missing, cannot fence $compose_name" >&2; return 1; }
  [[ -f "$marker" ]] && { echo "WARN: fence marker already exists: $marker" >&2; return 1; }

  local line compose_file new_compose removed=0
  line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
  [[ -n "$line" ]] || { echo "WARN: COMPOSE_FILE not found in $env_file" >&2; return 1; }

  compose_file="${line#COMPOSE_FILE=}"
  compose_file="${compose_file#\"}"
  compose_file="${compose_file%\"}"

  local part new_parts=()
  IFS=':' read -ra parts <<< "$compose_file"
  for part in "${parts[@]}"; do
    if [[ "$part" == "$yml_segment" ]]; then
      if ! printf '%s\n' "$part" > "$marker"; then
        echo "WARN: failed to write fence marker $marker" >&2
        return 1
      fi
      removed=1
    else
      new_parts+=("$part")
    fi
  done
  unset IFS

  if (( removed == 0 )); then
    echo "WARN: $yml_segment not present in COMPOSE_FILE (already fenced?)" >&2
    return 1
  fi

  new_compose=$(IFS=':'; echo "${new_parts[*]}")
  if grep -q '^COMPOSE_FILE=' "$env_file"; then
    sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${new_compose}|" "$env_file"
  else
    echo "COMPOSE_FILE=${new_compose}" >> "$env_file"
  fi
  echo "Fenced $yml_segment from COMPOSE_FILE (marker: $marker)"
}

# Restore a fenced compose segment before service-scoped start (start needs the entry).
unfence_compose_in_env() {
  local compose_name=$1
  local yml_segment="${compose_name}.yml"
  local env_file="$RPC_ENV_FILE"
  local marker
  marker=$(compose_fence_marker_path "$compose_name")

  [[ -f "$marker" ]] || return 0

  local segment line compose_file
  segment=$(cat "$marker")
  [[ -n "$segment" ]] || { rm -f "$marker"; return 0; }

  if [[ -f "$env_file" ]]; then
    line=$(grep -E '^COMPOSE_FILE=' "$env_file" | head -n 1)
    compose_file="${line#COMPOSE_FILE=}"
    compose_file="${compose_file#\"}"
    compose_file="${compose_file%\"}"
    case ":$compose_file:" in
      *:"$segment":*) ;;
      *)
        if [[ -n "$compose_file" ]]; then
          compose_file="${compose_file}:${segment}"
        else
          compose_file="$segment"
        fi
        sed -i "s|^COMPOSE_FILE=.*|COMPOSE_FILE=${compose_file}|" "$env_file"
        ;;
    esac
  else
    echo "COMPOSE_FILE=${segment}" > "$env_file"
  fi

  rm -f "$marker"
  echo "Unfenced $yml_segment into COMPOSE_FILE"
}

is_ephemeral_volume_key() {
  local key=$1
  local compose_file=$2

  # op-node rollup config volumes
  [[ "$key" == *_node_config ]] && return 0

  if [[ -n "$compose_file" && -f "$compose_file" ]]; then
    local ephemeral
    ephemeral=$(yaml2json < "$compose_file" 2>/dev/null | jq -r '.["x-ephemeral-volumes"] // [] | .[]' 2>/dev/null)
    while IFS= read -r vol; do
      [[ -z "$vol" ]] && continue
      [[ "$key" == "$vol" ]] && return 0
    done <<< "$ephemeral"
  fi

  return 1
}

get_volume_keys() {
  local compose_file=$1
  yaml2json < "$compose_file" 2>/dev/null | jq -r '.volumes | keys[]' 2>/dev/null
}

get_persistent_volume_keys() {
  local compose_file=$1
  local key

  while IFS= read -r key; do
    [[ -z "$key" ]] && continue
    if ! is_ephemeral_volume_key "$key" "$compose_file"; then
      echo "$key"
    fi
  done < <(get_volume_keys "$compose_file")
}

# Returns 0 when a backup HTTP/WebDAV URL refers to this machine.
is_local_backup_url() {
  local url=$1
  [[ -z "$url" ]] && return 1

  local host="${url#*://}"
  host="${host%%/*}"
  host="${host%%:*}"
  [[ -z "$host" ]] && return 1

  case "$host" in
    localhost|127.0.0.1|::1|0.0.0.0) return 0 ;;
  esac

  local env_file
  env_file="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.env"
  if [[ -f "$env_file" ]]; then
    local domain ip
    domain=$(grep -E '^DOMAIN=' "$env_file" | head -n 1 | cut -d= -f2- | tr -d '"' | tr -d "'")
    ip=$(grep -E '^IP=' "$env_file" | head -n 1 | cut -d= -f2- | tr -d '"' | tr -d "'")
    [[ -n "$domain" && "$host" == "$domain" ]] && return 0
    [[ -n "$ip" && "$host" == "$ip" ]] && return 0
  fi

  local local_fqdn local_short
  local_fqdn=$(hostname -f 2>/dev/null || true)
  local_short=$(hostname 2>/dev/null || true)
  [[ -n "$local_fqdn" && "$host" == "$local_fqdn" ]] && return 0
  [[ -n "$local_short" && "$host" == "$local_short" ]] && return 0

  return 1
}

# Delete the /slowdisk static-file offload targets for a single volume key, so removing a
# volume also frees the static data behind the symlinks (otherwise it leaks on /slowdisk).
# Safe: scoped to the key and to /slowdisk, and only removes dirs matching the
# rpc_<key>__data_ naming this repo creates (the same pattern delete-volumes.sh deletes).
delete_slowdisk_targets_for_key() {
  local key=$1
  [[ -z "$key" ]] && return 0
  [[ -d /slowdisk ]] || return 0
  local data_dir="/var/lib/docker/volumes/rpc_${key}/_data"
  # 1) follow live symlinks in the volume and delete their /slowdisk targets
  if [[ -d "$data_dir" ]]; then
    while IFS= read -r -d '' l; do
      local t; t=$(readlink -f "$l" 2>/dev/null)
      if [[ -n "$t" && -d "$t" && "$t" == /slowdisk/rpc_${key}__data_* ]]; then
        echo "    rm slowdisk static target $t"; rm -rf "$t"
      fi
    done < <(find "$data_dir" -type l -print0 2>/dev/null)
  fi
  # 2) sweep any orphaned targets for this key (re-restore / dangling-symlink safety)
  local t
  shopt -s nullglob
  for t in /slowdisk/rpc_${key}__data_*; do
    [[ -d "$t" ]] && { echo "    rm orphan slowdisk static target $t"; rm -rf "$t"; }
  done
  shopt -u nullglob
}