reload_dshackle.sh: make gateway reconnect conditional on upstream/disabled-method removals
Operator ruling: NEVER force a reconnect on a routine reload/rpc-update run. Only upstream REMOVALS or new disabled-method ADDITIONS require a gateway reconnect to be seen by dRPC gateways. Implement snapshot-based comparison: - Before SIGHUP, read current advertised set (upstream IDs + disabled methods) - Compare against previous snapshot (if exists) - Drop established gRPC connections ONLY IF: * an upstream ID disappeared, OR * a new disabled-method entry appeared - Pure additions or no change: reload only, no drop - First run with no prior snapshot: do NOT drop, just write snapshot - Always rewrite snapshot after successful reload - All operations best-effort (failures never fail the script) Live-measured: upstream/method additions propagate over existing connection in ~16s (no reconnect needed). Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -5,6 +5,137 @@
|
||||
# survive every configure-drpc regeneration. See apply-dshackle-overrides.py.
|
||||
[ -f /root/rpc/apply-dshackle-overrides.py ] && python3 /root/rpc/apply-dshackle-overrides.py
|
||||
|
||||
# --- Snapshot-based conditional gateway reconnect ---
|
||||
# Operator ruling (2026-08-27): NEVER force a reconnect on a routine reload/rpc-update run.
|
||||
# Only upstream REMOVALS or new disabled-method ADDITIONS require a gateway reconnect.
|
||||
#
|
||||
# For each dshackle container, we snapshot the advertised set:
|
||||
# (a) upstream ids present in rendered yaml configs
|
||||
# (b) disabled-method entries (from methods.disabled across all upstreams)
|
||||
# into a persisted file: /root/rpc/.dshackle-adv-<container>.snapshot
|
||||
#
|
||||
# On each run:
|
||||
# - Read current config state from disk
|
||||
# - If previous snapshot exists, compare: drop connections ONLY IF
|
||||
# * an upstream id disappeared (was in snapshot, not in current), OR
|
||||
# * a new disabled-method entry appeared (in current, not in snapshot)
|
||||
# - Pure additions or no change: reload only, no drop
|
||||
# - First run with no prior snapshot: do NOT drop, just write snapshot after reload
|
||||
# - Always rewrite snapshot after successful reload (for next run to compare against)
|
||||
# - All snapshot/compare operations are best-effort; failures never fail the script
|
||||
|
||||
# Helper: extract the host config directory path for a dshackle container.
|
||||
# dshackle containers mount their config at /etc/dshackle; we find the source of that mount.
|
||||
get_config_dir() {
|
||||
local cid="$1"
|
||||
local mount
|
||||
mount=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/etc/dshackle"}}{{.Source}}{{end}}{{end}}' "$cid" 2>/dev/null) || true
|
||||
[ -n "$mount" ] && echo "$mount" && return 0
|
||||
# Fallback: try common locations (best-effort)
|
||||
for d in /root/rpc/main_configs /root/rpc/free_configs; do
|
||||
[ -d "$d" ] && echo "$d" && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Helper: extract advertised set (upstream ids + disabled methods) from config directory.
|
||||
# Outputs two lines to stdout:
|
||||
# upstream_ids (space-separated, sorted)
|
||||
# disabled_methods (space-separated, sorted)
|
||||
get_advertised_set() {
|
||||
local config_dir="$1"
|
||||
local upstream_ids=""
|
||||
local disabled_methods=""
|
||||
|
||||
[ -d "$config_dir" ] || return 1
|
||||
|
||||
# Use python3 to parse YAML (available on host). Best-effort: if python fails, return empty.
|
||||
python3 -c "
|
||||
import yaml, glob, sys, os
|
||||
config_dir = sys.argv[1]
|
||||
upstream_ids = set()
|
||||
disabled_methods = set()
|
||||
for path in glob.glob(os.path.join(config_dir, '*.yaml')):
|
||||
try:
|
||||
with open(path) as f:
|
||||
doc = yaml.safe_load(f)
|
||||
if not isinstance(doc, dict):
|
||||
continue
|
||||
for up in doc.get('upstreams') or []:
|
||||
if not isinstance(up, dict):
|
||||
continue
|
||||
uid = up.get('id')
|
||||
if uid:
|
||||
upstream_ids.add(uid)
|
||||
methods = up.get('methods', {})
|
||||
if isinstance(methods, dict):
|
||||
for m in methods.get('disabled') or []:
|
||||
if isinstance(m, dict):
|
||||
name = m.get('name')
|
||||
if name:
|
||||
disabled_methods.add(name)
|
||||
elif isinstance(m, str):
|
||||
disabled_methods.add(m)
|
||||
except Exception:
|
||||
pass
|
||||
print(' '.join(sorted(upstream_ids)))
|
||||
print(' '.join(sorted(disabled_methods)))
|
||||
" "$config_dir" 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
|
||||
# Helper: read snapshot file. Returns upstream_ids and disabled_methods.
|
||||
read_snapshot() {
|
||||
local snapshot_file="$1"
|
||||
[ -f "$snapshot_file" ] || return 1
|
||||
local line1 line2
|
||||
line1=$(head -1 "$snapshot_file" 2>/dev/null) || true
|
||||
line2=$(tail -1 "$snapshot_file" 2>/dev/null) || true
|
||||
[ -n "$line1" ] && echo "$line1"
|
||||
[ -n "$line2" ] && echo "$line2"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Helper: write snapshot file.
|
||||
write_snapshot() {
|
||||
local snapshot_file="$1"
|
||||
local upstream_ids="$2"
|
||||
local disabled_methods="$3"
|
||||
echo "$upstream_ids" > "$snapshot_file" 2>/dev/null || true
|
||||
echo "$disabled_methods" >> "$snapshot_file" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Helper: check if drop is needed.
|
||||
# Args: old_upstream_ids, old_disabled_methods, new_upstream_ids, new_disabled_methods
|
||||
# Returns 0 (true) if drop is needed, 1 (false) otherwise.
|
||||
drop_needed() {
|
||||
local old_ids="$1" new_ids="$3"
|
||||
local old_methods="$2" new_methods="$4"
|
||||
|
||||
# Check if any upstream id disappeared (in old, not in new)
|
||||
for uid in $old_ids; do
|
||||
[ -z "$uid" ] && continue
|
||||
# Check if uid is NOT in new_ids
|
||||
found=0
|
||||
for nuid in $new_ids; do
|
||||
[ "$uid" = "$nuid" ] && found=1 && break
|
||||
done
|
||||
[ "$found" = 0 ] && return 0 # drop needed
|
||||
done
|
||||
|
||||
# Check if any new disabled method appeared (in new, not in old)
|
||||
for meth in $new_methods; do
|
||||
[ -z "$meth" ] && continue
|
||||
found=0
|
||||
for ometh in $old_methods; do
|
||||
[ "$meth" = "$ometh" ] && found=1 && break
|
||||
done
|
||||
[ "$found" = 0 ] && return 0 # drop needed
|
||||
done
|
||||
|
||||
return 1 # no drop needed
|
||||
}
|
||||
|
||||
# 1:1 invariant (2026-07-15): refuse to ACTIVATE a dshackle config routing >1 node for the same
|
||||
# chain — we can't attribute traffic to multiple nodes behind one proxy (no per-upstream request
|
||||
# metric; conn-seconds biases it), and the attribution model + planner assume 1:1. The validator
|
||||
@@ -37,6 +168,40 @@ RC=0
|
||||
for CID in $(docker ps -q -f "name=dshackle"); do
|
||||
NAME=$(docker inspect -f '{{.Name}}' "$CID" 2>/dev/null | sed 's|^/||')
|
||||
[ -n "$NAME" ] || NAME="$CID"
|
||||
|
||||
# --- Snapshot-based conditional gateway reconnect ---
|
||||
# Determine config directory and snapshot file for this container.
|
||||
CONFIG_DIR=$(get_config_dir "$CID") || true
|
||||
# Sanitize name for use in snapshot filename: replace / with -, remove leading -
|
||||
SANITIZED_NAME=$(echo "$NAME" | tr '/' '-' | sed 's/^-//')
|
||||
SNAPSHOT_FILE="/root/rpc/.dshackle-adv-${SANITIZED_NAME}.snapshot"
|
||||
|
||||
# Read current advertised set from the (new) on-disk config.
|
||||
CURRENT_SET=$(get_advertised_set "$CONFIG_DIR") || true
|
||||
CURRENT_UPSTREAMS=$(echo "$CURRENT_SET" | head -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
CURRENT_METHODS=$(echo "$CURRENT_SET" | tail -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
|
||||
# Read previous snapshot if it exists.
|
||||
PREVIOUS_UPSTREAMS=""
|
||||
PREVIOUS_METHODS=""
|
||||
HAS_PREVIOUS_SNAPSHOT=0
|
||||
if [ -f "$SNAPSHOT_FILE" ]; then
|
||||
PREVIOUS_SET=$(read_snapshot "$SNAPSHOT_FILE") || true
|
||||
PREVIOUS_UPSTREAMS=$(echo "$PREVIOUS_SET" | head -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
PREVIOUS_METHODS=$(echo "$PREVIOUS_SET" | tail -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
HAS_PREVIOUS_SNAPSHOT=1
|
||||
fi
|
||||
|
||||
# Decide if we need to drop connections: only if there were removals.
|
||||
NEED_DROP=0
|
||||
if [ "$HAS_PREVIOUS_SNAPSHOT" -eq 1 ]; then
|
||||
# Has previous snapshot: check for removals
|
||||
if drop_needed "$PREVIOUS_UPSTREAMS" "$PREVIOUS_METHODS" "$CURRENT_UPSTREAMS" "$CURRENT_METHODS"; then
|
||||
NEED_DROP=1
|
||||
fi
|
||||
fi
|
||||
# If no previous snapshot, NEED_DROP stays 0 (first run: do NOT drop)
|
||||
|
||||
T0=$(date +%s)
|
||||
if ! docker kill --signal=HUP "$CID" >/dev/null 2>&1; then
|
||||
echo "reload_dshackle: WARNING: failed to SIGHUP $NAME" >&2
|
||||
@@ -52,11 +217,17 @@ for CID in $(docker ps -q -f "name=dshackle"); do
|
||||
REASON="concurrent HUP dropped (reload skipped)"
|
||||
fi
|
||||
[ -z "$REASON" ] && {
|
||||
# Drop established gateway gRPC connections to force dRPC edge to reconnect
|
||||
# and re-read the advertised chain/method list. Gateways only re-initialize
|
||||
# advertisements on reconnection. Best-effort: failure does NOT fail the script.
|
||||
PID=$(docker inspect -f '{{.State.Pid}}' "$CID" 2>/dev/null) && \
|
||||
nsenter -t "$PID" -n ss -K state established '( sport = :2449 )' >/dev/null 2>&1
|
||||
# Reload succeeded. Always rewrite snapshot for next run.
|
||||
write_snapshot "$SNAPSHOT_FILE" "$CURRENT_UPSTREAMS" "$CURRENT_METHODS"
|
||||
|
||||
# Drop established gateway gRPC connections ONLY if removals were detected.
|
||||
if [ "$NEED_DROP" -eq 1 ]; then
|
||||
PID=$(docker inspect -f '{{.State.Pid}}' "$CID" 2>/dev/null) && \
|
||||
nsenter -t "$PID" -n ss -K state established '( sport = :2449 )' >/dev/null 2>&1
|
||||
echo "reload_dshackle: $NAME reload applied, gateway connections dropped (removals detected)" >&2
|
||||
else
|
||||
echo "reload_dshackle: $NAME reload applied, no removals — gateway connections kept" >&2
|
||||
fi
|
||||
continue
|
||||
}
|
||||
echo "reload_dshackle: $NAME reload did NOT apply ($REASON) — restarting to load config fresh" >&2
|
||||
@@ -71,7 +242,15 @@ for CID in $(docker ps -q -f "name=dshackle"); do
|
||||
sleep 2
|
||||
done
|
||||
if [ -n "$UP" ]; then
|
||||
# Restart succeeded. Always rewrite snapshot for next run.
|
||||
write_snapshot "$SNAPSHOT_FILE" "$CURRENT_UPSTREAMS" "$CURRENT_METHODS"
|
||||
echo "reload_dshackle: $NAME restarted, config applied on clean start" >&2
|
||||
# On restart, the config is freshly loaded, so we need to drop if removals were detected.
|
||||
if [ "$NEED_DROP" -eq 1 ]; then
|
||||
PID=$(docker inspect -f '{{.State.Pid}}' "$CID" 2>/dev/null) && \
|
||||
nsenter -t "$PID" -n ss -K state established '( sport = :2449 )' >/dev/null 2>&1
|
||||
echo "reload_dshackle: $NAME restart applied, gateway connections dropped (removals detected)" >&2
|
||||
fi
|
||||
else
|
||||
echo "reload_dshackle: ERROR: $NAME not running after restart — manual intervention" >&2
|
||||
RC=1
|
||||
|
||||
Reference in New Issue
Block a user