Claude Agent
3478d286f6
Operator ruling: NEVER force a reconnect on a routine reload/rpc-update run. Only upstream REMOVALS or new disabled-method ADDITIONS require a gateway reconnect to be seen by dRPC gateways. Implement snapshot-based comparison: - Before SIGHUP, read current advertised set (upstream IDs + disabled methods) - Compare against previous snapshot (if exists) - Drop established gRPC connections ONLY IF: * an upstream ID disappeared, OR * a new disabled-method entry appeared - Pure additions or no change: reload only, no drop - First run with no prior snapshot: do NOT drop, just write snapshot - Always rewrite snapshot after successful reload - All operations best-effort (failures never fail the script) Live-measured: upstream/method additions propagate over existing connection in ~16s (no reconnect needed). Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
261 lines
11 KiB
Bash
Executable File
261 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Re-weave host-local upstream overrides (method disables etc.) into the freshly
|
|
# pushed configs BEFORE signaling dshackle - makes /root/rpc-local/dshackle-overrides.yaml
|
|
# survive every configure-drpc regeneration. See apply-dshackle-overrides.py.
|
|
[ -f /root/rpc/apply-dshackle-overrides.py ] && python3 /root/rpc/apply-dshackle-overrides.py
|
|
|
|
# --- Snapshot-based conditional gateway reconnect ---
|
|
# Operator ruling (2026-08-27): NEVER force a reconnect on a routine reload/rpc-update run.
|
|
# Only upstream REMOVALS or new disabled-method ADDITIONS require a gateway reconnect.
|
|
#
|
|
# For each dshackle container, we snapshot the advertised set:
|
|
# (a) upstream ids present in rendered yaml configs
|
|
# (b) disabled-method entries (from methods.disabled across all upstreams)
|
|
# into a persisted file: /root/rpc/.dshackle-adv-<container>.snapshot
|
|
#
|
|
# On each run:
|
|
# - Read current config state from disk
|
|
# - If previous snapshot exists, compare: drop connections ONLY IF
|
|
# * an upstream id disappeared (was in snapshot, not in current), OR
|
|
# * a new disabled-method entry appeared (in current, not in snapshot)
|
|
# - Pure additions or no change: reload only, no drop
|
|
# - First run with no prior snapshot: do NOT drop, just write snapshot after reload
|
|
# - Always rewrite snapshot after successful reload (for next run to compare against)
|
|
# - All snapshot/compare operations are best-effort; failures never fail the script
|
|
|
|
# Helper: extract the host config directory path for a dshackle container.
|
|
# dshackle containers mount their config at /etc/dshackle; we find the source of that mount.
|
|
get_config_dir() {
|
|
local cid="$1"
|
|
local mount
|
|
mount=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/etc/dshackle"}}{{.Source}}{{end}}{{end}}' "$cid" 2>/dev/null) || true
|
|
[ -n "$mount" ] && echo "$mount" && return 0
|
|
# Fallback: try common locations (best-effort)
|
|
for d in /root/rpc/main_configs /root/rpc/free_configs; do
|
|
[ -d "$d" ] && echo "$d" && return 0
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Helper: extract advertised set (upstream ids + disabled methods) from config directory.
|
|
# Outputs two lines to stdout:
|
|
# upstream_ids (space-separated, sorted)
|
|
# disabled_methods (space-separated, sorted)
|
|
get_advertised_set() {
|
|
local config_dir="$1"
|
|
local upstream_ids=""
|
|
local disabled_methods=""
|
|
|
|
[ -d "$config_dir" ] || return 1
|
|
|
|
# Use python3 to parse YAML (available on host). Best-effort: if python fails, return empty.
|
|
python3 -c "
|
|
import yaml, glob, sys, os
|
|
config_dir = sys.argv[1]
|
|
upstream_ids = set()
|
|
disabled_methods = set()
|
|
for path in glob.glob(os.path.join(config_dir, '*.yaml')):
|
|
try:
|
|
with open(path) as f:
|
|
doc = yaml.safe_load(f)
|
|
if not isinstance(doc, dict):
|
|
continue
|
|
for up in doc.get('upstreams') or []:
|
|
if not isinstance(up, dict):
|
|
continue
|
|
uid = up.get('id')
|
|
if uid:
|
|
upstream_ids.add(uid)
|
|
methods = up.get('methods', {})
|
|
if isinstance(methods, dict):
|
|
for m in methods.get('disabled') or []:
|
|
if isinstance(m, dict):
|
|
name = m.get('name')
|
|
if name:
|
|
disabled_methods.add(name)
|
|
elif isinstance(m, str):
|
|
disabled_methods.add(m)
|
|
except Exception:
|
|
pass
|
|
print(' '.join(sorted(upstream_ids)))
|
|
print(' '.join(sorted(disabled_methods)))
|
|
" "$config_dir" 2>/dev/null
|
|
return 0
|
|
}
|
|
|
|
# Helper: read snapshot file. Returns upstream_ids and disabled_methods.
|
|
read_snapshot() {
|
|
local snapshot_file="$1"
|
|
[ -f "$snapshot_file" ] || return 1
|
|
local line1 line2
|
|
line1=$(head -1 "$snapshot_file" 2>/dev/null) || true
|
|
line2=$(tail -1 "$snapshot_file" 2>/dev/null) || true
|
|
[ -n "$line1" ] && echo "$line1"
|
|
[ -n "$line2" ] && echo "$line2"
|
|
return 0
|
|
}
|
|
|
|
# Helper: write snapshot file.
|
|
write_snapshot() {
|
|
local snapshot_file="$1"
|
|
local upstream_ids="$2"
|
|
local disabled_methods="$3"
|
|
echo "$upstream_ids" > "$snapshot_file" 2>/dev/null || true
|
|
echo "$disabled_methods" >> "$snapshot_file" 2>/dev/null || true
|
|
}
|
|
|
|
# Helper: check if drop is needed.
|
|
# Args: old_upstream_ids, old_disabled_methods, new_upstream_ids, new_disabled_methods
|
|
# Returns 0 (true) if drop is needed, 1 (false) otherwise.
|
|
drop_needed() {
|
|
local old_ids="$1" new_ids="$3"
|
|
local old_methods="$2" new_methods="$4"
|
|
|
|
# Check if any upstream id disappeared (in old, not in new)
|
|
for uid in $old_ids; do
|
|
[ -z "$uid" ] && continue
|
|
# Check if uid is NOT in new_ids
|
|
found=0
|
|
for nuid in $new_ids; do
|
|
[ "$uid" = "$nuid" ] && found=1 && break
|
|
done
|
|
[ "$found" = 0 ] && return 0 # drop needed
|
|
done
|
|
|
|
# Check if any new disabled method appeared (in new, not in old)
|
|
for meth in $new_methods; do
|
|
[ -z "$meth" ] && continue
|
|
found=0
|
|
for ometh in $old_methods; do
|
|
[ "$meth" = "$ometh" ] && found=1 && break
|
|
done
|
|
[ "$found" = 0 ] && return 0 # drop needed
|
|
done
|
|
|
|
return 1 # no drop needed
|
|
}
|
|
|
|
# 1:1 invariant (2026-07-15): refuse to ACTIVATE a dshackle config routing >1 node for the same
|
|
# chain — we can't attribute traffic to multiple nodes behind one proxy (no per-upstream request
|
|
# metric; conn-seconds biases it), and the attribution model + planner assume 1:1. The validator
|
|
# reads /root/rpc/main_configs/*.yaml; on violation it exits 1 and we keep the live config.
|
|
# See /root/proxy-1to1-invariant-plan.md + rpc/validate-dshackle-1to1.py.
|
|
if [ -f /root/rpc/validate-dshackle-1to1.py ]; then
|
|
python3 /root/rpc/validate-dshackle-1to1.py /root/rpc/main_configs || {
|
|
echo "reload_dshackle.sh: REFUSING reload — 1:1 invariant violated (above); keeping live config" >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
|
|
# Signal dshackle to reload, then VERIFY the reload applied. dshackle's SIGHUP reload is
|
|
# unreliable for upstream REMOVALS (and method-set changes, same remove+add path): it can
|
|
# throw internally, log "Config is not reloaded, cause - ...", and SILENTLY keep serving the
|
|
# old config (in-memory config updated, runtime selectors stale = drift). Note
|
|
# "Reloading config has been completed" is NOT success — it's a finally-block and prints even
|
|
# when a processor threw. There is no runtime admin API to mutate upstreams, so a container
|
|
# restart is the only way to actually apply such changes.
|
|
# Per container: SIGHUP, check the post-SIGHUP logs for a failure/drop marker, restart on
|
|
# failure. Exit 0 when the config ends up applied (via SIGHUP or restart); non-zero only if a
|
|
# needed restart failed or the container didn't come back (so the deploy fails loudly instead
|
|
# of silently drifting).
|
|
set -u
|
|
FAIL_MARK='Config is not reloaded, cause -' # a reload processor threw (removal/method bug)
|
|
DROP_MARK='Reloading is in progress' # a concurrent HUP was dropped (reload skipped)
|
|
SETTLE=3 # seconds for the SIGHUP handler to run + log
|
|
RC=0
|
|
|
|
for CID in $(docker ps -q -f "name=dshackle"); do
|
|
NAME=$(docker inspect -f '{{.Name}}' "$CID" 2>/dev/null | sed 's|^/||')
|
|
[ -n "$NAME" ] || NAME="$CID"
|
|
|
|
# --- Snapshot-based conditional gateway reconnect ---
|
|
# Determine config directory and snapshot file for this container.
|
|
CONFIG_DIR=$(get_config_dir "$CID") || true
|
|
# Sanitize name for use in snapshot filename: replace / with -, remove leading -
|
|
SANITIZED_NAME=$(echo "$NAME" | tr '/' '-' | sed 's/^-//')
|
|
SNAPSHOT_FILE="/root/rpc/.dshackle-adv-${SANITIZED_NAME}.snapshot"
|
|
|
|
# Read current advertised set from the (new) on-disk config.
|
|
CURRENT_SET=$(get_advertised_set "$CONFIG_DIR") || true
|
|
CURRENT_UPSTREAMS=$(echo "$CURRENT_SET" | head -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
|
CURRENT_METHODS=$(echo "$CURRENT_SET" | tail -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
|
|
|
# Read previous snapshot if it exists.
|
|
PREVIOUS_UPSTREAMS=""
|
|
PREVIOUS_METHODS=""
|
|
HAS_PREVIOUS_SNAPSHOT=0
|
|
if [ -f "$SNAPSHOT_FILE" ]; then
|
|
PREVIOUS_SET=$(read_snapshot "$SNAPSHOT_FILE") || true
|
|
PREVIOUS_UPSTREAMS=$(echo "$PREVIOUS_SET" | head -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
|
PREVIOUS_METHODS=$(echo "$PREVIOUS_SET" | tail -1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
|
HAS_PREVIOUS_SNAPSHOT=1
|
|
fi
|
|
|
|
# Decide if we need to drop connections: only if there were removals.
|
|
NEED_DROP=0
|
|
if [ "$HAS_PREVIOUS_SNAPSHOT" -eq 1 ]; then
|
|
# Has previous snapshot: check for removals
|
|
if drop_needed "$PREVIOUS_UPSTREAMS" "$PREVIOUS_METHODS" "$CURRENT_UPSTREAMS" "$CURRENT_METHODS"; then
|
|
NEED_DROP=1
|
|
fi
|
|
fi
|
|
# If no previous snapshot, NEED_DROP stays 0 (first run: do NOT drop)
|
|
|
|
T0=$(date +%s)
|
|
if ! docker kill --signal=HUP "$CID" >/dev/null 2>&1; then
|
|
echo "reload_dshackle: WARNING: failed to SIGHUP $NAME" >&2
|
|
RC=1
|
|
continue
|
|
fi
|
|
sleep "$SETTLE"
|
|
LOGS=$(docker logs --since "$T0" "$CID" 2>&1)
|
|
REASON=""
|
|
if echo "$LOGS" | grep -qF "$FAIL_MARK"; then
|
|
REASON=$(echo "$LOGS" | grep -F "$FAIL_MARK" | head -1)
|
|
elif echo "$LOGS" | grep -qF "$DROP_MARK"; then
|
|
REASON="concurrent HUP dropped (reload skipped)"
|
|
fi
|
|
[ -z "$REASON" ] && {
|
|
# Reload succeeded. Always rewrite snapshot for next run.
|
|
write_snapshot "$SNAPSHOT_FILE" "$CURRENT_UPSTREAMS" "$CURRENT_METHODS"
|
|
|
|
# Drop established gateway gRPC connections ONLY if removals were detected.
|
|
if [ "$NEED_DROP" -eq 1 ]; then
|
|
PID=$(docker inspect -f '{{.State.Pid}}' "$CID" 2>/dev/null) && \
|
|
nsenter -t "$PID" -n ss -K state established '( sport = :2449 )' >/dev/null 2>&1
|
|
echo "reload_dshackle: $NAME reload applied, gateway connections dropped (removals detected)" >&2
|
|
else
|
|
echo "reload_dshackle: $NAME reload applied, no removals — gateway connections kept" >&2
|
|
fi
|
|
continue
|
|
}
|
|
echo "reload_dshackle: $NAME reload did NOT apply ($REASON) — restarting to load config fresh" >&2
|
|
if ! docker restart "$CID" >/dev/null 2>&1; then
|
|
echo "reload_dshackle: ERROR: restart failed for $NAME — config NOT applied, manual intervention" >&2
|
|
RC=1
|
|
continue
|
|
fi
|
|
UP=""
|
|
for _ in $(seq 1 30); do
|
|
if docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null | grep -q true; then UP=1; break; fi
|
|
sleep 2
|
|
done
|
|
if [ -n "$UP" ]; then
|
|
# Restart succeeded. Always rewrite snapshot for next run.
|
|
write_snapshot "$SNAPSHOT_FILE" "$CURRENT_UPSTREAMS" "$CURRENT_METHODS"
|
|
echo "reload_dshackle: $NAME restarted, config applied on clean start" >&2
|
|
# On restart, the config is freshly loaded, so we need to drop if removals were detected.
|
|
if [ "$NEED_DROP" -eq 1 ]; then
|
|
PID=$(docker inspect -f '{{.State.Pid}}' "$CID" 2>/dev/null) && \
|
|
nsenter -t "$PID" -n ss -K state established '( sport = :2449 )' >/dev/null 2>&1
|
|
echo "reload_dshackle: $NAME restart applied, gateway connections dropped (removals detected)" >&2
|
|
fi
|
|
else
|
|
echo "reload_dshackle: ERROR: $NAME not running after restart — manual intervention" >&2
|
|
RC=1
|
|
fi
|
|
done
|
|
|
|
exit $RC
|