Claude Agent
81c9addac9
Extend reload_dshackle.sh to drop established gateway gRPC connections on port 2449 after a successful SIGHUP config reload, forcing the dRPC edge to reconnect within ~1s and re-read the advertised chain/method list. Gateways only re-initialize advertisements on reconnection (verified live: 0.03-0.06s serving gap, 33 re-probes in 4s). This allows deploy-time upstream additions (new chains like mova, new backends like us-50) to be seen by dRPC immediately rather than waiting for natural reconnection. The TCP-level drop is best-effort: failure does NOT fail the script. Operator ruling: this stays a TCP-level drop, NOT a dshackle code change. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
82 lines
3.9 KiB
Bash
Executable File
82 lines
3.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Re-weave host-local upstream overrides (method disables etc.) into the freshly
|
|
# pushed configs BEFORE signaling dshackle - makes /root/rpc-local/dshackle-overrides.yaml
|
|
# survive every configure-drpc regeneration. See apply-dshackle-overrides.py.
|
|
[ -f /root/rpc/apply-dshackle-overrides.py ] && python3 /root/rpc/apply-dshackle-overrides.py
|
|
|
|
# 1:1 invariant (2026-07-15): refuse to ACTIVATE a dshackle config routing >1 node for the same
|
|
# chain — we can't attribute traffic to multiple nodes behind one proxy (no per-upstream request
|
|
# metric; conn-seconds biases it), and the attribution model + planner assume 1:1. The validator
|
|
# reads /root/rpc/main_configs/*.yaml; on violation it exits 1 and we keep the live config.
|
|
# See /root/proxy-1to1-invariant-plan.md + rpc/validate-dshackle-1to1.py.
|
|
if [ -f /root/rpc/validate-dshackle-1to1.py ]; then
|
|
python3 /root/rpc/validate-dshackle-1to1.py /root/rpc/main_configs || {
|
|
echo "reload_dshackle.sh: REFUSING reload — 1:1 invariant violated (above); keeping live config" >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
|
|
# Signal dshackle to reload, then VERIFY the reload applied. dshackle's SIGHUP reload is
|
|
# unreliable for upstream REMOVALS (and method-set changes, same remove+add path): it can
|
|
# throw internally, log "Config is not reloaded, cause - ...", and SILENTLY keep serving the
|
|
# old config (in-memory config updated, runtime selectors stale = drift). Note
|
|
# "Reloading config has been completed" is NOT success — it's a finally-block and prints even
|
|
# when a processor threw. There is no runtime admin API to mutate upstreams, so a container
|
|
# restart is the only way to actually apply such changes.
|
|
# Per container: SIGHUP, check the post-SIGHUP logs for a failure/drop marker, restart on
|
|
# failure. Exit 0 when the config ends up applied (via SIGHUP or restart); non-zero only if a
|
|
# needed restart failed or the container didn't come back (so the deploy fails loudly instead
|
|
# of silently drifting).
|
|
set -u
|
|
FAIL_MARK='Config is not reloaded, cause -' # a reload processor threw (removal/method bug)
|
|
DROP_MARK='Reloading is in progress' # a concurrent HUP was dropped (reload skipped)
|
|
SETTLE=3 # seconds for the SIGHUP handler to run + log
|
|
RC=0
|
|
|
|
for CID in $(docker ps -q -f "name=dshackle"); do
|
|
NAME=$(docker inspect -f '{{.Name}}' "$CID" 2>/dev/null | sed 's|^/||')
|
|
[ -n "$NAME" ] || NAME="$CID"
|
|
T0=$(date +%s)
|
|
if ! docker kill --signal=HUP "$CID" >/dev/null 2>&1; then
|
|
echo "reload_dshackle: WARNING: failed to SIGHUP $NAME" >&2
|
|
RC=1
|
|
continue
|
|
fi
|
|
sleep "$SETTLE"
|
|
LOGS=$(docker logs --since "$T0" "$CID" 2>&1)
|
|
REASON=""
|
|
if echo "$LOGS" | grep -qF "$FAIL_MARK"; then
|
|
REASON=$(echo "$LOGS" | grep -F "$FAIL_MARK" | head -1)
|
|
elif echo "$LOGS" | grep -qF "$DROP_MARK"; then
|
|
REASON="concurrent HUP dropped (reload skipped)"
|
|
fi
|
|
[ -z "$REASON" ] && {
|
|
# Drop established gateway gRPC connections to force dRPC edge to reconnect
|
|
# and re-read the advertised chain/method list. Gateways only re-initialize
|
|
# advertisements on reconnection. Best-effort: failure does NOT fail the script.
|
|
PID=$(docker inspect -f '{{.State.Pid}}' "$CID" 2>/dev/null) && \
|
|
nsenter -t "$PID" -n ss -K state established '( sport = :2449 )' >/dev/null 2>&1
|
|
continue
|
|
}
|
|
echo "reload_dshackle: $NAME reload did NOT apply ($REASON) — restarting to load config fresh" >&2
|
|
if ! docker restart "$CID" >/dev/null 2>&1; then
|
|
echo "reload_dshackle: ERROR: restart failed for $NAME — config NOT applied, manual intervention" >&2
|
|
RC=1
|
|
continue
|
|
fi
|
|
UP=""
|
|
for _ in $(seq 1 30); do
|
|
if docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null | grep -q true; then UP=1; break; fi
|
|
sleep 2
|
|
done
|
|
if [ -n "$UP" ]; then
|
|
echo "reload_dshackle: $NAME restarted, config applied on clean start" >&2
|
|
else
|
|
echo "reload_dshackle: ERROR: $NAME not running after restart — manual intervention" >&2
|
|
RC=1
|
|
fi
|
|
done
|
|
|
|
exit $RC
|