rob
d40fbfaac3
dshackle's SIGHUP reload silently fails for upstream REMOVALS (and method-set
changes, same remove+add path): it logs "Config is not reloaded, cause - ...",
updates the in-memory config but leaves runtime selectors stale (drift), and keeps
serving the old config. No runtime admin API exists to mutate upstreams, so a
container restart is the only way to actually apply such changes.
Replace the blind SIGHUP with: per-container SIGHUP, then check post-SIGHUP logs
for the failure marker ("Config is not reloaded, cause -") or a dropped concurrent
HUP ("Reloading is in progress"); on either, docker restart to load config fresh.
"Reloading config has been completed" is NOT success (finally-block) — deliberately
not treated as such. Exit 0 when config ends up applied (via SIGHUP or restart);
non-zero only if a needed restart fails or the container doesn't come back.
Layered on top of the override-weave (apply-dshackle-overrides.py) and the 1:1
invariant gate (validate-dshackle-1to1.py) — both run unchanged before the reload.
Co-Authored-By: Claude <noreply@anthropic.com>
75 lines
3.6 KiB
Bash
Executable File
75 lines
3.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Re-weave host-local upstream overrides (method disables etc.) into the freshly
|
|
# pushed configs BEFORE signaling dshackle - makes /root/rpc-local/dshackle-overrides.yaml
|
|
# survive every configure-drpc regeneration. See apply-dshackle-overrides.py.
|
|
[ -f /root/rpc/apply-dshackle-overrides.py ] && python3 /root/rpc/apply-dshackle-overrides.py
|
|
|
|
# 1:1 invariant (2026-07-15): refuse to ACTIVATE a dshackle config routing >1 node for the same
|
|
# chain — we can't attribute traffic to multiple nodes behind one proxy (no per-upstream request
|
|
# metric; conn-seconds biases it), and the attribution model + planner assume 1:1. The validator
|
|
# reads /root/rpc/main_configs/*.yaml; on violation it exits 1 and we keep the live config.
|
|
# See /root/proxy-1to1-invariant-plan.md + rpc/validate-dshackle-1to1.py.
|
|
if [ -f /root/rpc/validate-dshackle-1to1.py ]; then
|
|
python3 /root/rpc/validate-dshackle-1to1.py /root/rpc/main_configs || {
|
|
echo "reload_dshackle.sh: REFUSING reload — 1:1 invariant violated (above); keeping live config" >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
|
|
# Signal dshackle to reload, then VERIFY the reload applied. dshackle's SIGHUP reload is
|
|
# unreliable for upstream REMOVALS (and method-set changes, same remove+add path): it can
|
|
# throw internally, log "Config is not reloaded, cause - ...", and SILENTLY keep serving the
|
|
# old config (in-memory config updated, runtime selectors stale = drift). Note
|
|
# "Reloading config has been completed" is NOT success — it's a finally-block and prints even
|
|
# when a processor threw. There is no runtime admin API to mutate upstreams, so a container
|
|
# restart is the only way to actually apply such changes.
|
|
# Per container: SIGHUP, check the post-SIGHUP logs for a failure/drop marker, restart on
|
|
# failure. Exit 0 when the config ends up applied (via SIGHUP or restart); non-zero only if a
|
|
# needed restart failed or the container didn't come back (so the deploy fails loudly instead
|
|
# of silently drifting).
|
|
set -u
|
|
FAIL_MARK='Config is not reloaded, cause -' # a reload processor threw (removal/method bug)
|
|
DROP_MARK='Reloading is in progress' # a concurrent HUP was dropped (reload skipped)
|
|
SETTLE=3 # seconds for the SIGHUP handler to run + log
|
|
RC=0
|
|
|
|
for CID in $(docker ps -q -f "name=dshackle"); do
|
|
NAME=$(docker inspect -f '{{.Name}}' "$CID" 2>/dev/null | sed 's|^/||')
|
|
[ -n "$NAME" ] || NAME="$CID"
|
|
T0=$(date +%s)
|
|
if ! docker kill --signal=HUP "$CID" >/dev/null 2>&1; then
|
|
echo "reload_dshackle: WARNING: failed to SIGHUP $NAME" >&2
|
|
RC=1
|
|
continue
|
|
fi
|
|
sleep "$SETTLE"
|
|
LOGS=$(docker logs --since "$T0" "$CID" 2>&1)
|
|
REASON=""
|
|
if echo "$LOGS" | grep -qF "$FAIL_MARK"; then
|
|
REASON=$(echo "$LOGS" | grep -F "$FAIL_MARK" | head -1)
|
|
elif echo "$LOGS" | grep -qF "$DROP_MARK"; then
|
|
REASON="concurrent HUP dropped (reload skipped)"
|
|
fi
|
|
[ -z "$REASON" ] && continue # SIGHUP applied the config — done
|
|
echo "reload_dshackle: $NAME reload did NOT apply ($REASON) — restarting to load config fresh" >&2
|
|
if ! docker restart "$CID" >/dev/null 2>&1; then
|
|
echo "reload_dshackle: ERROR: restart failed for $NAME — config NOT applied, manual intervention" >&2
|
|
RC=1
|
|
continue
|
|
fi
|
|
UP=""
|
|
for _ in $(seq 1 30); do
|
|
if docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null | grep -q true; then UP=1; break; fi
|
|
sleep 2
|
|
done
|
|
if [ -n "$UP" ]; then
|
|
echo "reload_dshackle: $NAME restarted, config applied on clean start" >&2
|
|
else
|
|
echo "reload_dshackle: ERROR: $NAME not running after restart — manual intervention" >&2
|
|
RC=1
|
|
fi
|
|
done
|
|
|
|
exit $RC
|