cometbft: WS Upgrade matcher case-insensitive + ct_ensure_wasm + statesync-skip-if-data

Per cursor cosmos handoff. (1) Traefik WS router rule Headers(Upgrade,websocket) is
case-SENSITIVE -> clients sending 'Upgrade: WebSocket' (python websocket-client) fall
through to the RPC router (200/400 not 101). rpc-client.yml now emits
HeadersRegexp(Upgrade,(?i)websocket) for all split-WS chains; regenerated cosmos+avalanche.
(2) Refactor into cometbft-common.sh: ct_configure_statesync skips if data/application.db
exists; new ct_ensure_wasm seeds CosmWasm/IBC-08 wasm (statesync omits them). init.sh calls
both. README documents wasm/version/WS gotchas. 45 other split-WS composes get HeadersRegexp
on their next regen.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

avalanche/go/avalanche-fuji-go-archive-leveldb.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-fuji-go-archive-leveldb.service=avalanche-fuji-go-archive-leveldb
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji-archive`) || Path(`/avalanche-fuji-archive/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji-archive`) || Path(`/avalanche-fuji-archive/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.rule=(Path(`/avalanche-fuji-archive`) || Path(`/avalanche-fuji-archive/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.rule=(Path(`/avalanche-fuji-archive`) || Path(`/avalanche-fuji-archive/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-fuji-go-archive-leveldb-ws.middlewares=avalanche-fuji-go-archive-leveldb-stripprefix, avalanche-fuji-go-archive-leveldb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-fuji-go-pruned-leveldb.prune.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-fuji-go-pruned-leveldb.service=avalanche-fuji-go-pruned-leveldb
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.middlewares=avalanche-fuji-go-pruned-leveldb-stripprefix, avalanche-fuji-go-pruned-leveldb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-fuji-go-pruned-leveldb.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-fuji-go-pruned-leveldb.service=avalanche-fuji-go-pruned-leveldb
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-fuji-go-pruned-leveldb-ws.middlewares=avalanche-fuji-go-pruned-leveldb-stripprefix, avalanche-fuji-go-pruned-leveldb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-fuji-go-pruned-pebbledb.prune.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-fuji-go-pruned-pebbledb.service=avalanche-fuji-go-pruned-pebbledb
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.middlewares=avalanche-fuji-go-pruned-pebbledb-stripprefix, avalanche-fuji-go-pruned-pebbledb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-fuji-go-pruned-pebbledb.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-fuji-go-pruned-pebbledb.service=avalanche-fuji-go-pruned-pebbledb
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-fuji`) || Path(`/avalanche-fuji/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-fuji-go-pruned-pebbledb-ws.middlewares=avalanche-fuji-go-pruned-pebbledb-stripprefix, avalanche-fuji-go-pruned-pebbledb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-mainnet-go-archive-leveldb.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-mainnet-go-archive-leveldb.service=avalanche-mainnet-go-archive-leveldb
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet-archive`) || Path(`/avalanche-mainnet-archive/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet-archive`) || Path(`/avalanche-mainnet-archive/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.rule=(Path(`/avalanche-mainnet-archive`) || Path(`/avalanche-mainnet-archive/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.rule=(Path(`/avalanche-mainnet-archive`) || Path(`/avalanche-mainnet-archive/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-mainnet-go-archive-leveldb-ws.middlewares=avalanche-mainnet-go-archive-leveldb-stripprefix, avalanche-mainnet-go-archive-leveldb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-mainnet-go-pruned-leveldb.prune.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-mainnet-go-pruned-leveldb.service=avalanche-mainnet-go-pruned-leveldb
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.middlewares=avalanche-mainnet-go-pruned-leveldb-stripprefix, avalanche-mainnet-go-pruned-leveldb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-mainnet-go-pruned-leveldb.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-mainnet-go-pruned-leveldb.service=avalanche-mainnet-go-pruned-leveldb
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-mainnet-go-pruned-leveldb-ws.middlewares=avalanche-mainnet-go-pruned-leveldb-stripprefix, avalanche-mainnet-go-pruned-leveldb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-mainnet-go-pruned-pebbledb.prune.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb.service=avalanche-mainnet-go-pruned-pebbledb
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.middlewares=avalanche-mainnet-go-pruned-pebbledb-stripprefix, avalanche-mainnet-go-pruned-pebbledb-set-ws-path, ipallowlist
 
 volumes:

avalanche/go/avalanche-mainnet-go-pruned-pebbledb.yml

@@ -87,8 +87,8 @@ services:
       - traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb.service=avalanche-mainnet-go-pruned-pebbledb
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=Host(`$DOMAIN`) && (Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.rule=(Path(`/avalanche-mainnet`) || Path(`/avalanche-mainnet/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.avalanche-mainnet-go-pruned-pebbledb-ws.middlewares=avalanche-mainnet-go-pruned-pebbledb-stripprefix, avalanche-mainnet-go-pruned-pebbledb-set-ws-path, ipallowlist
 
 volumes:

cosmos/gaiad/cosmos-mainnet-gaiad-pruned.yml

@@ -96,8 +96,8 @@ services:
       - traefik.http.routers.cosmos-mainnet-gaiad-pruned.service=cosmos-mainnet-gaiad-pruned
       - ${NO_SSL:-traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.entrypoints=websecure}
       - ${NO_SSL:-traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.tls.certresolver=myresolver}
-      - ${NO_SSL:-traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.rule=Host(`$DOMAIN`) && (Path(`/cosmos-mainnet-pruned`) || Path(`/cosmos-mainnet-pruned/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:-traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.rule=Host(`$DOMAIN`) && (Path(`/cosmos-mainnet-pruned`) || Path(`/cosmos-mainnet-pruned/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
-      - ${NO_SSL:+traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.rule=(Path(`/cosmos-mainnet-pruned`) || Path(`/cosmos-mainnet-pruned/`)) && Headers(`Upgrade`, `websocket`)}
+      - ${NO_SSL:+traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.rule=(Path(`/cosmos-mainnet-pruned`) || Path(`/cosmos-mainnet-pruned/`)) && HeadersRegexp(`Upgrade`, `(?i)websocket`)}
       - traefik.http.routers.cosmos-mainnet-gaiad-pruned-ws.middlewares=cosmos-mainnet-gaiad-pruned-stripprefix, cosmos-mainnet-gaiad-pruned-set-ws-path, ipallowlist
 
 volumes:

cosmos/scripts/cometbft-common.sh

@@ -167,6 +167,15 @@ ct_set_min_gas_prices() {
 ct_configure_statesync() {
   _cfg="$1"; _rpc="$2"; _offset="${3:-2000}"
   [ -f "$_cfg" ] || return 0
+  # NEVER re-arm statesync on a node that already has application state (a restored
+  # snapshot or a prior sync). Re-statesyncing over it leaves a broken/partial datadir and,
+  # for wasm chains, drops the wasm files -> startup panic. _cfg is $HOME/config/config.toml,
+  # so application state lives at $HOME/data/application.db.
+  _home=$(dirname "$(dirname "$_cfg")")
+  if [ -e "$_home/data/application.db" ]; then
+    ct_log "statesync: existing data dir, skipping"
+    return 0
+  fi
   [ -n "$_rpc" ] || { ct_log "statesync: no RPC servers given, skipping"; return 0; }
   _primary=$(echo "$_rpc" | cut -d, -f1)
   _latest=$(curl -s "$_primary/block" | jq -r '.result.block.header.height // .block.header.height' 2>/dev/null || true)
@@ -189,3 +198,26 @@ ct_configure_statesync() {
   }" "$_cfg"
   return 0
 }
+
+# ct_ensure_wasm HOME_DIR WASM_SNAPSHOT_URL
+#   CosmWasm + IBC 08-wasm bytecode are FILES on disk that state-sync does NOT restore, so
+#   a state-synced wasm chain panics at startup ("wasmlckeeper failed initialize pinned codes
+#   / Error opening Wasm file"). Seed them from a wasm-only snapshot (e.g. polkachu
+#   cosmos_wasmonly.tar.lz4) when the wasm dir is missing/empty. No-op if URL unset or wasm
+#   already present. Best-effort (logs on failure); the fully robust path for wasm chains is a
+#   FULL snapshot restore. Requires lz4 + tar (installed here).
+ct_ensure_wasm() {
+  _home="$1"; _url="$2"
+  [ -n "$_url" ] || return 0
+  if [ -d "$_home/wasm" ] && [ -n "$(ls -A "$_home/wasm" 2>/dev/null)" ]; then
+    return 0   # wasm already present
+  fi
+  ct_log "wasm: empty, fetching snapshot $_url"
+  ct_apk lz4 tar
+  if curl -sL "$_url" | lz4 -dc | tar -xf - -C "$_home"; then
+    ct_log "wasm: extracted into $_home"
+  else
+    ct_log "WARN wasm: fetch/extract failed ($_url)"
+  fi
+  return 0
+}

cosmos/scripts/init.sh

@@ -37,29 +37,8 @@ ct_patch_p2p             "$CONFIG_DIR/config.toml" "$IP" "${P2P_PORT:-26656}"
 ct_merge_seeds           "$CONFIG_DIR/config.toml" "$SEEDS"
 ct_set_persistent_peers  "$CONFIG_DIR/config.toml" "$PERSISTENT_PEERS"
 ct_set_moniker           "$CONFIG_DIR/config.toml" "$MONIKER"
-
+ct_configure_statesync   "$CONFIG_DIR/config.toml" "$STATESYNC_RPC"   # internally skips if data/application.db exists
-# Statesync ONLY on a fresh node. If application state already exists (a restored snapshot
+ct_ensure_wasm           "$HOME_DIR" "$WASM_SNAPSHOT_URL"             # seeds CosmWasm/IBC-08 wasm if missing (statesync omits them)
-# or prior sync), do NOT re-arm statesync — it would try to re-restore and, because statesync
-# omits the CosmWasm + IBC 08-wasm blobs, the node panics at startup
-# ("wasmlckeeper failed initialize pinned codes / Error opening Wasm file").
-if [ -e "$HOME_DIR/data/application.db" ]; then
-  ct_log "statesync: existing data dir, skipping"
-else
-  ct_configure_statesync "$CONFIG_DIR/config.toml" "$STATESYNC_RPC"
-  # statesync restores IAVL state but NOT the wasm files gaia needs at startup. Seed them
-  # from the wasm-only snapshot (best-effort; the fully robust alternative for wasm chains is
-  # a full snapshot restore). Extracts wasm/ (CosmWasm) + the IBC 08-light-client wasm.
-  if [ -n "$WASM_SNAPSHOT_URL" ] && { [ ! -d "$HOME_DIR/wasm" ] || [ -z "$(ls -A "$HOME_DIR/wasm" 2>/dev/null)" ]; }; then
-    ct_log "fetching wasm snapshot $WASM_SNAPSHOT_URL"
-    ct_apk lz4 tar
-    if curl -sL "$WASM_SNAPSHOT_URL" | lz4 -dc | tar -xf - -C "$HOME_DIR"; then
-      ct_log "wasm snapshot extracted into $HOME_DIR"
-    else
-      ct_log "WARN wasm snapshot fetch/extract failed"
-    fi
-  fi
-fi
-
 ct_seed_priv_validator_state "$HOME_DIR"
 
 exec gaiad start --home "$HOME_DIR" --minimum-gas-prices "$MIN_GAS" "$@"

scripts/README.cometbft.md

@@ -95,3 +95,22 @@ cosmos:
   are idempotent and don't clobber a synced datadir.
 - Migrating an existing problem chain (haqq/sei/zero-gravity/berachain) onto this base is a
   deliberate, separate step — diff the rendered compose + re-test live before trusting it.
+
+## CosmWasm / version gotchas (learned on cosmos-hub, 2026-06-16)
+- **Match the binary version to the live chain.** Cosmos chains hard-fork via governance; an old
+  binary panics `upgrade handler is missing for vX.Y.Z` (or halts at the upgrade height). Check the
+  live version: `curl <rpc>/abci_info | jq .result.response.version`. NOTE ghcr `tags/list` can LAG /
+  omit the newest tag — verify pullability with a direct manifest HEAD, not the tag list.
+- **state-sync does NOT restore CosmWasm + IBC 08-wasm files.** A wasm chain then panics at startup
+  (`wasmlckeeper failed initialize pinned codes / Error opening Wasm file`). Two fixes, both wired into
+  the lib: `ct_configure_statesync` skips if `data/application.db` already exists (idempotent restart),
+  and `ct_ensure_wasm <home> <url>` seeds a wasm-only snapshot (polkachu `*_wasmonly.tar.lz4`) when the
+  wasm dir is empty. Set `wasm_snapshot_url` in context for any CosmWasm chain.
+- **Fresh bootstrap of a wasm chain: prefer the FULL polkachu snapshot** over state-sync —
+  `https://snapshots.polkachu.com/snapshots/<chain>/<chain>_<HEIGHT>.tar.lz4` includes everything
+  (state + `wasm/` + `data/08-light-client/`). state-sync + `ct_ensure_wasm` is the lighter path but
+  the wasm extract paths/timing are best-effort.
+- **Traefik WS rule must be case-insensitive**: use `HeadersRegexp('Upgrade','(?i)websocket')`, NOT
+  `Headers('Upgrade','websocket')` — the latter is case-sensitive and clients sending `Upgrade: WebSocket`
+  (e.g. python websocket-client) fall through to the RPC router (200/400 instead of a 101 upgrade).
+  Driven by `client_ws_path`; the shared rpc-client.yml emits the regexp form for all split-WS chains.

scripts/cometbft-common.sh

@@ -167,6 +167,15 @@ ct_set_min_gas_prices() {
 ct_configure_statesync() {
   _cfg="$1"; _rpc="$2"; _offset="${3:-2000}"
   [ -f "$_cfg" ] || return 0
+  # NEVER re-arm statesync on a node that already has application state (a restored
+  # snapshot or a prior sync). Re-statesyncing over it leaves a broken/partial datadir and,
+  # for wasm chains, drops the wasm files -> startup panic. _cfg is $HOME/config/config.toml,
+  # so application state lives at $HOME/data/application.db.
+  _home=$(dirname "$(dirname "$_cfg")")
+  if [ -e "$_home/data/application.db" ]; then
+    ct_log "statesync: existing data dir, skipping"
+    return 0
+  fi
   [ -n "$_rpc" ] || { ct_log "statesync: no RPC servers given, skipping"; return 0; }
   _primary=$(echo "$_rpc" | cut -d, -f1)
   _latest=$(curl -s "$_primary/block" | jq -r '.result.block.header.height // .block.header.height' 2>/dev/null || true)
@@ -189,3 +198,26 @@ ct_configure_statesync() {
   }" "$_cfg"
   return 0
 }
+
+# ct_ensure_wasm HOME_DIR WASM_SNAPSHOT_URL
+#   CosmWasm + IBC 08-wasm bytecode are FILES on disk that state-sync does NOT restore, so
+#   a state-synced wasm chain panics at startup ("wasmlckeeper failed initialize pinned codes
+#   / Error opening Wasm file"). Seed them from a wasm-only snapshot (e.g. polkachu
+#   cosmos_wasmonly.tar.lz4) when the wasm dir is missing/empty. No-op if URL unset or wasm
+#   already present. Best-effort (logs on failure); the fully robust path for wasm chains is a
+#   FULL snapshot restore. Requires lz4 + tar (installed here).
+ct_ensure_wasm() {
+  _home="$1"; _url="$2"
+  [ -n "$_url" ] || return 0
+  if [ -d "$_home/wasm" ] && [ -n "$(ls -A "$_home/wasm" 2>/dev/null)" ]; then
+    return 0   # wasm already present
+  fi
+  ct_log "wasm: empty, fetching snapshot $_url"
+  ct_apk lz4 tar
+  if curl -sL "$_url" | lz4 -dc | tar -xf - -C "$_home"; then
+    ct_log "wasm: extracted into $_home"
+  else
+    ct_log "WARN wasm: fetch/extract failed ($_url)"
+  fi
+  return 0
+}