ethereum-rpc-docker/apply-dshackle-overrides.py

#!/usr/bin/env python3
"""apply-dshackle-overrides.py - weave HOST-LOCAL dshackle upstream overrides into
the generated main_configs/*.yaml so they survive every configure-drpc regeneration.

Override file (NOT in the repo - host-local, like rpc-local/compose.d):
    /root/rpc-local/dshackle-overrides.yaml

Schema (match by upstream id OR by chain; id wins):
    upstreams:
      uk-4-base-mainnet-op-reth-pruned:        # or e.g. "chain:base"
        method_groups_disabled: [trace, debug] # removed from enabled, added to disabled
        methods_disabled: [eth_getFilterLogs]  # per-method disables (methods.disabled)
        options:
          disable-validation: true             # merged into upstream options

Rationale: heavy methods at flat per-call pay poison the chain's latency scoring
(operator method-eviction rule 2026-07-10; see method-economics.py). The disable
decision is local to the gateway's market, so the config lives on the gateway.

Called by reload_dshackle.sh before signaling dshackle - i.e. re-applied on every
ansible config push. NOTE: dshackle only re-reads upstream method lists on RESTART;
this keeps the on-disk config permanently correct, the next restart activates it.
Idempotent. Silent no-op when the override file is absent.
"""
import glob
import os
import sys

import yaml

OVERRIDES = '/root/rpc-local/dshackle-overrides.yaml'
CONFIG_DIR = '/root/rpc/main_configs'


def apply(upstream, ov):
    changed = False
    mg = upstream.setdefault('method-groups', {}) or {}
    for grp in ov.get('method_groups_disabled') or []:
        en = mg.get('enabled') or []
        if grp in en:
            en.remove(grp)
            mg['enabled'] = en
            changed = True
        dis = mg.get('disabled') or []
        if grp not in dis:
            dis.append(grp)
            mg['disabled'] = dis
            changed = True
    if mg:
        upstream['method-groups'] = mg
    for meth in ov.get('methods_disabled') or []:
        methods = upstream.setdefault('methods', {}) or {}
        dis = methods.get('disabled') or []
        if not any((d.get('name') if isinstance(d, dict) else d) == meth for d in dis):
            dis.append({'name': meth})
            methods['disabled'] = dis
            upstream['methods'] = methods
            changed = True
    for k, v in (ov.get('options') or {}).items():
        opts = upstream.setdefault('options', {}) or {}
        if opts.get(k) != v:
            opts[k] = v
            upstream['options'] = opts
            changed = True
    return changed


ENVFILE = '/root/rpc/.env'


def env_overrides():
    """Chain-level overrides from .env (rendered from host_vars env_overrides):
        DSHACKLE_<CHAIN>_METHOD_GROUPS_DISABLED=trace,debug
        DSHACKLE_<CHAIN>_METHODS_DISABLED=eth_getFilterLogs
    <CHAIN> = dshackle chain name, upper, '-' -> '_' (e.g. BASE, ETH_BEACON_CHAIN)."""
    spec = {}
    if not os.path.exists(ENVFILE):
        return spec
    import re as _re
    for line in open(ENVFILE):
        m = _re.match(r'DSHACKLE_([A-Z0-9_]+)_(METHOD_GROUPS|METHODS)_DISABLED=(.*)', line.strip())
        if not m:
            continue
        chain = m.group(1).lower().replace('_', '-')
        key = 'method_groups_disabled' if m.group(2) == 'METHOD_GROUPS' else 'methods_disabled'
        vals = [v.strip() for v in m.group(3).split(',') if v.strip()]
        if vals:
            spec.setdefault(f'chain:{chain}', {}).setdefault(key, []).extend(vals)
    return spec


def main():
    spec = env_overrides()
    if os.path.exists(OVERRIDES):
        for k, v in ((yaml.safe_load(open(OVERRIDES)) or {}).get('upstreams') or {}).items():
            cur = spec.setdefault(k, {})
            for kk, vv in (v or {}).items():
                if isinstance(vv, list):
                    cur[kk] = sorted(set((cur.get(kk) or []) + vv))
                else:
                    cur.setdefault(kk, vv)
    if not spec:
        return
    total = 0
    for path in glob.glob(os.path.join(CONFIG_DIR, '*.yaml')):
        try:
            doc = yaml.safe_load(open(path))
        except yaml.YAMLError:
            continue
        if not isinstance(doc, dict) or 'upstreams' not in doc:
            continue
        changed = False
        for up in doc['upstreams'] or []:
            if not isinstance(up, dict):
                continue
            ov = spec.get(up.get('id')) or spec.get(f"chain:{up.get('chain')}")
            if ov and apply(up, ov):
                changed = True
        if changed:
            text = yaml.safe_dump(doc, default_flow_style=False, sort_keys=False)
            # dshackle silently refuses to hot-reload configs containing TABS
            # (operator gotcha 2026-07-10) - fail loudly, never write a tabbed file.
            if '\t' in text:
                print(f'dshackle-overrides: REFUSING to write {path} - tab in output')
                continue
            with open(path, 'w') as f:
                f.write(text)
            total += 1
            print(f'dshackle-overrides: rewove {path}')
    # tab audit on ALL configs (even unchanged): a tabbed file silently fails
    # dshackle's hot-reload, so disk and running state diverge invisibly.
    for path in glob.glob(os.path.join(CONFIG_DIR, '*.yaml')):
        try:
            if '\t' in open(path, errors='replace').read():
                print(f'dshackle-overrides: WARNING {path} contains TABS - dshackle will SILENTLY skip reloading it')
        except OSError:
            pass
    if total:
        print(f'dshackle-overrides: {total} file(s) rewoven (dshackle hot-reloads on the HUP that follows)')


if __name__ == '__main__':
    main()