From 2809e87d3314db2e29eb6bff19216c0099b97651 Mon Sep 17 00:00:00 2001 From: a10zn8 Date: Fri, 10 Feb 2023 21:33:58 +0400 Subject: [PATCH] config merge --- docs/04-upstream-config.adoc | 32 +++++++++++++- .../dshackle/config/ChainsConfig.kt | 30 ++++++++++++- .../dshackle/config/ChainsConfigReader.kt | 18 +++++++- .../dshackle/config/MainConfigReader.kt | 7 +-- .../dshackle/config/UpstreamsConfigReader.kt | 9 +++- src/main/resources/chains.yaml | 2 +- .../config/ChainsConfigReaderSpec.groovy | 23 +++++++--- .../config/UpstreamsConfigReaderSpec.groovy | 38 ++++++++-------- .../dshackle/config/ChainsConfigTest.kt | 44 +++++++++++++++++++ src/test/resources/configs/chains-basic.yaml | 6 +++ 10 files changed, 169 insertions(+), 40 deletions(-) create mode 100644 src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt diff --git a/docs/04-upstream-config.adoc b/docs/04-upstream-config.adoc index 22063ecb..03279dc0 100644 --- a/docs/04-upstream-config.adoc +++ b/docs/04-upstream-config.adoc @@ -32,8 +32,6 @@ If you request balance for an address that is not indexed then it returns 0 bala - To track all transactions you need to setup index for transactions, which is disabled by default. Run it with `-reindex` option, or set `txindex=1` in the config. -=== - === Example Configuration .upstreams.yaml @@ -317,3 +315,33 @@ For JSON RPC and Websockets a Basic Authentication can be used: - `username` - username - `password` - password + +=== Chains specific configuration +We can use chain settings to specify chain specific behavior, for example rules for dshackle to work with upstream statuses + +.chains.yaml +[source,yaml] +---- +chain-settings: + default: + lags: + syncing: 6 + lagging: 1 + chains: + - id: eth + lags: + syncing: 6 + lagging: 1 + - id: polygon + lags: + syncing: 20 + lagging: 10 +---- +Options +[cols="2,5a"] +|=== +| Option | Description + +| `lags.syncing` | the size of the lag after which the upstream is determined to be syncing +| `lags.lagging` | the size of the lag after which the upstream is determined to be lagging +|=== diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt index 6565e6d8..1a603d4b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt @@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Chain import java.lang.IllegalStateException -class ChainsConfig(private val chains: Map?, val currentDefault: RawChainConfig?) { +data class ChainsConfig(private val chains: Map, val currentDefault: RawChainConfig?) { companion object { @JvmStatic fun default(): ChainsConfig = ChainsConfig(emptyMap(), RawChainConfig.default()) @@ -25,7 +25,7 @@ class ChainsConfig(private val chains: Map?, val currentD fun resolve(chain: Chain): ChainConfig { val default = currentDefault ?: panic() - val raw = chains?.get(chain) ?: default + val raw = chains[chain] ?: default return ChainConfig( laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(), @@ -33,5 +33,31 @@ class ChainsConfig(private val chains: Map?, val currentD ) } + fun patch(patch: ChainsConfig) = ChainsConfig( + merge(this.chains, patch.chains), + merge(this.currentDefault!!, patch.currentDefault) + ) + + private fun merge( + current: RawChainConfig, + patch: RawChainConfig? + ) = RawChainConfig( + syncingLagSize = patch?.syncingLagSize ?: current.syncingLagSize, + laggingLagSize = patch?.laggingLagSize ?: current.laggingLagSize + ) + + private fun merge( + current: Map, + patch: Map + ): Map { + val currentMut = current.toMutableMap() + + for (k in patch) { + currentMut.merge(k.key, k.value) { v1, v2 -> merge(v1, v2) } + } + + return currentMut.toMap() + } + fun panic(): Nothing = throw IllegalStateException("Chains settings state is illegal - default config is null") } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt index ca41a618..4240fafe 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt @@ -3,10 +3,24 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Global import org.yaml.snakeyaml.nodes.CollectionNode import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream class ChainsConfigReader : YamlConfigReader() { + private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!! + override fun read(input: MappingNode?): ChainsConfig { + val default = readInternal(defaultConfig) + val current = readInternal(input) + return default.patch(current) + } + + fun readInternal(input: InputStream): ChainsConfig { + val configNode = readNode(input) + return readInternal(configNode) + } + + fun readInternal(input: MappingNode?): ChainsConfig { return getMapping(input, "chain-settings")?.let { val chains = getList(it, "chains")?.let { @@ -37,8 +51,8 @@ class ChainsConfigReader : YamlConfigReader() { private fun readChains(node: CollectionNode): List> { return node.value.mapNotNull { - val key = getValueAsString(it, "name") - ?: throw InvalidConfigYamlException(filename, it.startMark, "chain name required") + val key = getValueAsString(it, "id") + ?: throw InvalidConfigYamlException(filename, it.startMark, "chain id required") val value = readChain(it) if (value != null) { return@mapNotNull key to value diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index aa11af2d..6548a0a2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -16,17 +16,12 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.FileResolver -import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode class MainConfigReader( fileResolver: FileResolver ) : YamlConfigReader() { - companion object { - private val log = LoggerFactory.getLogger(MainConfigReader::class.java) - } - private val authConfigReader = AuthConfigReader() private val proxyConfigReader = ProxyConfigReader() private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver) @@ -39,7 +34,7 @@ class MainConfigReader( private val compressionConfigReader = CompressionConfigReader() private val chainsConfigReader = ChainsConfigReader() - override fun read(input: MappingNode?): MainConfig? { + override fun read(input: MappingNode?): MainConfig { val config = MainConfig() getValueAsString(input, "host")?.let { config.host = it diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 9ede1a5b..b55a0c80 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -20,6 +20,7 @@ import io.emeraldpay.dshackle.FileResolver import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream import java.net.URI import java.time.Duration import java.util.Locale @@ -38,6 +39,10 @@ class UpstreamsConfigReader( } } + fun readInternal(input: InputStream): UpstreamsConfig? { + val configNode = readNode(input) + return readInternal(configNode) + } fun readInternal(input: MappingNode?): UpstreamsConfig { val config = UpstreamsConfig() @@ -55,7 +60,7 @@ class UpstreamsConfigReader( getValueAsString(input, "include")?.let { path -> fileResolver.resolve(path).let { file -> if (file.exists() && file.isFile && file.canRead()) { - read(file.inputStream())?.let { + readInternal(file.inputStream())?.let { it.upstreams.forEach { upstream -> config.upstreams.add(upstream) } } } else { @@ -67,7 +72,7 @@ class UpstreamsConfigReader( getListOfString(input, "include")?.forEach { path -> fileResolver.resolve(path).let { file -> if (file.exists() && file.isFile && file.canRead()) { - read(file.inputStream())?.let { + readInternal(file.inputStream())?.let { it.upstreams.forEach { upstream -> config.upstreams.add(upstream) } } } else { diff --git a/src/main/resources/chains.yaml b/src/main/resources/chains.yaml index ad84322a..2c061bd5 100644 --- a/src/main/resources/chains.yaml +++ b/src/main/resources/chains.yaml @@ -10,7 +10,7 @@ chain-settings: lags: syncing: 6 lagging: 1 - - name: polygon + - id: polygon lags: syncing: 20 lagging: 10 diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy index b0ef34eb..6d674be4 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy @@ -28,13 +28,24 @@ class ChainsConfigReaderSpec extends Specification { def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml") when: def config = reader.read(stream) - def act = config.resolve(Chain.ETHEREUM) - def act2 = config.resolve(Chain.POLYGON) + def eth = config.resolve(Chain.ETHEREUM) + def pol = config.resolve(Chain.POLYGON) + def opt = config.resolve(Chain.OPTIMISM) + def sep = config.resolve(Chain.TESTNET_SEPOLIA) then: - act.laggingLagSize == 5 - act.syncingLagSize == 10 + eth.laggingLagSize == 1 + eth.syncingLagSize == 6 - act2.laggingLagSize == 1 - act2.syncingLagSize == 6 + pol.laggingLagSize == 10 + pol.syncingLagSize == 20 + + opt.laggingLagSize == 3 + opt.syncingLagSize == 6 + + sep.laggingLagSize == 1 + sep.syncingLagSize == 10 + + eth.laggingLagSize == 1 + eth.syncingLagSize == 6 } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy index 4d198ec0..5b625f31 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy @@ -27,7 +27,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -75,7 +75,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-only.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -100,7 +100,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-full.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -127,7 +127,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -155,7 +155,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-ethereum-pos.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -175,7 +175,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin-esplora.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -204,7 +204,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ds.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -227,7 +227,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-labels.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -248,7 +248,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-options.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -264,7 +264,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-defaults.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -287,7 +287,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.upstreams.get(0)) { @@ -307,7 +307,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods-quorum.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.upstreams.get(0)) { @@ -330,7 +330,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-id.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -357,7 +357,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -369,7 +369,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -381,7 +381,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-2.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 3 @@ -394,7 +394,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-invalid.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -406,7 +406,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-node-id.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -424,7 +424,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-method-groups.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 diff --git a/src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt new file mode 100644 index 00000000..d068b9eb --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt @@ -0,0 +1,44 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Chain +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +internal class ChainsConfigTest { + + @Test + fun patch() { + val orig = ChainsConfig( + mapOf( + Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 0), + Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2), + Chain.POLYGON to ChainsConfig.RawChainConfig(3, 4) + ), + ChainsConfig.RawChainConfig(1, 2) + ) + + val patch = ChainsConfig( + mapOf( + Chain.BITCOIN to ChainsConfig.RawChainConfig(null, 10000), + Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11), + Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999) + ), + ChainsConfig.RawChainConfig(100, null) + ) + + val res = orig.patch(patch) + + assertEquals( + ChainsConfig( + mapOf( + Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 10000), + Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2), + Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11), + Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999) + ), + ChainsConfig.RawChainConfig(100, 2) + ), + res + ) + } +} diff --git a/src/test/resources/configs/chains-basic.yaml b/src/test/resources/configs/chains-basic.yaml index 606e552e..8f28e16d 100644 --- a/src/test/resources/configs/chains-basic.yaml +++ b/src/test/resources/configs/chains-basic.yaml @@ -10,3 +10,9 @@ chain-settings: lags: syncing: 6 lagging: 1 + - id: optimism + lags: + lagging: 3 + - id: sepolia + lags: + syncing: 10 \ No newline at end of file