config merge

docs/04-upstream-config.adoc

@@ -32,8 +32,6 @@ If you request balance for an address that is not indexed then it returns 0 bala
 - To track all transactions you need to setup index for transactions, which is disabled by default.
 Run it with `-reindex` option, or set `txindex=1` in the config.
 
-===
-
 === Example Configuration
 
 .upstreams.yaml
@@ -317,3 +315,33 @@ For JSON RPC and Websockets a Basic Authentication can be used:
 
 - `username` - username
 - `password` - password
+
+=== Chains specific configuration
+We can use chain settings to specify chain specific behavior, for example rules for dshackle to work with upstream statuses
+
+.chains.yaml
+[source,yaml]
+----
+chain-settings:
+  default:
+    lags:
+      syncing: 6
+      lagging: 1
+  chains:
+    - id: eth
+      lags:
+        syncing: 6
+        lagging: 1
+    - id: polygon
+      lags:
+        syncing: 20
+        lagging: 10
+----
+Options
+[cols="2,5a"]
+|===
+| Option | Description
+
+| `lags.syncing` | the size of the lag after which the upstream is determined to be syncing
+| `lags.lagging` | the size of the lag after which the upstream is determined to be lagging
+|===

src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt

@@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.config
 import io.emeraldpay.dshackle.Chain
 import java.lang.IllegalStateException
 
-class ChainsConfig(private val chains: Map<Chain, RawChainConfig>?, val currentDefault: RawChainConfig?) {
+data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val currentDefault: RawChainConfig?) {
     companion object {
         @JvmStatic
         fun default(): ChainsConfig = ChainsConfig(emptyMap(), RawChainConfig.default())
@@ -25,7 +25,7 @@ class ChainsConfig(private val chains: Map<Chain, RawChainConfig>?, val currentD
 
     fun resolve(chain: Chain): ChainConfig {
         val default = currentDefault ?: panic()
-        val raw = chains?.get(chain) ?: default
+        val raw = chains[chain] ?: default
 
         return ChainConfig(
             laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(),
@@ -33,5 +33,31 @@ class ChainsConfig(private val chains: Map<Chain, RawChainConfig>?, val currentD
         )
     }
 
+    fun patch(patch: ChainsConfig) = ChainsConfig(
+        merge(this.chains, patch.chains),
+        merge(this.currentDefault!!, patch.currentDefault)
+    )
+
+    private fun merge(
+        current: RawChainConfig,
+        patch: RawChainConfig?
+    ) = RawChainConfig(
+        syncingLagSize = patch?.syncingLagSize ?: current.syncingLagSize,
+        laggingLagSize = patch?.laggingLagSize ?: current.laggingLagSize
+    )
+
+    private fun merge(
+        current: Map<Chain, RawChainConfig>,
+        patch: Map<Chain, RawChainConfig>
+    ): Map<Chain, RawChainConfig> {
+        val currentMut = current.toMutableMap()
+
+        for (k in patch) {
+            currentMut.merge(k.key, k.value) { v1, v2 -> merge(v1, v2) }
+        }
+
+        return currentMut.toMap()
+    }
+
     fun panic(): Nothing = throw IllegalStateException("Chains settings state is illegal - default config is null")
 }

src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt

@@ -3,10 +3,24 @@ package io.emeraldpay.dshackle.config
 import io.emeraldpay.dshackle.Global
 import org.yaml.snakeyaml.nodes.CollectionNode
 import org.yaml.snakeyaml.nodes.MappingNode
+import java.io.InputStream
 
 class ChainsConfigReader : YamlConfigReader<ChainsConfig>() {
 
+    private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!!
+
     override fun read(input: MappingNode?): ChainsConfig {
+        val default = readInternal(defaultConfig)
+        val current = readInternal(input)
+        return default.patch(current)
+    }
+
+    fun readInternal(input: InputStream): ChainsConfig {
+        val configNode = readNode(input)
+        return readInternal(configNode)
+    }
+
+    fun readInternal(input: MappingNode?): ChainsConfig {
         return getMapping(input, "chain-settings")?.let {
 
             val chains = getList<MappingNode>(it, "chains")?.let {
@@ -37,8 +51,8 @@ class ChainsConfigReader : YamlConfigReader<ChainsConfig>() {
 
     private fun readChains(node: CollectionNode<MappingNode>): List<Pair<String, ChainsConfig.RawChainConfig>> {
         return node.value.mapNotNull {
-            val key = getValueAsString(it, "name")
+            val key = getValueAsString(it, "id")
-                ?: throw InvalidConfigYamlException(filename, it.startMark, "chain name required")
+                ?: throw InvalidConfigYamlException(filename, it.startMark, "chain id required")
             val value = readChain(it)
             if (value != null) {
                 return@mapNotNull key to value

src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt

@@ -16,17 +16,12 @@
 package io.emeraldpay.dshackle.config
 
 import io.emeraldpay.dshackle.FileResolver
-import org.slf4j.LoggerFactory
 import org.yaml.snakeyaml.nodes.MappingNode
 
 class MainConfigReader(
     fileResolver: FileResolver
 ) : YamlConfigReader<MainConfig>() {
 
-    companion object {
-        private val log = LoggerFactory.getLogger(MainConfigReader::class.java)
-    }
-
     private val authConfigReader = AuthConfigReader()
     private val proxyConfigReader = ProxyConfigReader()
     private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver)
@@ -39,7 +34,7 @@ class MainConfigReader(
     private val compressionConfigReader = CompressionConfigReader()
     private val chainsConfigReader = ChainsConfigReader()
 
-    override fun read(input: MappingNode?): MainConfig? {
+    override fun read(input: MappingNode?): MainConfig {
         val config = MainConfig()
         getValueAsString(input, "host")?.let {
             config.host = it

src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt

@@ -20,6 +20,7 @@ import io.emeraldpay.dshackle.FileResolver
 import org.apache.commons.lang3.StringUtils
 import org.slf4j.LoggerFactory
 import org.yaml.snakeyaml.nodes.MappingNode
+import java.io.InputStream
 import java.net.URI
 import java.time.Duration
 import java.util.Locale
@@ -38,6 +39,10 @@ class UpstreamsConfigReader(
         }
     }
 
+    fun readInternal(input: InputStream): UpstreamsConfig? {
+        val configNode = readNode(input)
+        return readInternal(configNode)
+    }
     fun readInternal(input: MappingNode?): UpstreamsConfig {
         val config = UpstreamsConfig()
 
@@ -55,7 +60,7 @@ class UpstreamsConfigReader(
         getValueAsString(input, "include")?.let { path ->
             fileResolver.resolve(path).let { file ->
                 if (file.exists() && file.isFile && file.canRead()) {
-                    read(file.inputStream())?.let {
+                    readInternal(file.inputStream())?.let {
                         it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
                     }
                 } else {
@@ -67,7 +72,7 @@ class UpstreamsConfigReader(
         getListOfString(input, "include")?.forEach { path ->
             fileResolver.resolve(path).let { file ->
                 if (file.exists() && file.isFile && file.canRead()) {
-                    read(file.inputStream())?.let {
+                    readInternal(file.inputStream())?.let {
                         it.upstreams.forEach { upstream -> config.upstreams.add(upstream) }
                     }
                 } else {

src/main/resources/chains.yaml

@@ -10,7 +10,7 @@ chain-settings:
       lags:
         syncing: 6
         lagging: 1
-    - name: polygon
+    - id: polygon
       lags:
         syncing: 20
         lagging: 10

src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy

@@ -28,13 +28,24 @@ class ChainsConfigReaderSpec extends Specification {
         def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml")
         when:
         def config = reader.read(stream)
-        def act = config.resolve(Chain.ETHEREUM)
+        def eth = config.resolve(Chain.ETHEREUM)
-        def act2 = config.resolve(Chain.POLYGON)
+        def pol = config.resolve(Chain.POLYGON)
+        def opt = config.resolve(Chain.OPTIMISM)
+        def sep = config.resolve(Chain.TESTNET_SEPOLIA)
         then:
-        act.laggingLagSize == 5
+        eth.laggingLagSize == 1
-        act.syncingLagSize == 10
+        eth.syncingLagSize == 6
 
-        act2.laggingLagSize == 1
+        pol.laggingLagSize == 10
-        act2.syncingLagSize == 6
+        pol.syncingLagSize == 20
+
+        opt.laggingLagSize == 3
+        opt.syncingLagSize == 6
+
+        sep.laggingLagSize == 1
+        sep.syncingLagSize == 10
+
+        eth.laggingLagSize == 1
+        eth.syncingLagSize == 6
     }
 }

src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy

@@ -27,7 +27,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         with(act.defaultOptions) {
@@ -75,7 +75,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-only.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 1
@@ -100,7 +100,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-full.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 1
@@ -127,7 +127,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         with(act.defaultOptions) {
@@ -155,7 +155,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("upstreams-ethereum-pos.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 1
@@ -175,7 +175,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin-esplora.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         with(act.defaultOptions) {
@@ -204,7 +204,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ds.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 1
@@ -227,7 +227,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-labels.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 2
@@ -248,7 +248,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-options.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 2
@@ -264,7 +264,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-defaults.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         with(act.defaultOptions) {
@@ -287,7 +287,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         with(act.upstreams.get(0)) {
@@ -307,7 +307,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods-quorum.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         with(act.upstreams.get(0)) {
@@ -330,7 +330,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-id.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 1
@@ -357,7 +357,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 2
@@ -369,7 +369,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 2
@@ -381,7 +381,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-2.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 3
@@ -394,7 +394,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-invalid.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 2
@@ -406,7 +406,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("upstreams-node-id.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 2
@@ -424,7 +424,7 @@ class UpstreamsConfigReaderSpec extends Specification {
         setup:
         def config = this.class.getClassLoader().getResourceAsStream("upstreams-method-groups.yaml")
         when:
-        def act = reader.read(config)
+        def act = reader.readInternal(config)
         then:
         act != null
         act.upstreams.size() == 1

src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt

@@ -0,0 +1,44 @@
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.dshackle.Chain
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+
+internal class ChainsConfigTest {
+
+    @Test
+    fun patch() {
+        val orig = ChainsConfig(
+            mapOf(
+                Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 0),
+                Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2),
+                Chain.POLYGON to ChainsConfig.RawChainConfig(3, 4)
+            ),
+            ChainsConfig.RawChainConfig(1, 2)
+        )
+
+        val patch = ChainsConfig(
+            mapOf(
+                Chain.BITCOIN to ChainsConfig.RawChainConfig(null, 10000),
+                Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11),
+                Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999)
+            ),
+            ChainsConfig.RawChainConfig(100, null)
+        )
+
+        val res = orig.patch(patch)
+
+        assertEquals(
+            ChainsConfig(
+                mapOf(
+                    Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 10000),
+                    Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2),
+                    Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11),
+                    Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999)
+                ),
+                ChainsConfig.RawChainConfig(100, 2)
+            ),
+            res
+        )
+    }
+}

src/test/resources/configs/chains-basic.yaml

@@ -10,3 +10,9 @@ chain-settings:
       lags:
         syncing: 6
         lagging: 1
+    - id: optimism
+      lags:
+        lagging: 3
+    - id: sepolia
+      lags:
+        syncing: 10