From ff4c538568a28959c35403684b83c1696ef069fd Mon Sep 17 00:00:00 2001 From: a10zn8 Date: Thu, 9 Feb 2023 20:45:10 +0400 Subject: [PATCH 1/3] added chains configuration --- docs/04-upstream-config.adoc | 2 + .../kotlin/io/emeraldpay/dshackle/Config.kt | 14 +++--- .../dshackle/config/ChainsConfig.kt | 19 ++++++++ .../dshackle/config/ChainsConfigReader.kt | 45 +++++++++++++++++++ .../emeraldpay/dshackle/config/MainConfig.kt | 1 + .../dshackle/config/MainConfigReader.kt | 4 ++ .../dshackle/startup/ConfiguredUpstreams.kt | 42 ++++++++++++----- .../dshackle/upstream/DefaultUpstream.kt | 31 ++++--------- .../upstream/bitcoin/BitcoinRpcUpstream.kt | 6 ++- .../upstream/bitcoin/BitcoinUpstream.kt | 11 +++-- .../upstream/ethereum/EthereumRpcUpstream.kt | 6 ++- .../upstream/ethereum/EthereumUpstream.kt | 6 ++- .../ethereum_pos/EthereumPosRpcUpstream.kt | 8 ++-- .../ethereum_pos/EthereumPosUpstream.kt | 6 ++- .../upstream/grpc/BitcoinGrpcUpstream.kt | 7 ++- .../upstream/grpc/EthereumGrpcUpstream.kt | 7 ++- .../upstream/grpc/EthereumPosGrpcUpstream.kt | 8 +++- .../dshackle/upstream/grpc/GrpcUpstreams.kt | 10 +++-- .../config/ChainsConfigReaderSpec.groovy | 40 +++++++++++++++++ .../startup/ConfiguredUpstreamsSpec.groovy | 16 ++++--- .../test/EthereumPosRpcUpstreamMock.groovy | 5 ++- .../test/EthereumRpcUpstreamMock.groovy | 5 ++- .../dshackle/upstream/FilteredApisSpec.groovy | 4 +- .../grpc/EthereumGrpcUpstreamSpec.groovy | 7 +-- src/test/resources/configs/chains-basic.yaml | 7 +++ 25 files changed, 237 insertions(+), 80 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt create mode 100644 src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy create mode 100644 src/test/resources/configs/chains-basic.yaml diff --git a/docs/04-upstream-config.adoc b/docs/04-upstream-config.adoc index c749fc90..22063ecb 100644 --- a/docs/04-upstream-config.adoc +++ b/docs/04-upstream-config.adoc @@ -32,6 +32,8 @@ If you request balance for an address that is not indexed then it returns 0 bala - To track all transactions you need to setup index for transactions, which is disabled by default. Run it with `-reindex` option, or set `txindex=1` in the config. +=== + === Example Configuration .upstreams.yaml diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index 0c2a9d9f..ffc4913a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -16,14 +16,7 @@ */ package io.emeraldpay.dshackle -import io.emeraldpay.dshackle.config.CacheConfig -import io.emeraldpay.dshackle.config.HealthConfig -import io.emeraldpay.dshackle.config.MainConfig -import io.emeraldpay.dshackle.config.MainConfigReader -import io.emeraldpay.dshackle.config.MonitoringConfig -import io.emeraldpay.dshackle.config.SignatureConfig -import io.emeraldpay.dshackle.config.TokensConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.config.* import org.bouncycastle.jce.provider.BouncyCastleProvider import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -141,4 +134,9 @@ open class Config( open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig { return mainConfig.health } + + @Bean + open fun chainsConfig(mainConfig: MainConfig): ChainsConfig { + return mainConfig.chains + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt new file mode 100644 index 00000000..6872ed9c --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt @@ -0,0 +1,19 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Chain + +class ChainsConfig(var chains: Map, val currentDefault: ChainConfig) { + companion object { + @JvmStatic + fun default(): ChainsConfig = ChainsConfig(emptyMap(), ChainConfig.default()) + } + + data class ChainConfig(val syncingLagSize: Int, val laggingLagSize: Int) { + companion object { + @JvmStatic + fun default() = ChainConfig(6, 1) + } + } + + fun resolve(chain: Chain) = chains[chain] ?: currentDefault +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt new file mode 100644 index 00000000..db3a08c0 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt @@ -0,0 +1,45 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Global +import org.yaml.snakeyaml.nodes.CollectionNode +import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream + +class ChainsConfigReader : YamlConfigReader(), ConfigReader { + + fun read(input: InputStream): ChainsConfig { + val configNode = readNode(input) + return read(configNode) + } + + override fun read(input: MappingNode?): ChainsConfig { + val chains = getList(input, "chains")?.let { + readChains(it) + } + + if (chains == null) { + return ChainsConfig.default() + } else { + + val default = chains.firstOrNull { it.first == "default" }?.second ?: ChainsConfig.ChainConfig.default() + + return ChainsConfig( + chains.filter { it.first != "default" } + .map { Global.chainById(it.first) to it.second } + .associateBy({ it.first }, { it.second }), + default + ) + } + } + + private fun readChains(node: CollectionNode): List> { + return node.value.map { + val key = getValueAsString(it, "name") ?: throw IllegalArgumentException() + val value = ChainsConfig.ChainConfig( + getValueAsInt(it, "syncing-size") ?: throw IllegalArgumentException(), + getValueAsInt(it, "lagging-size") ?: throw IllegalArgumentException() + ) + key to value + } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index 94c06678..8715842c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -29,4 +29,5 @@ class MainConfig { var health: HealthConfig = HealthConfig.default() var signature: SignatureConfig? = null var compression: CompressionConfig = CompressionConfig.default() + var chains: ChainsConfig = ChainsConfig.default() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index 79e29c2d..66d5ceab 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -38,6 +38,7 @@ class MainConfigReader( private val healthConfigReader = HealthConfigReader() private val signatureConfigReader = SignatureConfigReader(fileResolver) private val compressionConfigReader = CompressionConfigReader() + private val chainsConfigReader = ChainsConfigReader() fun read(input: InputStream): MainConfig? { val configNode = readNode(input) @@ -87,6 +88,9 @@ class MainConfigReader( compressionConfigReader.read(input).let { config.compression = it } + chainsConfigReader.read(input).let { + config.chains = it + } return config } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index ebd86bde..0945972e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -21,6 +21,7 @@ import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.upstream.* @@ -64,7 +65,8 @@ open class ConfiguredUpstreams( private val callTargets: CallTargetsHolder, private val eventPublisher: ApplicationEventPublisher, @Qualifier("grpcChannelExecutor") - private val channelExecutor: Executor + private val channelExecutor: Executor, + private val chainsConfig: ChainsConfig ) : ApplicationRunner { private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) @@ -95,11 +97,22 @@ open class ConfiguredUpstreams( .merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults()) val upstream = when (BlockchainType.from(chain)) { BlockchainType.EVM_POW -> { - buildEthereumUpstream(up.nodeId, up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options) + buildEthereumUpstream( + up.nodeId, + up.cast(UpstreamsConfig.EthereumConnection::class.java), + chain, + options, + chainsConfig.resolve(chain) + ) } BlockchainType.BITCOIN -> { - buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options) + buildBitcoinUpstream( + up.cast(UpstreamsConfig.BitcoinConnection::class.java), + chain, + options, + chainsConfig.resolve(chain) + ) } BlockchainType.EVM_POS -> { @@ -107,7 +120,8 @@ open class ConfiguredUpstreams( up.nodeId, up.cast(UpstreamsConfig.EthereumPosConnection::class.java), chain, - options + options, + chainsConfig.resolve(chain) ) } } @@ -167,7 +181,8 @@ open class ConfiguredUpstreams( nodeId: Int?, config: UpstreamsConfig.Upstream, chain: Chain, - options: UpstreamsConfig.Options + options: UpstreamsConfig.Options, + chainConf: ChainsConfig.ChainConfig ): Upstream? { val conn = config.connection!! val execution = conn.execution @@ -200,7 +215,8 @@ open class ConfiguredUpstreams( options, config.role, methods, QuorumForLabels.QuorumItem(1, config.labels), - connectorFactory + connectorFactory, + chainConf ) upstream.start() return upstream @@ -209,7 +225,8 @@ open class ConfiguredUpstreams( private fun buildBitcoinUpstream( config: UpstreamsConfig.Upstream, chain: Chain, - options: UpstreamsConfig.Options + options: UpstreamsConfig.Options, + chainConf: ChainsConfig.ChainConfig ): Upstream? { val conn = config.connection!! val httpFactory = buildHttpFactory(conn) @@ -242,7 +259,7 @@ open class ConfiguredUpstreams( chain, directApi, head, options, config.role, QuorumForLabels.QuorumItem(1, config.labels), - methods, esplora + methods, esplora, chainConf ) upstream.start() return upstream @@ -252,7 +269,8 @@ open class ConfiguredUpstreams( nodeId: Int?, config: UpstreamsConfig.Upstream, chain: Chain, - options: UpstreamsConfig.Options + options: UpstreamsConfig.Options, + chainConf: ChainsConfig.ChainConfig ): EthereumRpcUpstream? { val conn = config.connection!! @@ -279,7 +297,8 @@ open class ConfiguredUpstreams( options, config.role, methods, QuorumForLabels.QuorumItem(1, config.labels), - connectorFactory + connectorFactory, + chainConf ) upstream.start() return upstream @@ -309,7 +328,8 @@ open class ConfiguredUpstreams( endpoint.upstreamRating, config.labels, grpcUpstreamsScheduler, - channelExecutor + channelExecutor, + chainsConfig ).apply { timeout = options.timeout } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt index 3b52ecb7..46ed8e42 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.calls.CallMethods @@ -33,36 +34,20 @@ abstract class DefaultUpstream( private val options: UpstreamsConfig.Options, private val role: UpstreamsConfig.UpstreamRole, private val targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? + node: QuorumForLabels.QuorumItem?, + private val chainConfig: ChainsConfig.ChainConfig ) : Upstream { - constructor( - id: String, - hash: Byte, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - targets: CallMethods? - ) : - this( - id, - hash, - Long.MAX_VALUE, - UpstreamAvailability.UNAVAILABLE, - options, - role, - targets, - QuorumForLabels.QuorumItem.empty() - ) - constructor( id: String, hash: Byte, options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - node: QuorumForLabels.QuorumItem? + node: QuorumForLabels.QuorumItem?, + chainConfig: ChainsConfig.ChainConfig ) : - this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node) + this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node, chainConfig) companion object { private val log = LoggerFactory.getLogger(DefaultUpstream::class.java) @@ -111,8 +96,8 @@ abstract class DefaultUpstream( } return if (proposed == UpstreamAvailability.OK) { when { - lag > 6 -> UpstreamAvailability.SYNCING - lag > 1 -> UpstreamAvailability.LAGGING + lag > chainConfig.syncingLagSize -> UpstreamAvailability.SYNCING + lag > chainConfig.laggingLagSize -> UpstreamAvailability.LAGGING else -> proposed } } else proposed diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt index fee85f3c..749ead9c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels @@ -37,8 +38,9 @@ open class BitcoinRpcUpstream( role: UpstreamsConfig.UpstreamRole, node: QuorumForLabels.QuorumItem, callMethods: CallMethods, - esploraClient: EsploraClient? = null -) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient), Lifecycle { + esploraClient: EsploraClient? = null, + chainConfig: ChainsConfig.ChainConfig +) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient, chainConfig), Lifecycle { companion object { private val log = LoggerFactory.getLogger(BitcoinRpcUpstream::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt index 330d645e..c8218960 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.DefaultUpstream @@ -30,15 +31,17 @@ abstract class BitcoinUpstream( role: UpstreamsConfig.UpstreamRole, callMethods: CallMethods, node: QuorumForLabels.QuorumItem, - val esploraClient: EsploraClient? = null -) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node) { + val esploraClient: EsploraClient? = null, + private val chainConfig: ChainsConfig.ChainConfig +) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node, chainConfig) { constructor( id: String, chain: Chain, options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole - ) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty()) + role: UpstreamsConfig.UpstreamRole, + chainConfig: ChainsConfig.ChainConfig + ) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty(), null, chainConfig) companion object { private val log = LoggerFactory.getLogger(BitcoinUpstream::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt index 3661d31e..f3182f2a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt @@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels @@ -40,8 +41,9 @@ open class EthereumRpcUpstream( role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, private val node: QuorumForLabels.QuorumItem?, - connectorFactory: ConnectorFactory -) : EthereumUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled { + connectorFactory: ConnectorFactory, + chainConfig: ChainsConfig.ChainConfig +) : EthereumUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled { private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java) private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions()) private val connector: EthereumConnector = connectorFactory.create(this, validator, chain) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt index ba9046a9..ea87aff0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.Capability @@ -28,8 +29,9 @@ abstract class EthereumUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? -) : DefaultUpstream(id, hash, options, role, targets, node) { + private val node: QuorumForLabels.QuorumItem?, + chainConfig: ChainsConfig.ChainConfig +) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) { private val capabilities = if (options.providesBalance != false) { setOf(Capability.RPC, Capability.BALANCE) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt index c2a027fa..51f0b86c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt @@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels @@ -39,9 +40,10 @@ open class EthereumPosRpcUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem?, - connectorFactory: ConnectorFactory -) : EthereumPosUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled { + node: QuorumForLabels.QuorumItem?, + connectorFactory: ConnectorFactory, + chainConfig: ChainsConfig.ChainConfig +) : EthereumPosUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled { private val log = LoggerFactory.getLogger(EthereumPosRpcUpstream::class.java) private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions()) private val connector: EthereumConnector = connectorFactory.create(this, validator, chain) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt index a3f7a84f..fadb6921 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.Capability @@ -28,8 +29,9 @@ abstract class EthereumPosUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? -) : DefaultUpstream(id, hash, options, role, targets, node) { + private val node: QuorumForLabels.QuorumItem?, + chainConfig: ChainsConfig.ChainConfig +) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) { private val capabilities = if (options.providesBalance != false) { setOf(Capability.RPC, Capability.BALANCE) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt index 5eedd426..3e8db1d1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt @@ -19,6 +19,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId @@ -51,12 +52,14 @@ class BitcoinGrpcUpstream( chain: Chain, val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, private val client: JsonRpcGrpcClient, - overrideLabels: UpstreamsConfig.Labels? + overrideLabels: UpstreamsConfig.Labels?, + chainConfig: ChainsConfig.ChainConfig ) : BitcoinUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", chain, UpstreamsConfig.Options.getDefaults(), - role + role, + chainConfig ), GrpcUpstream, Lifecycle { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt index 2806f0dd..1b2891b2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt @@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc.ReactorBlockchainStub import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId @@ -57,14 +58,16 @@ open class EthereumGrpcUpstream( private val chain: Chain, private val remote: ReactorBlockchainStub, private val client: JsonRpcGrpcClient, - overrideLabels: UpstreamsConfig.Labels? + overrideLabels: UpstreamsConfig.Labels?, + chainConfig: ChainsConfig.ChainConfig ) : EthereumUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", hash, UpstreamsConfig.Options.getDefaults(), role, null, - null + null, + chainConfig ), GrpcUpstream, Lifecycle { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt index 50abdf1d..f2ecd62b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt @@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId @@ -58,13 +59,16 @@ open class EthereumPosGrpcUpstream( private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, client: JsonRpcGrpcClient, nodeRating: Int, - overrideLabels: UpstreamsConfig.Labels? + overrideLabels: UpstreamsConfig.Labels?, + chainConfig: ChainsConfig.ChainConfig ) : EthereumPosUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", hash, UpstreamsConfig.Options.getDefaults(), role, - null, null + null, + null, + chainConfig ), GrpcUpstream, Lifecycle { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt index 13035eb6..115c812c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt @@ -24,6 +24,7 @@ import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.config.AuthConfig +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.upstream.DefaultUpstream @@ -64,7 +65,8 @@ class GrpcUpstreams( private val nodeRating: Int, private val labels: UpstreamsConfig.Labels, private val chainStatusScheduler: Scheduler, - private val grpcExecutor: Executor + private val grpcExecutor: Executor, + private val chainsConfig: ChainsConfig ) { private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java) @@ -185,13 +187,13 @@ class GrpcUpstreams( private val creators: Map DefaultUpstream> = mapOf( BlockchainType.EVM_POW to { chain, rpcClient -> - EthereumGrpcUpstream(id, hash, role, chain, client, rpcClient, labels) + EthereumGrpcUpstream(id, hash, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain)) }, BlockchainType.EVM_POS to { chain, rpcClient -> - EthereumPosGrpcUpstream(id, hash, role, chain, client, rpcClient, nodeRating, labels) + EthereumPosGrpcUpstream(id, hash, role, chain, client, rpcClient, nodeRating, labels, chainsConfig.resolve(chain)) }, BlockchainType.BITCOIN to { chain, rpcClient -> - BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels) + BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain)) } ) diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy new file mode 100644 index 00000000..b0ef34eb --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2019 ETCDEV GmbH + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Chain +import spock.lang.Specification + +class ChainsConfigReaderSpec extends Specification { + + ChainsConfigReader reader = new ChainsConfigReader() + + def "Parse standard config"() { + setup: + def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml") + when: + def config = reader.read(stream) + def act = config.resolve(Chain.ETHEREUM) + def act2 = config.resolve(Chain.POLYGON) + then: + act.laggingLagSize == 5 + act.syncingLagSize == 10 + + act2.laggingLagSize == 1 + act2.syncingLagSize == 6 + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy index 870d9430..e4e89bb0 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy @@ -1,6 +1,7 @@ package io.emeraldpay.dshackle.startup import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.quorum.NonEmptyQuorum import io.emeraldpay.dshackle.upstream.CallTargetsHolder @@ -23,7 +24,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) def methods = new UpstreamsConfig.Methods( [ @@ -49,7 +51,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) def methods = new UpstreamsConfig.Methods( [ @@ -74,7 +77,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) expect: configurer.getHash(node, src) == expected @@ -94,7 +98,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) when: def h1 = configurer.getHash(null, "hohoho") @@ -119,7 +124,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) def methodsGroup = new UpstreamsConfig.MethodGroups( ["filter"] as Set, diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy index 8b12e580..9569bcee 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy @@ -16,7 +16,7 @@ */ package io.emeraldpay.dshackle.test - +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods @@ -73,7 +73,8 @@ class EthereumPosRpcUpstreamMock extends EthereumPosRpcUpstream { UpstreamsConfig.UpstreamRole.PRIMARY, methods, new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(labels)), - new ConnectorFactoryMock(api, new EthereumHeadMock())) + new ConnectorFactoryMock(api, new EthereumHeadMock()), + ChainsConfig.ChainConfig.default()) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) setStatus(UpstreamAvailability.OK) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy index 4a490d5a..a822326a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy @@ -16,7 +16,7 @@ */ package io.emeraldpay.dshackle.test - +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods @@ -65,7 +65,8 @@ class EthereumRpcUpstreamMock extends EthereumRpcUpstream { UpstreamsConfig.UpstreamRole.PRIMARY, methods, new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()), - new ConnectorFactoryMock(api, new EthereumHeadMock())) + new ConnectorFactoryMock(api, new EthereumHeadMock()), + ChainsConfig.ChainConfig.default()) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) setStatus(UpstreamAvailability.OK) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy index 9d9312ae..efa0505e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.test.EthereumApiStub @@ -57,7 +58,8 @@ class FilteredApisSpec extends Specification { UpstreamsConfig.UpstreamRole.PRIMARY, ethereumTargets, new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)), - connectorFactory + connectorFactory, + ChainsConfig.ChainConfig.default() ) } def matcher = new Selector.LabelMatcher("test", ["foo"]) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy index e13f355e..85d14ec2 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy @@ -22,6 +22,7 @@ import io.emeraldpay.api.proto.BlockchainGrpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.test.MockGrpcServer @@ -83,7 +84,7 @@ class EthereumGrpcUpstreamSpec extends Specification { ) } }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null) + def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null, ChainsConfig.ChainConfig.default()) upstream.setLag(0) upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) @@ -141,7 +142,7 @@ class EthereumGrpcUpstreamSpec extends Specification { ) } }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics), null) + def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics), null, ChainsConfig.ChainConfig.default()) upstream.setLag(0) upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) @@ -203,7 +204,7 @@ class EthereumGrpcUpstreamSpec extends Specification { finished.complete(true) } }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null) + def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null, ChainsConfig.ChainConfig.default()) upstream.setLag(0) upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) diff --git a/src/test/resources/configs/chains-basic.yaml b/src/test/resources/configs/chains-basic.yaml new file mode 100644 index 00000000..27cf52de --- /dev/null +++ b/src/test/resources/configs/chains-basic.yaml @@ -0,0 +1,7 @@ +version: v1 + +chains: + - name: eth + syncing-size: 10 + lagging-size: 5 + From ad0d0b2b57549e8d93a9d061c8dd83714a852784 Mon Sep 17 00:00:00 2001 From: a10zn8 Date: Fri, 10 Feb 2023 20:22:35 +0400 Subject: [PATCH 2/3] refactoring --- .../dshackle/config/AccessLogReader.kt | 8 +-- .../dshackle/config/AuthConfigReader.kt | 6 +- .../dshackle/config/CacheConfigReader.kt | 8 +-- .../dshackle/config/ChainsConfig.kt | 24 +++++++- .../dshackle/config/ChainsConfigReader.kt | 55 ++++++++++--------- .../config/CompressionConfigReader.kt | 8 +-- .../dshackle/config/HealthConfigReader.kt | 8 +-- .../dshackle/config/MainConfigReader.kt | 8 +-- .../dshackle/config/MonitoringConfigReader.kt | 14 +---- .../dshackle/config/ProxyConfigReader.kt | 9 +-- .../dshackle/config/SignatureConfigReader.kt | 14 +---- .../dshackle/config/TokensConfigReader.kt | 8 +-- .../dshackle/config/UpstreamsConfigReader.kt | 10 +--- .../dshackle/config/YamlConfigReader.kt | 8 ++- src/main/resources/chains.yaml | 16 ++++++ .../config/YamlConfigReaderSpec.groovy | 8 ++- src/test/resources/configs/chains-basic.yaml | 15 +++-- 17 files changed, 107 insertions(+), 120 deletions(-) create mode 100644 src/main/resources/chains.yaml diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt index bdc0e3b9..c4daab89 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt @@ -1,14 +1,8 @@ package io.emeraldpay.dshackle.config -import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -class AccessLogReader : YamlConfigReader(), ConfigReader { - - companion object { - private val log = LoggerFactory.getLogger(AccessLogReader::class.java) - } - +class AccessLogReader : YamlConfigReader() { override fun read(input: MappingNode?): AccessLogConfig { return getMapping(input, "accessLog")?.let { node -> val enabled = getValueAsBool(node, "enabled") ?: false diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt index 18efced6..272da506 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt @@ -19,7 +19,7 @@ package io.emeraldpay.dshackle.config import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -class AuthConfigReader : YamlConfigReader() { +class AuthConfigReader : YamlConfigReader() { companion object { private val log = LoggerFactory.getLogger(AuthConfigReader::class.java) @@ -82,4 +82,8 @@ class AuthConfigReader : YamlConfigReader() { auth } } + + override fun read(input: MappingNode?): AuthConfig? { + return null + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt index 1f779231..2cb8a772 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt @@ -17,19 +17,13 @@ package io.emeraldpay.dshackle.config import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream -class CacheConfigReader : YamlConfigReader(), ConfigReader { +class CacheConfigReader : YamlConfigReader() { companion object { private val log = LoggerFactory.getLogger(CacheConfigReader::class.java) } - fun read(input: InputStream): CacheConfig? { - val configNode = readNode(input) - return read(configNode) - } - override fun read(input: MappingNode?): CacheConfig? { return getMapping(input, "cache")?.let { node -> val config = CacheConfig() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt index 6872ed9c..6565e6d8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt @@ -1,11 +1,19 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Chain +import java.lang.IllegalStateException -class ChainsConfig(var chains: Map, val currentDefault: ChainConfig) { +class ChainsConfig(private val chains: Map?, val currentDefault: RawChainConfig?) { companion object { @JvmStatic - fun default(): ChainsConfig = ChainsConfig(emptyMap(), ChainConfig.default()) + fun default(): ChainsConfig = ChainsConfig(emptyMap(), RawChainConfig.default()) + } + + data class RawChainConfig(val syncingLagSize: Int?, val laggingLagSize: Int?) { + companion object { + @JvmStatic + fun default() = RawChainConfig(6, 1) + } } data class ChainConfig(val syncingLagSize: Int, val laggingLagSize: Int) { @@ -15,5 +23,15 @@ class ChainsConfig(var chains: Map, val currentDefault: Chai } } - fun resolve(chain: Chain) = chains[chain] ?: currentDefault + fun resolve(chain: Chain): ChainConfig { + val default = currentDefault ?: panic() + val raw = chains?.get(chain) ?: default + + return ChainConfig( + laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(), + syncingLagSize = raw.syncingLagSize ?: default.syncingLagSize ?: panic(), + ) + } + + fun panic(): Nothing = throw IllegalStateException("Chains settings state is illegal - default config is null") } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt index db3a08c0..ca41a618 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt @@ -3,43 +3,48 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Global import org.yaml.snakeyaml.nodes.CollectionNode import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream -class ChainsConfigReader : YamlConfigReader(), ConfigReader { - - fun read(input: InputStream): ChainsConfig { - val configNode = readNode(input) - return read(configNode) - } +class ChainsConfigReader : YamlConfigReader() { override fun read(input: MappingNode?): ChainsConfig { - val chains = getList(input, "chains")?.let { - readChains(it) - } + return getMapping(input, "chain-settings")?.let { - if (chains == null) { - return ChainsConfig.default() - } else { + val chains = getList(it, "chains")?.let { + readChains(it) + } - val default = chains.firstOrNull { it.first == "default" }?.second ?: ChainsConfig.ChainConfig.default() + val default = getMapping(it, "default")?.let { + readChain(it) + } return ChainsConfig( - chains.filter { it.first != "default" } - .map { Global.chainById(it.first) to it.second } - .associateBy({ it.first }, { it.second }), + chains + ?.map { Global.chainById(it.first) to it.second } + ?.associateBy({ it.first }, { it.second }) ?: emptyMap(), default ) + } ?: ChainsConfig.default() + } + + private fun readChain(node: MappingNode): ChainsConfig.RawChainConfig? { + return getMapping(node, "lags")?.let { + return ChainsConfig.RawChainConfig( + getValueAsInt(it, "syncing"), + getValueAsInt(it, "lagging") + ) } } - private fun readChains(node: CollectionNode): List> { - return node.value.map { - val key = getValueAsString(it, "name") ?: throw IllegalArgumentException() - val value = ChainsConfig.ChainConfig( - getValueAsInt(it, "syncing-size") ?: throw IllegalArgumentException(), - getValueAsInt(it, "lagging-size") ?: throw IllegalArgumentException() - ) - key to value + private fun readChains(node: CollectionNode): List> { + return node.value.mapNotNull { + val key = getValueAsString(it, "name") + ?: throw InvalidConfigYamlException(filename, it.startMark, "chain name required") + val value = readChain(it) + if (value != null) { + return@mapNotNull key to value + } else { + return@mapNotNull null + } } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/CompressionConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/CompressionConfigReader.kt index 563a5661..51f08c7b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/CompressionConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/CompressionConfigReader.kt @@ -1,14 +1,8 @@ package io.emeraldpay.dshackle.config import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream - -class CompressionConfigReader : YamlConfigReader(), ConfigReader { - fun read(input: InputStream): CompressionConfig? { - val configNode = readNode(input) - return read(configNode) - } +class CompressionConfigReader : YamlConfigReader() { override fun read(input: MappingNode?): CompressionConfig { val config = CompressionConfig() getMapping(input, "compression")?.let { node -> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt index fb6a23f7..433005bc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -20,19 +20,13 @@ import io.emeraldpay.dshackle.Global import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.CollectionNode import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream -class HealthConfigReader : YamlConfigReader(), ConfigReader { +class HealthConfigReader : YamlConfigReader() { companion object { private val log = LoggerFactory.getLogger(HealthConfigReader::class.java) } - fun read(input: InputStream): HealthConfig { - val configNode = readNode(input) - return read(configNode) - } - override fun read(input: MappingNode?): HealthConfig { return readInternal(getMapping(input, "health")) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index 66d5ceab..aa11af2d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -18,11 +18,10 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.FileResolver import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream class MainConfigReader( fileResolver: FileResolver -) : YamlConfigReader(), ConfigReader { +) : YamlConfigReader() { companion object { private val log = LoggerFactory.getLogger(MainConfigReader::class.java) @@ -40,11 +39,6 @@ class MainConfigReader( private val compressionConfigReader = CompressionConfigReader() private val chainsConfigReader = ChainsConfigReader() - fun read(input: InputStream): MainConfig? { - val configNode = readNode(input) - return read(configNode) - } - override fun read(input: MappingNode?): MainConfig? { val config = MainConfig() getValueAsString(input, "host")?.let { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt index db0a84f1..e4aed615 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt @@ -15,21 +15,9 @@ */ package io.emeraldpay.dshackle.config -import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream - -class MonitoringConfigReader : YamlConfigReader(), ConfigReader { - - companion object { - private val log = LoggerFactory.getLogger(MonitoringConfigReader::class.java) - } - - fun read(input: InputStream): MonitoringConfig { - val configNode = readNode(input) - return read(configNode) - } +class MonitoringConfigReader : YamlConfigReader() { override fun read(input: MappingNode?): MonitoringConfig { return readInternal(getMapping(input, "monitoring")) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt index 9c53dca1..b9ebbc31 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt @@ -21,25 +21,18 @@ import io.emeraldpay.dshackle.Global import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream /** * Read YAML config, part related to Proxy configuration */ -class ProxyConfigReader : YamlConfigReader(), ConfigReader { +class ProxyConfigReader : YamlConfigReader() { companion object { private val log = LoggerFactory.getLogger(ProxyConfigReader::class.java) } - private var filename = "dshackle.yaml" private val authConfigReader = AuthConfigReader() - fun read(input: InputStream): ProxyConfig? { - val configNode = readNode(input) - return read(configNode) - } - override fun read(input: MappingNode?): ProxyConfig? { return readInternal(getMapping(input, "proxy")) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt index 8ea7ccf4..c40a59e1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt @@ -1,21 +1,9 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.FileResolver -import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream - -class SignatureConfigReader(val fileResolver: FileResolver) : YamlConfigReader(), ConfigReader { - - companion object { - private val log = LoggerFactory.getLogger(SignatureConfig::class.java) - } - - fun read(input: InputStream): SignatureConfig? { - val configNode = readNode(input) - return read(configNode) - } +class SignatureConfigReader(val fileResolver: FileResolver) : YamlConfigReader() { override fun read(input: MappingNode?): SignatureConfig? { return getMapping(input, "signed-response")?.let { node -> val config = SignatureConfig() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt index 85205277..733d09de 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt @@ -18,18 +18,12 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Global import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream import java.util.Locale -class TokensConfigReader : YamlConfigReader(), ConfigReader { +class TokensConfigReader : YamlConfigReader() { private val log = LoggerFactory.getLogger(TokensConfigReader::class.java) - fun read(input: InputStream): TokensConfig? { - val configNode = readNode(input) - return read(configNode) - } - override fun read(input: MappingNode?): TokensConfig? { val tokens = getList(input, "tokens")?.value?.map { node -> val token = TokensConfig.Token() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 3eaa9e60..9ede1a5b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -20,31 +20,25 @@ import io.emeraldpay.dshackle.FileResolver import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode -import java.io.InputStream import java.net.URI import java.time.Duration import java.util.Locale class UpstreamsConfigReader( private val fileResolver: FileResolver -) : YamlConfigReader(), ConfigReader { +) : YamlConfigReader() { private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java) private val authConfigReader = AuthConfigReader() private val knownNodeIds: MutableSet = HashSet() - fun read(input: InputStream): UpstreamsConfig? { - val configNode = readNode(input) - return readInternal(configNode) - } - override fun read(input: MappingNode?): UpstreamsConfig? { return getMapping(input, "cluster")?.let { readInternal(it) } } - fun readInternal(input: MappingNode?): UpstreamsConfig? { + fun readInternal(input: MappingNode?): UpstreamsConfig { val config = UpstreamsConfig() getList(input, "defaults")?.value?.forEach { opts -> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt index aa256d60..9a277c0a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt @@ -25,9 +25,15 @@ import java.io.InputStream import java.io.InputStreamReader import java.util.Locale -abstract class YamlConfigReader { +abstract class YamlConfigReader : ConfigReader { private val envVariables = EnvVariables() + val filename = "dshackle.yaml" + + fun read(input: InputStream): T? { + val configNode = readNode(input) + return read(configNode) + } fun readNode(input: String): MappingNode { return readNode(input.byteInputStream()) } diff --git a/src/main/resources/chains.yaml b/src/main/resources/chains.yaml new file mode 100644 index 00000000..ad84322a --- /dev/null +++ b/src/main/resources/chains.yaml @@ -0,0 +1,16 @@ +version: v1 + +chain-settings: + default: + lags: + syncing: 6 + lagging: 1 + chains: + - id: eth + lags: + syncing: 6 + lagging: 1 + - name: polygon + lags: + syncing: 20 + lagging: 10 diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/YamlConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/YamlConfigReaderSpec.groovy index 8e7cd277..bb9d9b98 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/YamlConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/YamlConfigReaderSpec.groovy @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.config +import org.jetbrains.annotations.Nullable import org.yaml.snakeyaml.Yaml import org.yaml.snakeyaml.nodes.MappingNode import spock.lang.Specification @@ -41,5 +42,10 @@ class YamlConfigReaderSpec extends Specification { return new Yaml().compose(new StringReader("$key: $value")) as MappingNode } - class Impl extends YamlConfigReader {} + class Impl extends YamlConfigReader { + @Override + Object read(@Nullable MappingNode input) { + return null + } + } } diff --git a/src/test/resources/configs/chains-basic.yaml b/src/test/resources/configs/chains-basic.yaml index 27cf52de..606e552e 100644 --- a/src/test/resources/configs/chains-basic.yaml +++ b/src/test/resources/configs/chains-basic.yaml @@ -1,7 +1,12 @@ version: v1 -chains: - - name: eth - syncing-size: 10 - lagging-size: 5 - +chain-settings: + default: + lags: + syncing: 6 + lagging: 1 + chains: + - id: eth + lags: + syncing: 6 + lagging: 1 From 2809e87d3314db2e29eb6bff19216c0099b97651 Mon Sep 17 00:00:00 2001 From: a10zn8 Date: Fri, 10 Feb 2023 21:33:58 +0400 Subject: [PATCH 3/3] config merge --- docs/04-upstream-config.adoc | 32 +++++++++++++- .../dshackle/config/ChainsConfig.kt | 30 ++++++++++++- .../dshackle/config/ChainsConfigReader.kt | 18 +++++++- .../dshackle/config/MainConfigReader.kt | 7 +-- .../dshackle/config/UpstreamsConfigReader.kt | 9 +++- src/main/resources/chains.yaml | 2 +- .../config/ChainsConfigReaderSpec.groovy | 23 +++++++--- .../config/UpstreamsConfigReaderSpec.groovy | 38 ++++++++-------- .../dshackle/config/ChainsConfigTest.kt | 44 +++++++++++++++++++ src/test/resources/configs/chains-basic.yaml | 6 +++ 10 files changed, 169 insertions(+), 40 deletions(-) create mode 100644 src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt diff --git a/docs/04-upstream-config.adoc b/docs/04-upstream-config.adoc index 22063ecb..03279dc0 100644 --- a/docs/04-upstream-config.adoc +++ b/docs/04-upstream-config.adoc @@ -32,8 +32,6 @@ If you request balance for an address that is not indexed then it returns 0 bala - To track all transactions you need to setup index for transactions, which is disabled by default. Run it with `-reindex` option, or set `txindex=1` in the config. -=== - === Example Configuration .upstreams.yaml @@ -317,3 +315,33 @@ For JSON RPC and Websockets a Basic Authentication can be used: - `username` - username - `password` - password + +=== Chains specific configuration +We can use chain settings to specify chain specific behavior, for example rules for dshackle to work with upstream statuses + +.chains.yaml +[source,yaml] +---- +chain-settings: + default: + lags: + syncing: 6 + lagging: 1 + chains: + - id: eth + lags: + syncing: 6 + lagging: 1 + - id: polygon + lags: + syncing: 20 + lagging: 10 +---- +Options +[cols="2,5a"] +|=== +| Option | Description + +| `lags.syncing` | the size of the lag after which the upstream is determined to be syncing +| `lags.lagging` | the size of the lag after which the upstream is determined to be lagging +|=== diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt index 6565e6d8..1a603d4b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt @@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Chain import java.lang.IllegalStateException -class ChainsConfig(private val chains: Map?, val currentDefault: RawChainConfig?) { +data class ChainsConfig(private val chains: Map, val currentDefault: RawChainConfig?) { companion object { @JvmStatic fun default(): ChainsConfig = ChainsConfig(emptyMap(), RawChainConfig.default()) @@ -25,7 +25,7 @@ class ChainsConfig(private val chains: Map?, val currentD fun resolve(chain: Chain): ChainConfig { val default = currentDefault ?: panic() - val raw = chains?.get(chain) ?: default + val raw = chains[chain] ?: default return ChainConfig( laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(), @@ -33,5 +33,31 @@ class ChainsConfig(private val chains: Map?, val currentD ) } + fun patch(patch: ChainsConfig) = ChainsConfig( + merge(this.chains, patch.chains), + merge(this.currentDefault!!, patch.currentDefault) + ) + + private fun merge( + current: RawChainConfig, + patch: RawChainConfig? + ) = RawChainConfig( + syncingLagSize = patch?.syncingLagSize ?: current.syncingLagSize, + laggingLagSize = patch?.laggingLagSize ?: current.laggingLagSize + ) + + private fun merge( + current: Map, + patch: Map + ): Map { + val currentMut = current.toMutableMap() + + for (k in patch) { + currentMut.merge(k.key, k.value) { v1, v2 -> merge(v1, v2) } + } + + return currentMut.toMap() + } + fun panic(): Nothing = throw IllegalStateException("Chains settings state is illegal - default config is null") } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt index ca41a618..4240fafe 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt @@ -3,10 +3,24 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Global import org.yaml.snakeyaml.nodes.CollectionNode import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream class ChainsConfigReader : YamlConfigReader() { + private val defaultConfig = this::class.java.getResourceAsStream("/chains.yaml")!! + override fun read(input: MappingNode?): ChainsConfig { + val default = readInternal(defaultConfig) + val current = readInternal(input) + return default.patch(current) + } + + fun readInternal(input: InputStream): ChainsConfig { + val configNode = readNode(input) + return readInternal(configNode) + } + + fun readInternal(input: MappingNode?): ChainsConfig { return getMapping(input, "chain-settings")?.let { val chains = getList(it, "chains")?.let { @@ -37,8 +51,8 @@ class ChainsConfigReader : YamlConfigReader() { private fun readChains(node: CollectionNode): List> { return node.value.mapNotNull { - val key = getValueAsString(it, "name") - ?: throw InvalidConfigYamlException(filename, it.startMark, "chain name required") + val key = getValueAsString(it, "id") + ?: throw InvalidConfigYamlException(filename, it.startMark, "chain id required") val value = readChain(it) if (value != null) { return@mapNotNull key to value diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index aa11af2d..6548a0a2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -16,17 +16,12 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.FileResolver -import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode class MainConfigReader( fileResolver: FileResolver ) : YamlConfigReader() { - companion object { - private val log = LoggerFactory.getLogger(MainConfigReader::class.java) - } - private val authConfigReader = AuthConfigReader() private val proxyConfigReader = ProxyConfigReader() private val upstreamsConfigReader = UpstreamsConfigReader(fileResolver) @@ -39,7 +34,7 @@ class MainConfigReader( private val compressionConfigReader = CompressionConfigReader() private val chainsConfigReader = ChainsConfigReader() - override fun read(input: MappingNode?): MainConfig? { + override fun read(input: MappingNode?): MainConfig { val config = MainConfig() getValueAsString(input, "host")?.let { config.host = it diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 9ede1a5b..b55a0c80 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -20,6 +20,7 @@ import io.emeraldpay.dshackle.FileResolver import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream import java.net.URI import java.time.Duration import java.util.Locale @@ -38,6 +39,10 @@ class UpstreamsConfigReader( } } + fun readInternal(input: InputStream): UpstreamsConfig? { + val configNode = readNode(input) + return readInternal(configNode) + } fun readInternal(input: MappingNode?): UpstreamsConfig { val config = UpstreamsConfig() @@ -55,7 +60,7 @@ class UpstreamsConfigReader( getValueAsString(input, "include")?.let { path -> fileResolver.resolve(path).let { file -> if (file.exists() && file.isFile && file.canRead()) { - read(file.inputStream())?.let { + readInternal(file.inputStream())?.let { it.upstreams.forEach { upstream -> config.upstreams.add(upstream) } } } else { @@ -67,7 +72,7 @@ class UpstreamsConfigReader( getListOfString(input, "include")?.forEach { path -> fileResolver.resolve(path).let { file -> if (file.exists() && file.isFile && file.canRead()) { - read(file.inputStream())?.let { + readInternal(file.inputStream())?.let { it.upstreams.forEach { upstream -> config.upstreams.add(upstream) } } } else { diff --git a/src/main/resources/chains.yaml b/src/main/resources/chains.yaml index ad84322a..2c061bd5 100644 --- a/src/main/resources/chains.yaml +++ b/src/main/resources/chains.yaml @@ -10,7 +10,7 @@ chain-settings: lags: syncing: 6 lagging: 1 - - name: polygon + - id: polygon lags: syncing: 20 lagging: 10 diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy index b0ef34eb..6d674be4 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy @@ -28,13 +28,24 @@ class ChainsConfigReaderSpec extends Specification { def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml") when: def config = reader.read(stream) - def act = config.resolve(Chain.ETHEREUM) - def act2 = config.resolve(Chain.POLYGON) + def eth = config.resolve(Chain.ETHEREUM) + def pol = config.resolve(Chain.POLYGON) + def opt = config.resolve(Chain.OPTIMISM) + def sep = config.resolve(Chain.TESTNET_SEPOLIA) then: - act.laggingLagSize == 5 - act.syncingLagSize == 10 + eth.laggingLagSize == 1 + eth.syncingLagSize == 6 - act2.laggingLagSize == 1 - act2.syncingLagSize == 6 + pol.laggingLagSize == 10 + pol.syncingLagSize == 20 + + opt.laggingLagSize == 3 + opt.syncingLagSize == 6 + + sep.laggingLagSize == 1 + sep.syncingLagSize == 10 + + eth.laggingLagSize == 1 + eth.syncingLagSize == 6 } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy index 4d198ec0..5b625f31 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy @@ -27,7 +27,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -75,7 +75,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-only.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -100,7 +100,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-full.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -127,7 +127,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -155,7 +155,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-ethereum-pos.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -175,7 +175,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bitcoin-esplora.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -204,7 +204,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ds.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -227,7 +227,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-labels.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -248,7 +248,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-options.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -264,7 +264,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-defaults.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.defaultOptions) { @@ -287,7 +287,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.upstreams.get(0)) { @@ -307,7 +307,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-methods-quorum.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null with(act.upstreams.get(0)) { @@ -330,7 +330,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-no-id.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 @@ -357,7 +357,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-basic.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -369,7 +369,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -381,7 +381,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-2.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 3 @@ -394,7 +394,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-roles-invalid.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -406,7 +406,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-node-id.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 2 @@ -424,7 +424,7 @@ class UpstreamsConfigReaderSpec extends Specification { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-method-groups.yaml") when: - def act = reader.read(config) + def act = reader.readInternal(config) then: act != null act.upstreams.size() == 1 diff --git a/src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt new file mode 100644 index 00000000..d068b9eb --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/config/ChainsConfigTest.kt @@ -0,0 +1,44 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Chain +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +internal class ChainsConfigTest { + + @Test + fun patch() { + val orig = ChainsConfig( + mapOf( + Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 0), + Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2), + Chain.POLYGON to ChainsConfig.RawChainConfig(3, 4) + ), + ChainsConfig.RawChainConfig(1, 2) + ) + + val patch = ChainsConfig( + mapOf( + Chain.BITCOIN to ChainsConfig.RawChainConfig(null, 10000), + Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11), + Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999) + ), + ChainsConfig.RawChainConfig(100, null) + ) + + val res = orig.patch(patch) + + assertEquals( + ChainsConfig( + mapOf( + Chain.BITCOIN to ChainsConfig.RawChainConfig(0, 10000), + Chain.ETHEREUM to ChainsConfig.RawChainConfig(1, 2), + Chain.POLYGON to ChainsConfig.RawChainConfig(10, 11), + Chain.ARBITRUM to ChainsConfig.RawChainConfig(999, 999) + ), + ChainsConfig.RawChainConfig(100, 2) + ), + res + ) + } +} diff --git a/src/test/resources/configs/chains-basic.yaml b/src/test/resources/configs/chains-basic.yaml index 606e552e..8f28e16d 100644 --- a/src/test/resources/configs/chains-basic.yaml +++ b/src/test/resources/configs/chains-basic.yaml @@ -10,3 +10,9 @@ chain-settings: lags: syncing: 6 lagging: 1 + - id: optimism + lags: + lagging: 3 + - id: sepolia + lags: + syncing: 10 \ No newline at end of file