diff --git a/docs/04-upstream-config.adoc b/docs/04-upstream-config.adoc index c749fc90..22063ecb 100644 --- a/docs/04-upstream-config.adoc +++ b/docs/04-upstream-config.adoc @@ -32,6 +32,8 @@ If you request balance for an address that is not indexed then it returns 0 bala - To track all transactions you need to setup index for transactions, which is disabled by default. Run it with `-reindex` option, or set `txindex=1` in the config. +=== + === Example Configuration .upstreams.yaml diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index 0c2a9d9f..ffc4913a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -16,14 +16,7 @@ */ package io.emeraldpay.dshackle -import io.emeraldpay.dshackle.config.CacheConfig -import io.emeraldpay.dshackle.config.HealthConfig -import io.emeraldpay.dshackle.config.MainConfig -import io.emeraldpay.dshackle.config.MainConfigReader -import io.emeraldpay.dshackle.config.MonitoringConfig -import io.emeraldpay.dshackle.config.SignatureConfig -import io.emeraldpay.dshackle.config.TokensConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.config.* import org.bouncycastle.jce.provider.BouncyCastleProvider import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -141,4 +134,9 @@ open class Config( open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig { return mainConfig.health } + + @Bean + open fun chainsConfig(mainConfig: MainConfig): ChainsConfig { + return mainConfig.chains + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt new file mode 100644 index 00000000..6872ed9c --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt @@ -0,0 +1,19 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Chain + +class ChainsConfig(var chains: Map, val currentDefault: ChainConfig) { + companion object { + @JvmStatic + fun default(): ChainsConfig = ChainsConfig(emptyMap(), ChainConfig.default()) + } + + data class ChainConfig(val syncingLagSize: Int, val laggingLagSize: Int) { + companion object { + @JvmStatic + fun default() = ChainConfig(6, 1) + } + } + + fun resolve(chain: Chain) = chains[chain] ?: currentDefault +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt new file mode 100644 index 00000000..db3a08c0 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt @@ -0,0 +1,45 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Global +import org.yaml.snakeyaml.nodes.CollectionNode +import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream + +class ChainsConfigReader : YamlConfigReader(), ConfigReader { + + fun read(input: InputStream): ChainsConfig { + val configNode = readNode(input) + return read(configNode) + } + + override fun read(input: MappingNode?): ChainsConfig { + val chains = getList(input, "chains")?.let { + readChains(it) + } + + if (chains == null) { + return ChainsConfig.default() + } else { + + val default = chains.firstOrNull { it.first == "default" }?.second ?: ChainsConfig.ChainConfig.default() + + return ChainsConfig( + chains.filter { it.first != "default" } + .map { Global.chainById(it.first) to it.second } + .associateBy({ it.first }, { it.second }), + default + ) + } + } + + private fun readChains(node: CollectionNode): List> { + return node.value.map { + val key = getValueAsString(it, "name") ?: throw IllegalArgumentException() + val value = ChainsConfig.ChainConfig( + getValueAsInt(it, "syncing-size") ?: throw IllegalArgumentException(), + getValueAsInt(it, "lagging-size") ?: throw IllegalArgumentException() + ) + key to value + } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index 94c06678..8715842c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -29,4 +29,5 @@ class MainConfig { var health: HealthConfig = HealthConfig.default() var signature: SignatureConfig? = null var compression: CompressionConfig = CompressionConfig.default() + var chains: ChainsConfig = ChainsConfig.default() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index 79e29c2d..66d5ceab 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -38,6 +38,7 @@ class MainConfigReader( private val healthConfigReader = HealthConfigReader() private val signatureConfigReader = SignatureConfigReader(fileResolver) private val compressionConfigReader = CompressionConfigReader() + private val chainsConfigReader = ChainsConfigReader() fun read(input: InputStream): MainConfig? { val configNode = readNode(input) @@ -87,6 +88,9 @@ class MainConfigReader( compressionConfigReader.read(input).let { config.compression = it } + chainsConfigReader.read(input).let { + config.chains = it + } return config } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index ebd86bde..0945972e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -21,6 +21,7 @@ import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.upstream.* @@ -64,7 +65,8 @@ open class ConfiguredUpstreams( private val callTargets: CallTargetsHolder, private val eventPublisher: ApplicationEventPublisher, @Qualifier("grpcChannelExecutor") - private val channelExecutor: Executor + private val channelExecutor: Executor, + private val chainsConfig: ChainsConfig ) : ApplicationRunner { private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) @@ -95,11 +97,22 @@ open class ConfiguredUpstreams( .merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults()) val upstream = when (BlockchainType.from(chain)) { BlockchainType.EVM_POW -> { - buildEthereumUpstream(up.nodeId, up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options) + buildEthereumUpstream( + up.nodeId, + up.cast(UpstreamsConfig.EthereumConnection::class.java), + chain, + options, + chainsConfig.resolve(chain) + ) } BlockchainType.BITCOIN -> { - buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options) + buildBitcoinUpstream( + up.cast(UpstreamsConfig.BitcoinConnection::class.java), + chain, + options, + chainsConfig.resolve(chain) + ) } BlockchainType.EVM_POS -> { @@ -107,7 +120,8 @@ open class ConfiguredUpstreams( up.nodeId, up.cast(UpstreamsConfig.EthereumPosConnection::class.java), chain, - options + options, + chainsConfig.resolve(chain) ) } } @@ -167,7 +181,8 @@ open class ConfiguredUpstreams( nodeId: Int?, config: UpstreamsConfig.Upstream, chain: Chain, - options: UpstreamsConfig.Options + options: UpstreamsConfig.Options, + chainConf: ChainsConfig.ChainConfig ): Upstream? { val conn = config.connection!! val execution = conn.execution @@ -200,7 +215,8 @@ open class ConfiguredUpstreams( options, config.role, methods, QuorumForLabels.QuorumItem(1, config.labels), - connectorFactory + connectorFactory, + chainConf ) upstream.start() return upstream @@ -209,7 +225,8 @@ open class ConfiguredUpstreams( private fun buildBitcoinUpstream( config: UpstreamsConfig.Upstream, chain: Chain, - options: UpstreamsConfig.Options + options: UpstreamsConfig.Options, + chainConf: ChainsConfig.ChainConfig ): Upstream? { val conn = config.connection!! val httpFactory = buildHttpFactory(conn) @@ -242,7 +259,7 @@ open class ConfiguredUpstreams( chain, directApi, head, options, config.role, QuorumForLabels.QuorumItem(1, config.labels), - methods, esplora + methods, esplora, chainConf ) upstream.start() return upstream @@ -252,7 +269,8 @@ open class ConfiguredUpstreams( nodeId: Int?, config: UpstreamsConfig.Upstream, chain: Chain, - options: UpstreamsConfig.Options + options: UpstreamsConfig.Options, + chainConf: ChainsConfig.ChainConfig ): EthereumRpcUpstream? { val conn = config.connection!! @@ -279,7 +297,8 @@ open class ConfiguredUpstreams( options, config.role, methods, QuorumForLabels.QuorumItem(1, config.labels), - connectorFactory + connectorFactory, + chainConf ) upstream.start() return upstream @@ -309,7 +328,8 @@ open class ConfiguredUpstreams( endpoint.upstreamRating, config.labels, grpcUpstreamsScheduler, - channelExecutor + channelExecutor, + chainsConfig ).apply { timeout = options.timeout } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt index 3b52ecb7..46ed8e42 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.calls.CallMethods @@ -33,36 +34,20 @@ abstract class DefaultUpstream( private val options: UpstreamsConfig.Options, private val role: UpstreamsConfig.UpstreamRole, private val targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? + node: QuorumForLabels.QuorumItem?, + private val chainConfig: ChainsConfig.ChainConfig ) : Upstream { - constructor( - id: String, - hash: Byte, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - targets: CallMethods? - ) : - this( - id, - hash, - Long.MAX_VALUE, - UpstreamAvailability.UNAVAILABLE, - options, - role, - targets, - QuorumForLabels.QuorumItem.empty() - ) - constructor( id: String, hash: Byte, options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - node: QuorumForLabels.QuorumItem? + node: QuorumForLabels.QuorumItem?, + chainConfig: ChainsConfig.ChainConfig ) : - this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node) + this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node, chainConfig) companion object { private val log = LoggerFactory.getLogger(DefaultUpstream::class.java) @@ -111,8 +96,8 @@ abstract class DefaultUpstream( } return if (proposed == UpstreamAvailability.OK) { when { - lag > 6 -> UpstreamAvailability.SYNCING - lag > 1 -> UpstreamAvailability.LAGGING + lag > chainConfig.syncingLagSize -> UpstreamAvailability.SYNCING + lag > chainConfig.laggingLagSize -> UpstreamAvailability.LAGGING else -> proposed } } else proposed diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt index fee85f3c..749ead9c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels @@ -37,8 +38,9 @@ open class BitcoinRpcUpstream( role: UpstreamsConfig.UpstreamRole, node: QuorumForLabels.QuorumItem, callMethods: CallMethods, - esploraClient: EsploraClient? = null -) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient), Lifecycle { + esploraClient: EsploraClient? = null, + chainConfig: ChainsConfig.ChainConfig +) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient, chainConfig), Lifecycle { companion object { private val log = LoggerFactory.getLogger(BitcoinRpcUpstream::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt index 330d645e..c8218960 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.DefaultUpstream @@ -30,15 +31,17 @@ abstract class BitcoinUpstream( role: UpstreamsConfig.UpstreamRole, callMethods: CallMethods, node: QuorumForLabels.QuorumItem, - val esploraClient: EsploraClient? = null -) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node) { + val esploraClient: EsploraClient? = null, + private val chainConfig: ChainsConfig.ChainConfig +) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node, chainConfig) { constructor( id: String, chain: Chain, options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole - ) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty()) + role: UpstreamsConfig.UpstreamRole, + chainConfig: ChainsConfig.ChainConfig + ) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty(), null, chainConfig) companion object { private val log = LoggerFactory.getLogger(BitcoinUpstream::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt index 3661d31e..f3182f2a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt @@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels @@ -40,8 +41,9 @@ open class EthereumRpcUpstream( role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, private val node: QuorumForLabels.QuorumItem?, - connectorFactory: ConnectorFactory -) : EthereumUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled { + connectorFactory: ConnectorFactory, + chainConfig: ChainsConfig.ChainConfig +) : EthereumUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled { private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java) private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions()) private val connector: EthereumConnector = connectorFactory.create(this, validator, chain) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt index ba9046a9..ea87aff0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.Capability @@ -28,8 +29,9 @@ abstract class EthereumUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? -) : DefaultUpstream(id, hash, options, role, targets, node) { + private val node: QuorumForLabels.QuorumItem?, + chainConfig: ChainsConfig.ChainConfig +) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) { private val capabilities = if (options.providesBalance != false) { setOf(Capability.RPC, Capability.BALANCE) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt index c2a027fa..51f0b86c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt @@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels @@ -39,9 +40,10 @@ open class EthereumPosRpcUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem?, - connectorFactory: ConnectorFactory -) : EthereumPosUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled { + node: QuorumForLabels.QuorumItem?, + connectorFactory: ConnectorFactory, + chainConfig: ChainsConfig.ChainConfig +) : EthereumPosUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled { private val log = LoggerFactory.getLogger(EthereumPosRpcUpstream::class.java) private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions()) private val connector: EthereumConnector = connectorFactory.create(this, validator, chain) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt index a3f7a84f..fadb6921 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.Capability @@ -28,8 +29,9 @@ abstract class EthereumPosUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? -) : DefaultUpstream(id, hash, options, role, targets, node) { + private val node: QuorumForLabels.QuorumItem?, + chainConfig: ChainsConfig.ChainConfig +) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) { private val capabilities = if (options.providesBalance != false) { setOf(Capability.RPC, Capability.BALANCE) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt index 5eedd426..3e8db1d1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt @@ -19,6 +19,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId @@ -51,12 +52,14 @@ class BitcoinGrpcUpstream( chain: Chain, val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, private val client: JsonRpcGrpcClient, - overrideLabels: UpstreamsConfig.Labels? + overrideLabels: UpstreamsConfig.Labels?, + chainConfig: ChainsConfig.ChainConfig ) : BitcoinUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", chain, UpstreamsConfig.Options.getDefaults(), - role + role, + chainConfig ), GrpcUpstream, Lifecycle { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt index 2806f0dd..1b2891b2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt @@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc.ReactorBlockchainStub import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId @@ -57,14 +58,16 @@ open class EthereumGrpcUpstream( private val chain: Chain, private val remote: ReactorBlockchainStub, private val client: JsonRpcGrpcClient, - overrideLabels: UpstreamsConfig.Labels? + overrideLabels: UpstreamsConfig.Labels?, + chainConfig: ChainsConfig.ChainConfig ) : EthereumUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", hash, UpstreamsConfig.Options.getDefaults(), role, null, - null + null, + chainConfig ), GrpcUpstream, Lifecycle { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt index 50abdf1d..f2ecd62b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt @@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId @@ -58,13 +59,16 @@ open class EthereumPosGrpcUpstream( private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, client: JsonRpcGrpcClient, nodeRating: Int, - overrideLabels: UpstreamsConfig.Labels? + overrideLabels: UpstreamsConfig.Labels?, + chainConfig: ChainsConfig.ChainConfig ) : EthereumPosUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", hash, UpstreamsConfig.Options.getDefaults(), role, - null, null + null, + null, + chainConfig ), GrpcUpstream, Lifecycle { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt index 13035eb6..115c812c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt @@ -24,6 +24,7 @@ import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.config.AuthConfig +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.upstream.DefaultUpstream @@ -64,7 +65,8 @@ class GrpcUpstreams( private val nodeRating: Int, private val labels: UpstreamsConfig.Labels, private val chainStatusScheduler: Scheduler, - private val grpcExecutor: Executor + private val grpcExecutor: Executor, + private val chainsConfig: ChainsConfig ) { private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java) @@ -185,13 +187,13 @@ class GrpcUpstreams( private val creators: Map DefaultUpstream> = mapOf( BlockchainType.EVM_POW to { chain, rpcClient -> - EthereumGrpcUpstream(id, hash, role, chain, client, rpcClient, labels) + EthereumGrpcUpstream(id, hash, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain)) }, BlockchainType.EVM_POS to { chain, rpcClient -> - EthereumPosGrpcUpstream(id, hash, role, chain, client, rpcClient, nodeRating, labels) + EthereumPosGrpcUpstream(id, hash, role, chain, client, rpcClient, nodeRating, labels, chainsConfig.resolve(chain)) }, BlockchainType.BITCOIN to { chain, rpcClient -> - BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels) + BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain)) } ) diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy new file mode 100644 index 00000000..b0ef34eb --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/config/ChainsConfigReaderSpec.groovy @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2019 ETCDEV GmbH + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Chain +import spock.lang.Specification + +class ChainsConfigReaderSpec extends Specification { + + ChainsConfigReader reader = new ChainsConfigReader() + + def "Parse standard config"() { + setup: + def stream = this.class.getClassLoader().getResourceAsStream("configs/chains-basic.yaml") + when: + def config = reader.read(stream) + def act = config.resolve(Chain.ETHEREUM) + def act2 = config.resolve(Chain.POLYGON) + then: + act.laggingLagSize == 5 + act.syncingLagSize == 10 + + act2.laggingLagSize == 1 + act2.syncingLagSize == 6 + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy index 870d9430..e4e89bb0 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/startup/ConfiguredUpstreamsSpec.groovy @@ -1,6 +1,7 @@ package io.emeraldpay.dshackle.startup import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.quorum.NonEmptyQuorum import io.emeraldpay.dshackle.upstream.CallTargetsHolder @@ -23,7 +24,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) def methods = new UpstreamsConfig.Methods( [ @@ -49,7 +51,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) def methods = new UpstreamsConfig.Methods( [ @@ -74,7 +77,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) expect: configurer.getHash(node, src) == expected @@ -94,7 +98,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) when: def h1 = configurer.getHash(null, "hohoho") @@ -119,7 +124,8 @@ class ConfiguredUpstreamsSpec extends Specification { Stub(UpstreamsConfig), callTargetsHolder, Mock(ApplicationEventPublisher), - Executors.newFixedThreadPool(1) + Executors.newFixedThreadPool(1), + ChainsConfig.default() ) def methodsGroup = new UpstreamsConfig.MethodGroups( ["filter"] as Set, diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy index 8b12e580..9569bcee 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy @@ -16,7 +16,7 @@ */ package io.emeraldpay.dshackle.test - +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods @@ -73,7 +73,8 @@ class EthereumPosRpcUpstreamMock extends EthereumPosRpcUpstream { UpstreamsConfig.UpstreamRole.PRIMARY, methods, new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(labels)), - new ConnectorFactoryMock(api, new EthereumHeadMock())) + new ConnectorFactoryMock(api, new EthereumHeadMock()), + ChainsConfig.ChainConfig.default()) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) setStatus(UpstreamAvailability.OK) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy index 4a490d5a..a822326a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy @@ -16,7 +16,7 @@ */ package io.emeraldpay.dshackle.test - +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods @@ -65,7 +65,8 @@ class EthereumRpcUpstreamMock extends EthereumRpcUpstream { UpstreamsConfig.UpstreamRole.PRIMARY, methods, new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()), - new ConnectorFactoryMock(api, new EthereumHeadMock())) + new ConnectorFactoryMock(api, new EthereumHeadMock()), + ChainsConfig.ChainConfig.default()) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) setStatus(UpstreamAvailability.OK) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy index 9d9312ae..efa0505e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.test.EthereumApiStub @@ -57,7 +58,8 @@ class FilteredApisSpec extends Specification { UpstreamsConfig.UpstreamRole.PRIMARY, ethereumTargets, new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)), - connectorFactory + connectorFactory, + ChainsConfig.ChainConfig.default() ) } def matcher = new Selector.LabelMatcher("test", ["foo"]) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy index e13f355e..85d14ec2 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy @@ -22,6 +22,7 @@ import io.emeraldpay.api.proto.BlockchainGrpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.test.MockGrpcServer @@ -83,7 +84,7 @@ class EthereumGrpcUpstreamSpec extends Specification { ) } }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null) + def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null, ChainsConfig.ChainConfig.default()) upstream.setLag(0) upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) @@ -141,7 +142,7 @@ class EthereumGrpcUpstreamSpec extends Specification { ) } }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics), null) + def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics), null, ChainsConfig.ChainConfig.default()) upstream.setLag(0) upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) @@ -203,7 +204,7 @@ class EthereumGrpcUpstreamSpec extends Specification { finished.complete(true) } }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null) + def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null, ChainsConfig.ChainConfig.default()) upstream.setLag(0) upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) diff --git a/src/test/resources/configs/chains-basic.yaml b/src/test/resources/configs/chains-basic.yaml new file mode 100644 index 00000000..27cf52de --- /dev/null +++ b/src/test/resources/configs/chains-basic.yaml @@ -0,0 +1,7 @@ +version: v1 + +chains: + - name: eth + syncing-size: 10 + lagging-size: 5 +