added chains configuration

This commit is contained in:
a10zn8
2023-02-09 20:45:10 +04:00
parent 8649cc03aa
commit ff4c538568
25 changed files with 237 additions and 80 deletions

View File

@@ -16,14 +16,7 @@
*/
package io.emeraldpay.dshackle
import io.emeraldpay.dshackle.config.CacheConfig
import io.emeraldpay.dshackle.config.HealthConfig
import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.config.MainConfigReader
import io.emeraldpay.dshackle.config.MonitoringConfig
import io.emeraldpay.dshackle.config.SignatureConfig
import io.emeraldpay.dshackle.config.TokensConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.config.*
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
@@ -141,4 +134,9 @@ open class Config(
open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig {
return mainConfig.health
}
@Bean
open fun chainsConfig(mainConfig: MainConfig): ChainsConfig {
return mainConfig.chains
}
}

View File

@@ -0,0 +1,19 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
class ChainsConfig(var chains: Map<Chain, ChainConfig>, val currentDefault: ChainConfig) {
companion object {
@JvmStatic
fun default(): ChainsConfig = ChainsConfig(emptyMap(), ChainConfig.default())
}
data class ChainConfig(val syncingLagSize: Int, val laggingLagSize: Int) {
companion object {
@JvmStatic
fun default() = ChainConfig(6, 1)
}
}
fun resolve(chain: Chain) = chains[chain] ?: currentDefault
}

View File

@@ -0,0 +1,45 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import org.yaml.snakeyaml.nodes.CollectionNode
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
class ChainsConfigReader : YamlConfigReader(), ConfigReader<ChainsConfig> {
fun read(input: InputStream): ChainsConfig {
val configNode = readNode(input)
return read(configNode)
}
override fun read(input: MappingNode?): ChainsConfig {
val chains = getList<MappingNode>(input, "chains")?.let {
readChains(it)
}
if (chains == null) {
return ChainsConfig.default()
} else {
val default = chains.firstOrNull { it.first == "default" }?.second ?: ChainsConfig.ChainConfig.default()
return ChainsConfig(
chains.filter { it.first != "default" }
.map { Global.chainById(it.first) to it.second }
.associateBy({ it.first }, { it.second }),
default
)
}
}
private fun readChains(node: CollectionNode<MappingNode>): List<Pair<String, ChainsConfig.ChainConfig>> {
return node.value.map {
val key = getValueAsString(it, "name") ?: throw IllegalArgumentException()
val value = ChainsConfig.ChainConfig(
getValueAsInt(it, "syncing-size") ?: throw IllegalArgumentException(),
getValueAsInt(it, "lagging-size") ?: throw IllegalArgumentException()
)
key to value
}
}
}

View File

@@ -29,4 +29,5 @@ class MainConfig {
var health: HealthConfig = HealthConfig.default()
var signature: SignatureConfig? = null
var compression: CompressionConfig = CompressionConfig.default()
var chains: ChainsConfig = ChainsConfig.default()
}

View File

@@ -38,6 +38,7 @@ class MainConfigReader(
private val healthConfigReader = HealthConfigReader()
private val signatureConfigReader = SignatureConfigReader(fileResolver)
private val compressionConfigReader = CompressionConfigReader()
private val chainsConfigReader = ChainsConfigReader()
fun read(input: InputStream): MainConfig? {
val configNode = readNode(input)
@@ -87,6 +88,9 @@ class MainConfigReader(
compressionConfigReader.read(input).let {
config.compression = it
}
chainsConfigReader.read(input).let {
config.chains = it
}
return config
}
}

View File

@@ -21,6 +21,7 @@ import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.*
@@ -64,7 +65,8 @@ open class ConfiguredUpstreams(
private val callTargets: CallTargetsHolder,
private val eventPublisher: ApplicationEventPublisher,
@Qualifier("grpcChannelExecutor")
private val channelExecutor: Executor
private val channelExecutor: Executor,
private val chainsConfig: ChainsConfig
) : ApplicationRunner {
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
@@ -95,11 +97,22 @@ open class ConfiguredUpstreams(
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
val upstream = when (BlockchainType.from(chain)) {
BlockchainType.EVM_POW -> {
buildEthereumUpstream(up.nodeId, up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
buildEthereumUpstream(
up.nodeId,
up.cast(UpstreamsConfig.EthereumConnection::class.java),
chain,
options,
chainsConfig.resolve(chain)
)
}
BlockchainType.BITCOIN -> {
buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options)
buildBitcoinUpstream(
up.cast(UpstreamsConfig.BitcoinConnection::class.java),
chain,
options,
chainsConfig.resolve(chain)
)
}
BlockchainType.EVM_POS -> {
@@ -107,7 +120,8 @@ open class ConfiguredUpstreams(
up.nodeId,
up.cast(UpstreamsConfig.EthereumPosConnection::class.java),
chain,
options
options,
chainsConfig.resolve(chain)
)
}
}
@@ -167,7 +181,8 @@ open class ConfiguredUpstreams(
nodeId: Int?,
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumPosConnection>,
chain: Chain,
options: UpstreamsConfig.Options
options: UpstreamsConfig.Options,
chainConf: ChainsConfig.ChainConfig
): Upstream? {
val conn = config.connection!!
val execution = conn.execution
@@ -200,7 +215,8 @@ open class ConfiguredUpstreams(
options, config.role,
methods,
QuorumForLabels.QuorumItem(1, config.labels),
connectorFactory
connectorFactory,
chainConf
)
upstream.start()
return upstream
@@ -209,7 +225,8 @@ open class ConfiguredUpstreams(
private fun buildBitcoinUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
chain: Chain,
options: UpstreamsConfig.Options
options: UpstreamsConfig.Options,
chainConf: ChainsConfig.ChainConfig
): Upstream? {
val conn = config.connection!!
val httpFactory = buildHttpFactory(conn)
@@ -242,7 +259,7 @@ open class ConfiguredUpstreams(
chain, directApi, head,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods, esplora
methods, esplora, chainConf
)
upstream.start()
return upstream
@@ -252,7 +269,8 @@ open class ConfiguredUpstreams(
nodeId: Int?,
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
chain: Chain,
options: UpstreamsConfig.Options
options: UpstreamsConfig.Options,
chainConf: ChainsConfig.ChainConfig
): EthereumRpcUpstream? {
val conn = config.connection!!
@@ -279,7 +297,8 @@ open class ConfiguredUpstreams(
options, config.role,
methods,
QuorumForLabels.QuorumItem(1, config.labels),
connectorFactory
connectorFactory,
chainConf
)
upstream.start()
return upstream
@@ -309,7 +328,8 @@ open class ConfiguredUpstreams(
endpoint.upstreamRating,
config.labels,
grpcUpstreamsScheduler,
channelExecutor
channelExecutor,
chainsConfig
).apply {
timeout = options.timeout
}

View File

@@ -17,6 +17,7 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -33,36 +34,20 @@ abstract class DefaultUpstream(
private val options: UpstreamsConfig.Options,
private val role: UpstreamsConfig.UpstreamRole,
private val targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?
node: QuorumForLabels.QuorumItem?,
private val chainConfig: ChainsConfig.ChainConfig
) : Upstream {
constructor(
id: String,
hash: Byte,
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?
) :
this(
id,
hash,
Long.MAX_VALUE,
UpstreamAvailability.UNAVAILABLE,
options,
role,
targets,
QuorumForLabels.QuorumItem.empty()
)
constructor(
id: String,
hash: Byte,
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
node: QuorumForLabels.QuorumItem?
node: QuorumForLabels.QuorumItem?,
chainConfig: ChainsConfig.ChainConfig
) :
this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node)
this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node, chainConfig)
companion object {
private val log = LoggerFactory.getLogger(DefaultUpstream::class.java)
@@ -111,8 +96,8 @@ abstract class DefaultUpstream(
}
return if (proposed == UpstreamAvailability.OK) {
when {
lag > 6 -> UpstreamAvailability.SYNCING
lag > 1 -> UpstreamAvailability.LAGGING
lag > chainConfig.syncingLagSize -> UpstreamAvailability.SYNCING
lag > chainConfig.laggingLagSize -> UpstreamAvailability.LAGGING
else -> proposed
}
} else proposed

View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
@@ -37,8 +38,9 @@ open class BitcoinRpcUpstream(
role: UpstreamsConfig.UpstreamRole,
node: QuorumForLabels.QuorumItem,
callMethods: CallMethods,
esploraClient: EsploraClient? = null
) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient), Lifecycle {
esploraClient: EsploraClient? = null,
chainConfig: ChainsConfig.ChainConfig
) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient, chainConfig), Lifecycle {
companion object {
private val log = LoggerFactory.getLogger(BitcoinRpcUpstream::class.java)

View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.DefaultUpstream
@@ -30,15 +31,17 @@ abstract class BitcoinUpstream(
role: UpstreamsConfig.UpstreamRole,
callMethods: CallMethods,
node: QuorumForLabels.QuorumItem,
val esploraClient: EsploraClient? = null
) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node) {
val esploraClient: EsploraClient? = null,
private val chainConfig: ChainsConfig.ChainConfig
) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node, chainConfig) {
constructor(
id: String,
chain: Chain,
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole
) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty())
role: UpstreamsConfig.UpstreamRole,
chainConfig: ChainsConfig.ChainConfig
) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty(), null, chainConfig)
companion object {
private val log = LoggerFactory.getLogger(BitcoinUpstream::class.java)

View File

@@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
@@ -40,8 +41,9 @@ open class EthereumRpcUpstream(
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory
) : EthereumUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled {
connectorFactory: ConnectorFactory,
chainConfig: ChainsConfig.ChainConfig
) : EthereumUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java)
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain)

View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
@@ -28,8 +29,9 @@ abstract class EthereumUpstream(
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?
) : DefaultUpstream(id, hash, options, role, targets, node) {
private val node: QuorumForLabels.QuorumItem?,
chainConfig: ChainsConfig.ChainConfig
) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) {
private val capabilities = if (options.providesBalance != false) {
setOf(Capability.RPC, Capability.BALANCE)

View File

@@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
@@ -39,9 +40,10 @@ open class EthereumPosRpcUpstream(
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory
) : EthereumPosUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled {
node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory,
chainConfig: ChainsConfig.ChainConfig
) : EthereumPosUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumPosRpcUpstream::class.java)
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain)

View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
@@ -28,8 +29,9 @@ abstract class EthereumPosUpstream(
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?
) : DefaultUpstream(id, hash, options, role, targets, node) {
private val node: QuorumForLabels.QuorumItem?,
chainConfig: ChainsConfig.ChainConfig
) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) {
private val capabilities = if (options.providesBalance != false) {
setOf(Capability.RPC, Capability.BALANCE)

View File

@@ -19,6 +19,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
@@ -51,12 +52,14 @@ class BitcoinGrpcUpstream(
chain: Chain,
val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val client: JsonRpcGrpcClient,
overrideLabels: UpstreamsConfig.Labels?
overrideLabels: UpstreamsConfig.Labels?,
chainConfig: ChainsConfig.ChainConfig
) : BitcoinUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
chain,
UpstreamsConfig.Options.getDefaults(),
role
role,
chainConfig
),
GrpcUpstream,
Lifecycle {

View File

@@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc.ReactorBlockchainStub
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
@@ -57,14 +58,16 @@ open class EthereumGrpcUpstream(
private val chain: Chain,
private val remote: ReactorBlockchainStub,
private val client: JsonRpcGrpcClient,
overrideLabels: UpstreamsConfig.Labels?
overrideLabels: UpstreamsConfig.Labels?,
chainConfig: ChainsConfig.ChainConfig
) : EthereumUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
hash,
UpstreamsConfig.Options.getDefaults(),
role,
null,
null
null,
chainConfig
),
GrpcUpstream,
Lifecycle {

View File

@@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
@@ -58,13 +59,16 @@ open class EthereumPosGrpcUpstream(
private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
client: JsonRpcGrpcClient,
nodeRating: Int,
overrideLabels: UpstreamsConfig.Labels?
overrideLabels: UpstreamsConfig.Labels?,
chainConfig: ChainsConfig.ChainConfig
) : EthereumPosUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
hash,
UpstreamsConfig.Options.getDefaults(),
role,
null, null
null,
null,
chainConfig
),
GrpcUpstream,
Lifecycle {

View File

@@ -24,6 +24,7 @@ import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.DefaultUpstream
@@ -64,7 +65,8 @@ class GrpcUpstreams(
private val nodeRating: Int,
private val labels: UpstreamsConfig.Labels,
private val chainStatusScheduler: Scheduler,
private val grpcExecutor: Executor
private val grpcExecutor: Executor,
private val chainsConfig: ChainsConfig
) {
private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java)
@@ -185,13 +187,13 @@ class GrpcUpstreams(
private val creators: Map<BlockchainType, (chain: Chain, client: JsonRpcGrpcClient) -> DefaultUpstream> = mapOf(
BlockchainType.EVM_POW to { chain, rpcClient ->
EthereumGrpcUpstream(id, hash, role, chain, client, rpcClient, labels)
EthereumGrpcUpstream(id, hash, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain))
},
BlockchainType.EVM_POS to { chain, rpcClient ->
EthereumPosGrpcUpstream(id, hash, role, chain, client, rpcClient, nodeRating, labels)
EthereumPosGrpcUpstream(id, hash, role, chain, client, rpcClient, nodeRating, labels, chainsConfig.resolve(chain))
},
BlockchainType.BITCOIN to { chain, rpcClient ->
BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels)
BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain))
}
)