StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better track ws capabiltity for upstream through head liveness check (#277)

Browse Source
This commit is contained in:
Vyacheslav
2023-08-10 11:45:47 +03:00
committed by GitHub
parent dfbdb0618f
commit 621d8671be
23 changed files with 249 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfig.kt
View File

@@ -1,6 +1,7 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import java.time.Duration
data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val currentDefault: RawChainConfig?) {
companion object {
@@ -9,6 +10,7 @@ data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val curr
}
data class RawChainConfig(
var expectedBlockTime: Duration? = null,
var syncingLagSize: Int? = null,
var laggingLagSize: Int? = null,
var callLimitContract: String? = null,
@@ -25,6 +27,7 @@ data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val curr
}
data class ChainConfig(
val expectedBlockTime: Duration,
val syncingLagSize: Int,
val laggingLagSize: Int,
val options: UpstreamsConfig.PartialOptions,
@@ -32,7 +35,7 @@ data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val curr
) {
companion object {
@JvmStatic
fun default() = ChainConfig(6, 1, UpstreamsConfig.PartialOptions(), null)
fun default() = ChainConfig(Duration.ofSeconds(12), 6, 1, UpstreamsConfig.PartialOptions(), null)
}
}
@@ -45,7 +48,8 @@ data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val curr
laggingLagSize = raw.laggingLagSize ?: default.laggingLagSize ?: panic(),
syncingLagSize = raw.syncingLagSize ?: default.syncingLagSize ?: panic(),
options = options,
callLimitContract = raw.callLimitContract
callLimitContract = raw.callLimitContract,
expectedBlockTime = raw.expectedBlockTime ?: default.expectedBlockTime ?: panic(),
)
}
@@ -61,7 +65,8 @@ data class ChainsConfig(private val chains: Map<Chain, RawChainConfig>, val curr
syncingLagSize = patch?.syncingLagSize ?: current.syncingLagSize,
laggingLagSize = patch?.laggingLagSize ?: current.laggingLagSize,
options = patch?.options ?: current.options,
callLimitContract = patch?.callLimitContract ?: current.callLimitContract
callLimitContract = patch?.callLimitContract ?: current.callLimitContract,
expectedBlockTime = patch?.expectedBlockTime ?: current.expectedBlockTime
)
private fun merge(

3
src/main/kotlin/io/emeraldpay/dshackle/config/ChainsConfigReader.kt
View File

@@ -53,6 +53,9 @@ class ChainsConfigReader(
rawConfig.laggingLagSize = it
}
}
getValueAsDuration(node, "expected-block-time")?.let {
rawConfig.expectedBlockTime = it
}
getValueAsString(node, "call-validate-contract")?.let {
rawConfig.callLimitContract = it
}

4
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt
View File

@@ -30,6 +30,7 @@ open class UpstreamsConfig {
var upstreams: MutableList<Upstream<*>> = ArrayList<Upstream<*>>()
data class Options(
val disableUpstreamValidation: Boolean,
val disableValidation: Boolean,
val validationInterval: Int,
val timeout: Duration,
@@ -43,6 +44,7 @@ open class UpstreamsConfig {
open class PartialOptions {
var disableValidation: Boolean? = null
var disableUpstreamValidation: Boolean? = null
var validationInterval: Int? = null
set(value) {
require(value == null || value > 0) {
@@ -78,11 +80,13 @@ open class UpstreamsConfig {
copy.validateCalllimit = firstNonNull(overwrites.validateCalllimit, this.validateCalllimit)
copy.timeout = firstNonNull(overwrites.timeout, this.timeout)
copy.validateChain = firstNonNull(overwrites.validateChain, this.validateChain)
copy.disableUpstreamValidation = firstNonNull(overwrites.disableUpstreamValidation, this.disableUpstreamValidation)
return copy
}
fun buildOptions(): Options =
Options(
firstNonNull(this.disableUpstreamValidation, false)!!,
firstNonNull(this.disableValidation, false)!!,
firstNonNull(this.validationInterval, 30)!!,
firstNonNull(this.timeout, Defaults.timeout)!!,

12
src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt
View File

@@ -24,6 +24,8 @@ import org.yaml.snakeyaml.nodes.ScalarNode
import java.io.InputStream
import java.io.InputStreamReader
import java.util.Locale
import kotlin.time.Duration
import kotlin.time.toJavaDuration
abstract class YamlConfigReader<T> : ConfigReader<T> {
private val envVariables = EnvVariables()
@@ -92,6 +94,16 @@ abstract class YamlConfigReader<T> : ConfigReader<T> {
}?.let(envVariables::postProcess)
}
protected fun getValueAsDuration(mappingNode: MappingNode?, key: String): java.time.Duration? {
return getValue(mappingNode, key)?.let {
return@let if (it.isPlain) {
Duration.parse(it.value).toJavaDuration()
} else {
null
}
}
}
protected fun getValueAsInt(mappingNode: MappingNode?, key: String): Int? {
return getValue(mappingNode, key)?.let {
return@let if (it.isPlain) {

18
src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
View File

@@ -215,7 +215,8 @@ open class ConfiguredUpstreams(
chain,
urls,
NoChoiceWithPriorityForkChoice(conn.upstreamRating, config.id!!),
BlockValidator.ALWAYS_VALID
BlockValidator.ALWAYS_VALID,
chainConf
)
val methods = buildMethods(config, chain)
if (connectorFactory == null) {
@@ -236,7 +237,8 @@ open class ConfiguredUpstreams(
QuorumForLabels.QuorumItem(1, config.labels),
connectorFactory,
chainConf,
true
true,
eventPublisher
)
upstream.start()
if (!upstream.isRunning) return null
@@ -304,7 +306,8 @@ open class ConfiguredUpstreams(
chain,
urls,
MostWorkForkChoice(),
EthereumBlockValidator()
EthereumBlockValidator(),
chainConf
)
if (connectorFactory == null) {
return null
@@ -320,7 +323,8 @@ open class ConfiguredUpstreams(
QuorumForLabels.QuorumItem(1, config.labels),
connectorFactory,
chainConf,
false
false,
eventPublisher
)
upstream.start()
return upstream
@@ -413,7 +417,8 @@ open class ConfiguredUpstreams(
chain: Chain,
urls: ArrayList<URI>,
forkChoice: ForkChoice,
blockValidator: BlockValidator
blockValidator: BlockValidator,
chainsConf: ChainsConfig.ChainConfig
): EthereumConnectorFactory? {
val wsFactoryApi = buildWsFactory(id, chain, conn, urls)
val httpFactory = buildHttpFactory(conn, urls)
@@ -426,7 +431,8 @@ open class ConfiguredUpstreams(
forkChoice,
blockValidator,
wsConnectionResubscribeScheduler,
headScheduler
headScheduler,
chainsConf.expectedBlockTime
)
if (!connectorFactory.isValid()) {
log.warn("Upstream configuration is invalid (probably no http endpoint)")

26
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt
View File

@@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
@@ -30,10 +31,11 @@ import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumLabelsDetector
import org.springframework.context.ApplicationEventPublisher
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import java.util.concurrent.atomic.AtomicBoolean
open class EthereumLikeRpcUpstream(
id: String,
@@ -45,21 +47,21 @@ open class EthereumLikeRpcUpstream(
private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory,
chainConfig: ChainsConfig.ChainConfig,
skipEnhance: Boolean
skipEnhance: Boolean,
private val eventPublisher: ApplicationEventPublisher?
) : EthereumLikeUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(chain, this, getOptions(), chainConfig.callLimitContract)
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain, skipEnhance)
protected val connector: EthereumConnector = connectorFactory.create(this, validator, chain, skipEnhance)
private val labelsDetector = EthereumLabelsDetector(this.getIngressReader())
private var hasLiveSubscriptionHead: AtomicBoolean = AtomicBoolean(false)
private var validatorSubscription: Disposable? = null
override fun getCapabilities(): Set<Capability> {
return when (connector.getConnectorMode()) {
EthereumConnectorFactory.ConnectorMode.WS_ONLY,
EthereumConnectorFactory.ConnectorMode.RPC_REQUESTS_WITH_MIXED_HEAD,
EthereumConnectorFactory.ConnectorMode.RPC_REQUESTS_WITH_WS_HEAD ->
setOf(Capability.RPC, Capability.BALANCE, Capability.WS_HEAD)
EthereumConnectorFactory.ConnectorMode.RPC_ONLY -> setOf(Capability.RPC, Capability.BALANCE)
return if (hasLiveSubscriptionHead.get()) {
setOf(Capability.RPC, Capability.BALANCE, Capability.WS_HEAD)
} else {
setOf(Capability.RPC, Capability.BALANCE)
}
}
@@ -72,7 +74,7 @@ open class EthereumLikeRpcUpstream(
override fun start() {
log.info("Configured for ${chain.chainName}")
connector.start()
if (!validator.validateUpstreamSettings()) {
if (!getOptions().disableUpstreamValidation && !validator.validateUpstreamSettings()) {
connector.stop()
log.warn("Upstream ${getId()} couldn't start, invalid upstream settings")
return
@@ -86,6 +88,10 @@ open class EthereumLikeRpcUpstream(
validatorSubscription = validator.start()
.subscribe(this::setStatus)
}
connector.hasLiveSubscriptionHead().subscribe {
hasLiveSubscriptionHead.set(it)
eventPublisher?.publishEvent(UpstreamChangeEvent(chain, this, UpstreamChangeEvent.ChangeType.UPDATED))
}
labelsDetector.detectLabels()
.toStream()
.forEach {

41
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/HeadLivenessValidator.kt Normal file
View File

@@ -0,0 +1,41 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.Head
import reactor.core.publisher.Flux
import reactor.core.scheduler.Scheduler
import java.time.Duration
class HeadLivenessValidator(
val head: Head,
val expectedBlockTime: Duration,
val scheduler: Scheduler
) {
companion object {
const val CHECKED_BLOCKS_UNTIL_LIVE = 3
}
fun getFlux(): Flux<Boolean> {
// first we have moving window of 2 blocks and check that they are consecutive ones
return head.getFlux().buffer(2, 1).map {
it.last().height - it.first().height == 1L
}.scan(Pair(0, true)) { acc, value ->
// then we accumulate consecutive true events, false resets counter
if (value) {
Pair(acc.first + 1, true)
} else {
Pair(0, false)
}
}.flatMap { (count, value) ->
// we emit when we have false or checked CHECKED_BLOCKS_UNTIL_LIVE blocks
// CHECKED_BLOCKS_UNTIL_LIVE blocks == (CHECKED_BLOCKS_UNTIL_LIVE - 1) consecutive true
when {
count == (CHECKED_BLOCKS_UNTIL_LIVE - 1) -> Flux.just(true)
!value -> Flux.just(false)
else -> Flux.empty()
}
// finally, we timeout after we waited for double the time we needed to emit those blocks
}.timeout(expectedBlockTime.multipliedBy(CHECKED_BLOCKS_UNTIL_LIVE.toLong() * 2), Flux.just(false))
.distinctUntilChanged().subscribeOn(scheduler)
}
}

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnector.kt
View File

@@ -4,11 +4,12 @@ import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription
import reactor.core.publisher.Flux
interface EthereumConnector : Lifecycle {
fun getHead(): Head
fun getConnectorMode(): EthereumConnectorFactory.ConnectorMode
fun hasLiveSubscriptionHead(): Flux<Boolean>
fun getIngressReader(): JsonRpcReader

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt
View File

@@ -12,6 +12,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFact
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory.ConnectorMode.WS_ONLY
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import reactor.core.scheduler.Scheduler
import java.time.Duration
open class EthereumConnectorFactory(
private val connectorType: ConnectorMode,
@@ -20,7 +21,8 @@ open class EthereumConnectorFactory(
private val forkChoice: ForkChoice,
private val blockValidator: BlockValidator,
private val wsConnectionResubscribeScheduler: Scheduler,
private val headScheduler: Scheduler
private val headScheduler: Scheduler,
private val expectedBlockTime: Duration
) : ConnectorFactory {
override fun isValid(): Boolean {
@@ -57,7 +59,8 @@ open class EthereumConnectorFactory(
blockValidator,
skipEnhance,
wsConnectionResubscribeScheduler,
headScheduler
headScheduler,
expectedBlockTime
)
}
if (httpFactory == null) {
@@ -72,7 +75,8 @@ open class EthereumConnectorFactory(
blockValidator,
skipEnhance,
wsConnectionResubscribeScheduler,
headScheduler
headScheduler,
expectedBlockTime
)
}

13
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt
View File

@@ -11,6 +11,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsConnectionPoolFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead
import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidator
import io.emeraldpay.dshackle.upstream.ethereum.NoEthereumIngressSubscription
import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPool
import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptionsImpl
@@ -22,11 +23,12 @@ import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFact
import io.emeraldpay.dshackle.upstream.forkchoice.AlwaysForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.scheduler.Scheduler
import java.time.Duration
class EthereumRpcConnector(
private val connectorType: ConnectorMode,
connectorType: ConnectorMode,
private val directReader: JsonRpcReader,
wsFactory: EthereumWsConnectionPoolFactory?,
id: String,
@@ -34,16 +36,20 @@ class EthereumRpcConnector(
blockValidator: BlockValidator,
skipEnhance: Boolean,
wsConnectionResubscribeScheduler: Scheduler,
headScheduler: Scheduler
headScheduler: Scheduler,
expectedBlockTime: Duration
) : EthereumConnector, CachesEnabled {
private val pool: WsConnectionPool?
private val head: Head
private val liveness: HeadLivenessValidator
companion object {
private val log = LoggerFactory.getLogger(EthereumRpcConnector::class.java)
}
override fun getConnectorMode() = connectorType
override fun hasLiveSubscriptionHead(): Flux<Boolean> {
return liveness.getFlux()
}
init {
pool = wsFactory?.create(null)
@@ -93,6 +99,7 @@ class EthereumRpcConnector(
)
}
}
liveness = HeadLivenessValidator(head, expectedBlockTime, headScheduler)
}
override fun setCaches(caches: Caches) {

14
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt
View File

@@ -7,12 +7,15 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsConnectionPoolFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead
import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidator
import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPool
import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptionsImpl
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumWsIngressSubscription
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient
import reactor.core.publisher.Flux
import reactor.core.scheduler.Scheduler
import java.time.Duration
class EthereumWsConnector(
wsFactory: EthereumWsConnectionPoolFactory,
@@ -21,13 +24,14 @@ class EthereumWsConnector(
blockValidator: BlockValidator,
skipEnhance: Boolean,
wsConnectionResubscribeScheduler: Scheduler,
headScheduler: Scheduler
headScheduler: Scheduler,
expectedBlockTime: Duration
) : EthereumConnector {
private val pool: WsConnectionPool
private val reader: JsonRpcReader
private val head: EthereumWsHead
private val subscriptions: EthereumIngressSubscription
private val liveness: HeadLivenessValidator
init {
pool = wsFactory.create(upstream)
reader = JsonRpcWsClient(pool)
@@ -42,11 +46,13 @@ class EthereumWsConnector(
wsConnectionResubscribeScheduler,
headScheduler
)
liveness = HeadLivenessValidator(head, expectedBlockTime, headScheduler)
subscriptions = EthereumWsIngressSubscription(wsSubscriptions)
}
override fun getConnectorMode() = EthereumConnectorFactory.ConnectorMode.WS_ONLY
override fun hasLiveSubscriptionHead(): Flux<Boolean> {
return liveness.getFlux()
}
override fun start() {
pool.connect()
head.start()

22
src/main/resources/chains.yaml
View File

@@ -2,95 +2,112 @@ version: v1
chain-settings:
default:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
chains:
- id: eth
call-validate-contract: 0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
- id: goerli
call-validate-contract: 0xCD9303A1F6da2a68f465A579a24cc2Ee5AE2192f
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
- id: polygon
expected-block-time: 2.7s
lags:
syncing: 20
lagging: 10
- id: polygon-mumbai
expected-block-time: 2.7s
lags:
syncing: 20
lagging: 10
- id: arbitrum
expected-block-time: 260ms
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: arbitrum-testnet
expected-block-time: 1s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: optimism
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: optimism-testnet
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: arbitrum-nova
expected-block-time: 1s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: polygon-zkevm
expected-block-time: 2.7s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: polygon-zkevm-testnet
expected-block-time: 1m
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: zksync
expected-block-time: 5s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: zksync-testnet
expected-block-time: 5s
options:
disable-validation: true
lags:
syncing: 40
lagging: 20
- id: base
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: base-goerli
expected-block-time: 2s
options:
validate-peers: false
lags:
syncing: 40
lagging: 20
- id: avalanche
expected-block-time: 2s
options:
validate-peers: false
validate-syncing: false
@@ -98,6 +115,7 @@ chain-settings:
syncing: 10
lagging: 5
- id: avalanche-fuji
expected-block-time: 2s
options:
validate-peers: false
validate-syncing: false
@@ -105,24 +123,28 @@ chain-settings:
syncing: 10
lagging: 5
- id: gnosis
expected-block-time: 6s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: gnosis-chiado
expected-block-time: 6s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: fantom
expected-block-time: 3s
options:
validate-peers: false
lags:
syncing: 10
lagging: 5
- id: fantom-testnet
expected-block-time: 1m
options:
validate-peers: false
lags: