Ignore blocks from syncing nodes (#243)

2023-07-04 14:17:54 +04:00
parent 9b8e90fa2e
commit 634f704c2e
26 changed files with 191 additions and 227 deletions
--- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
@@ -42,8 +42,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.ZMQServer
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator
-import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosRpcUpstream
-import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
+import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsConnectionFactory
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsConnectionPoolFactory
 import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory
@@ -227,7 +226,7 @@ open class ConfiguredUpstreams(
            if (it.connectorMode == RPC_REQUESTS_WITH_MIXED_HEAD.name) it.rpc?.url ?: it.ws?.url else it.ws?.url ?: it.rpc?.url
        }
        val hash = getHash(nodeId, hashUrl!!)
-        val upstream = EthereumPosRpcUpstream(
+        val upstream = EthereumLikeRpcUpstream(
            config.id!!,
            hash,
            chain,
@@ -236,7 +235,8 @@ open class ConfiguredUpstreams(
            methods,
            QuorumForLabels.QuorumItem(1, config.labels),
            connectorFactory,
-            chainConf
+            chainConf,
+            true
        )
        upstream.start()
        return upstream
@@ -291,7 +291,7 @@ open class ConfiguredUpstreams(
        chain: Chain,
        options: UpstreamsConfig.Options,
        chainConf: ChainsConfig.ChainConfig
-    ): EthereumRpcUpstream? {
+    ): Upstream? {
        val conn = config.connection!!

        val urls = ArrayList<URI>()
@@ -310,7 +310,7 @@ open class ConfiguredUpstreams(
        }

        val hashUrl = if (conn.connectorMode == RPC_REQUESTS_WITH_MIXED_HEAD.name) conn.rpc?.url ?: conn.ws?.url else conn.ws?.url ?: conn.rpc?.url
-        val upstream = EthereumRpcUpstream(
+        val upstream = EthereumLikeRpcUpstream(
            config.id!!,
            getHash(nodeId, hashUrl!!),
            chain,
@@ -318,7 +318,8 @@ open class ConfiguredUpstreams(
            methods,
            QuorumForLabels.QuorumItem(1, config.labels),
            connectorFactory,
-            chainConf
+            chainConf,
+            false
        )
        upstream.start()
        return upstream
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt
@@ -149,7 +149,12 @@ abstract class AbstractHead @JvmOverloads constructor(
        future = null
    }

-    override fun onNoHeadUpdates() {
+    protected open fun onNoHeadUpdates() {
+        // NOOP
+    }
+
+    override fun onSyncingNode(isSyncing: Boolean) {
+        // NOOP
    }

    override fun start() {
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt
@@ -37,6 +37,6 @@ class EmptyHead : Head {
    override fun stop() {
    }

-    override fun onNoHeadUpdates() {
+    override fun onSyncingNode(isSyncing: Boolean) {
    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt
@@ -42,5 +42,5 @@ interface Head {

    fun stop()

-    fun onNoHeadUpdates()
+    fun onSyncingNode(isSyncing: Boolean)
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt
@@ -49,7 +49,7 @@ open class ERC20Balance {
        apis.request(1)
        return Flux.from(apis)
            .flatMap {
-                getBalance(it.cast(EthereumPosRpcUpstream::class.java), token, address)
+                getBalance(it.cast(EthereumLikeRpcUpstream::class.java), token, address)
            }
            .doOnNext {
                apis.resolve()
@@ -57,7 +57,7 @@ open class ERC20Balance {
            .next()
    }

-    open fun getBalance(upstream: EthereumPosRpcUpstream, token: ERC20Token, address: Address): Mono<BigInteger> {
+    open fun getBalance(upstream: EthereumLikeRpcUpstream, token: ERC20Token, address: Address): Mono<BigInteger> {
        return upstream
            .getIngressReader()
            .read(prepareEthCall(token, address, upstream.getHead()))
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt
@@ -32,19 +32,20 @@ import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
 import org.springframework.context.Lifecycle
 import reactor.core.Disposable

-open class EthereumRpcUpstream(
+open class EthereumLikeRpcUpstream(
    id: String,
    hash: Byte,
    val chain: Chain,
    options: UpstreamsConfig.Options,
    role: UpstreamsConfig.UpstreamRole,
    targets: CallMethods?,
-    private val node: QuorumForLabels.QuorumItem?,
+    node: QuorumForLabels.QuorumItem?,
    connectorFactory: ConnectorFactory,
-    chainConfig: ChainsConfig.ChainConfig
-) : EthereumUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
+    chainConfig: ChainsConfig.ChainConfig,
+    skipEnhance: Boolean
+) : EthereumLikeUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
    private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
-    private val connector: EthereumConnector = connectorFactory.create(this, validator, chain, false)
+    private val connector: EthereumConnector = connectorFactory.create(this, validator, chain, skipEnhance)

    private var validatorSubscription: Disposable? = null

--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeUpstream.kt
@@ -23,7 +23,7 @@ import io.emeraldpay.dshackle.upstream.Capability
 import io.emeraldpay.dshackle.upstream.DefaultUpstream
 import io.emeraldpay.dshackle.upstream.calls.CallMethods

-abstract class EthereumUpstream(
+abstract class EthereumLikeUpstream(
    id: String,
    hash: Byte,
    options: UpstreamsConfig.Options,
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt
@@ -47,7 +47,7 @@ import reactor.core.scheduler.Scheduler
@Suppress("UNCHECKED_CAST")
 open class EthereumMultistream(
    chain: Chain,
-    val upstreams: MutableList<EthereumUpstream>,
+    val upstreams: MutableList<EthereumLikeUpstream>,
    caches: Caches,
    private val headScheduler: Scheduler,
    tracer: Tracer,
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt
@@ -35,12 +35,14 @@ class EthereumRpcHead(
 ) : DefaultEthereumHead(upstreamId, forkChoice, blockValidator, headScheduler), Lifecycle {

    private var refreshSubscription: Disposable? = null
+    private var isSyncing = false

    override fun start() {
        super.start()
        refreshSubscription?.dispose()
        val base = Flux.interval(interval)
            .publishOn(headScheduler)
+            .filter { !isSyncing }
            .flatMap {
                getLatestBlock(api)
            }
@@ -51,6 +53,10 @@ class EthereumRpcHead(
        return refreshSubscription != null
    }

+    override fun onSyncingNode(isSyncing: Boolean) {
+        this.isSyncing = isSyncing
+    }
+
    override fun stop() {
        super.stop()
        refreshSubscription?.dispose()
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt
@@ -65,8 +65,7 @@ open class EthereumUpstreamValidator(
        if (!options.validateSyncing) {
            return Mono.just(UpstreamAvailability.OK)
        }
-        return upstream
-            .getIngressReader()
+        return upstream.getIngressReader()
            .read(JsonRpcRequest("eth_syncing", listOf()))
            .flatMap(JsonRpcResponse::requireResult)
            .map { objectMapper.readValue(it, SyncingJson::class.java) }
@@ -75,8 +74,10 @@ open class EthereumUpstreamValidator(
                Mono.fromCallable { log.warn("No response for eth_syncing from ${upstream.getId()}") }
                    .then(Mono.error(TimeoutException("Validation timeout for Syncing")))
            )
-            .map { value ->
-                if (value.isSyncing) {
+            .map {
+                val isSyncing = it.isSyncing
+                upstream.getHead().onSyncingNode(isSyncing)
+                if (isSyncing) {
                    UpstreamAvailability.SYNCING
                } else {
                    UpstreamAvailability.OK
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt
@@ -50,6 +50,7 @@ class EthereumWsHead(
    private var connectionId: String? = null
    private var subscribed = false
    private var connected = false
+    private var isSyncing = false

    private var subscription: Disposable? = null
    private val noHeadUpdatesSink = Sinks.many().multicast().directBestEffort<Boolean>()
@@ -78,6 +79,13 @@ class EthereumWsHead(
        noHeadUpdatesSink.tryEmitNext(true)
    }

+    override fun onSyncingNode(isSyncing: Boolean) {
+        if (isSyncing && !this.isSyncing) {
+            cancelSub()
+        }
+        this.isSyncing = isSyncing
+    }
+
    fun listenNewHeads(): Flux<BlockContainer> {
        return subscribe()
            .map {
@@ -131,8 +139,7 @@ class EthereumWsHead(

    override fun stop() {
        super.stop()
-        subscription?.dispose()
-        subscription = null
+        cancelSub()
        noHeadUpdatesSink.tryEmitComplete()
    }

@@ -168,10 +175,16 @@ class EthereumWsHead(
            noHeadUpdatesSink.asFlux(),
            connectionStates,
        ).subscribeOn(wsConnectionResubscribeScheduler)
-            .filter { it && !subscribed && connected }
+            .filter { it && !subscribed && connected && !isSyncing }
            .subscribe {
                log.warn("Restart ws head, upstreamId: $upstreamId")
                start()
            }
    }
+
+    private fun cancelSub() {
+        subscription?.dispose()
+        subscription = null
+        subscribed = false
+    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt
@@ -45,7 +45,7 @@ import reactor.core.scheduler.Scheduler
@Suppress("UNCHECKED_CAST")
 open class EthereumPosMultiStream(
    chain: Chain,
-    val upstreams: MutableList<EthereumPosUpstream>,
+    val upstreams: MutableList<EthereumLikeUpstream>,
    caches: Caches,
    private val headScheduler: Scheduler,
    tracer: Tracer
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt
@@ -1,103 +0,0 @@
-/**
- * Copyright (c) 2020 EmeraldPay, Inc
- * Copyright (c) 2019 ETCDEV GmbH
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.emeraldpay.dshackle.upstream.ethereum
-
-import io.emeraldpay.dshackle.Chain
-import io.emeraldpay.dshackle.cache.Caches
-import io.emeraldpay.dshackle.cache.CachesEnabled
-import io.emeraldpay.dshackle.config.ChainsConfig
-import io.emeraldpay.dshackle.config.UpstreamsConfig
-import io.emeraldpay.dshackle.reader.JsonRpcReader
-import io.emeraldpay.dshackle.startup.QuorumForLabels
-import io.emeraldpay.dshackle.upstream.Head
-import io.emeraldpay.dshackle.upstream.Lifecycle
-import io.emeraldpay.dshackle.upstream.Upstream
-import io.emeraldpay.dshackle.upstream.UpstreamAvailability
-import io.emeraldpay.dshackle.upstream.calls.CallMethods
-import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
-import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
-import reactor.core.Disposable
-
-open class EthereumPosRpcUpstream(
-    id: String,
-    hash: Byte,
-    val chain: Chain,
-    options: UpstreamsConfig.Options,
-    role: UpstreamsConfig.UpstreamRole,
-    targets: CallMethods?,
-    node: QuorumForLabels.QuorumItem?,
-    connectorFactory: ConnectorFactory,
-    chainConfig: ChainsConfig.ChainConfig
-) : EthereumPosUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled {
-    private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
-    private val connector: EthereumConnector = connectorFactory.create(this, validator, chain, true)
-
-    private var validatorSubscription: Disposable? = null
-
-    override fun setCaches(caches: Caches) {
-        if (connector is CachesEnabled) {
-            connector.setCaches(caches)
-        }
-    }
-
-    override fun start() {
-        log.info("Configured for ${chain.chainName}")
-        connector.start()
-        if (getOptions().disableValidation) {
-            log.warn("Disable validation for upstream ${this.getId()}")
-            this.setLag(0)
-            this.setStatus(UpstreamAvailability.OK)
-        } else {
-            log.debug("Start validation for upstream ${this.getId()}")
-            validatorSubscription = validator.start()
-                .subscribe(this::setStatus)
-        }
-    }
-    override fun getHead(): Head {
-        return connector.getHead()
-    }
-
-    override fun stop() {
-        validatorSubscription?.dispose()
-        validatorSubscription = null
-        connector.stop()
-    }
-
-    override fun isRunning(): Boolean {
-        return connector.isRunning()
-    }
-
-    override fun getIngressReader(): JsonRpcReader {
-        return connector.getIngressReader()
-    }
-
-    override fun isGrpc(): Boolean {
-        return false
-    }
-
-    @Suppress("UNCHECKED_CAST")
-    override fun <T : Upstream> cast(selfType: Class<T>): T {
-        if (!selfType.isAssignableFrom(this.javaClass)) {
-            throw ClassCastException("Cannot cast ${this.javaClass} to $selfType")
-        }
-        return this as T
-    }
-
-    override fun getIngressSubscription(): EthereumIngressSubscription {
-        return connector.getIngressSubscription()
-    }
-}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt
@@ -1,51 +0,0 @@
-/**
- * Copyright (c) 2020 EmeraldPay, Inc
- * Copyright (c) 2019 ETCDEV GmbH
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.emeraldpay.dshackle.upstream.ethereum
-
-import io.emeraldpay.dshackle.config.ChainsConfig
-import io.emeraldpay.dshackle.config.UpstreamsConfig
-import io.emeraldpay.dshackle.startup.QuorumForLabels
-import io.emeraldpay.dshackle.upstream.Capability
-import io.emeraldpay.dshackle.upstream.DefaultUpstream
-import io.emeraldpay.dshackle.upstream.calls.CallMethods
-
-abstract class EthereumPosUpstream(
-    id: String,
-    hash: Byte,
-    options: UpstreamsConfig.Options,
-    role: UpstreamsConfig.UpstreamRole,
-    targets: CallMethods?,
-    private val node: QuorumForLabels.QuorumItem?,
-    chainConfig: ChainsConfig.ChainConfig
-) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) {
-
-    private val capabilities = if (options.providesBalance != false) {
-        setOf(Capability.RPC, Capability.BALANCE)
-    } else {
-        setOf(Capability.RPC)
-    }
-
-    override fun getCapabilities(): Set<Capability> {
-        return capabilities
-    }
-
-    override fun getLabels(): Collection<UpstreamsConfig.Labels> {
-        return node?.let { listOf(it.labels) } ?: emptyList()
-    }
-
-    abstract fun getIngressSubscription(): EthereumIngressSubscription
-}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt
@@ -34,7 +34,7 @@ import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.UpstreamAvailability
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription
-import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
+import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeUpstream
 import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumDshackleIngressSubscription
 import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
@@ -62,7 +62,7 @@ open class EthereumGrpcUpstream(
    overrideLabels: UpstreamsConfig.Labels?,
    chainConfig: ChainsConfig.ChainConfig,
    headScheduler: Scheduler,
-) : EthereumUpstream(
+) : EthereumLikeUpstream(
    "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
    hash,
    UpstreamsConfig.PartialOptions.getDefaults().buildOptions(),
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt
@@ -33,7 +33,7 @@ import io.emeraldpay.dshackle.upstream.Lifecycle
 import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription
-import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosUpstream
+import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeUpstream
 import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumDshackleIngressSubscription
 import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
@@ -56,7 +56,7 @@ open class EthereumPosGrpcUpstream(
    overrideLabels: UpstreamsConfig.Labels?,
    chainConfig: ChainsConfig.ChainConfig,
    headScheduler: Scheduler,
-) : EthereumPosUpstream(
+) : EthereumLikeUpstream(
    "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
    hash,
    UpstreamsConfig.PartialOptions.getDefaults().buildOptions(),