From 9b400d8b2a87926a9d5ba355a7f054975818be5c Mon Sep 17 00:00:00 2001 From: terminal Date: Mon, 5 Sep 2022 18:08:33 +0300 Subject: [PATCH 1/3] block validator as s param for default ethereum head --- .../dshackle/startup/ConfiguredUpstreams.kt | 15 ++++++--------- .../upstream/ethereum/DefaultEthereumHead.kt | 6 ++++-- .../dshackle/upstream/ethereum/EthereumRpcHead.kt | 4 +++- .../dshackle/upstream/ethereum/EthereumWsHead.kt | 6 ++++-- .../connectors/EthereumConnectorFactory.kt | 9 ++++++--- .../ethereum/connectors/EthereumRpcConnector.kt | 11 +++++++---- .../ethereum/connectors/EthereumWsConnector.kt | 11 +++++------ .../dshackle/upstream/FilteredApisSpec.groovy | 2 +- .../ethereum/DefaultEthereumHeadSpec.groovy | 3 ++- 9 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index b9bff2e4..ded6ea18 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -20,11 +20,7 @@ import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.CurrentMultistreamHolder -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.HttpRpcFactory -import io.emeraldpay.dshackle.upstream.MergedHead -import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.* import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcHead import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinZMQHead @@ -33,6 +29,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock import io.emeraldpay.dshackle.upstream.bitcoin.ZMQServer import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods +import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosRpcUpstream import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory @@ -159,7 +156,7 @@ open class ConfiguredUpstreams( return null } val urls = ArrayList() - val connectorFactory = buildEthereumConnectorFactory(config.id!!, execution, chain, urls, NoChoiceWithPriorityForkChoice(conn.upstreamRating)) + val connectorFactory = buildEthereumConnectorFactory(config.id!!, execution, chain, urls, NoChoiceWithPriorityForkChoice(conn.upstreamRating), BlockValidator.ALWAYS_VALID) val methods = buildMethods(config, chain) if (connectorFactory == null) { return null @@ -228,7 +225,7 @@ open class ConfiguredUpstreams( val urls = ArrayList() val methods = buildMethods(config, chain) - val connectorFactory = buildEthereumConnectorFactory(config.id!!, conn, chain, urls, MostWorkForkChoice()) + val connectorFactory = buildEthereumConnectorFactory(config.id!!, conn, chain, urls, MostWorkForkChoice(), EthereumBlockValidator()) if (connectorFactory == null) { return null } @@ -297,11 +294,11 @@ open class ConfiguredUpstreams( } } - private fun buildEthereumConnectorFactory(id: String, conn: UpstreamsConfig.EthereumConnection, chain: Chain, urls: ArrayList, forkChoice: ForkChoice): EthereumConnectorFactory? { + private fun buildEthereumConnectorFactory(id: String, conn: UpstreamsConfig.EthereumConnection, chain: Chain, urls: ArrayList, forkChoice: ForkChoice, blockValidator: BlockValidator): EthereumConnectorFactory? { val wsFactoryApi = buildWsFactory(id, chain, conn, urls) val httpFactory = buildHttpFactory(conn, urls) log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}") - val connectorFactory = EthereumConnectorFactory(conn.preferHttp, wsFactoryApi, httpFactory, forkChoice) + val connectorFactory = EthereumConnectorFactory(conn.preferHttp, wsFactoryApi, httpFactory, forkChoice, blockValidator) if (!connectorFactory.isValid()) { log.warn("Upstream configuration is invalid (probably no http endpoint)") return null diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt index 16c5d98b..eee280c0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt @@ -19,6 +19,7 @@ import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.AbstractHead +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest @@ -28,8 +29,9 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Mono open class DefaultEthereumHead( - forkChoice: ForkChoice -) : Head, AbstractHead(forkChoice, EthereumBlockValidator()) { + forkChoice: ForkChoice, + blockValidator: BlockValidator +) : Head, AbstractHead(forkChoice, blockValidator) { companion object { private val log = LoggerFactory.getLogger(DefaultEthereumHead::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt index 104a9bba..bb1fb38d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse @@ -32,8 +33,9 @@ import java.util.concurrent.Executors class EthereumRpcHead( private val api: Reader, forkChoice: ForkChoice, + blockValidator: BlockValidator, private val interval: Duration = Duration.ofSeconds(10), -) : DefaultEthereumHead(forkChoice), Lifecycle { +) : DefaultEthereumHead(forkChoice, blockValidator), Lifecycle { companion object { val scheduler = diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt index 42c7b717..d008530f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient import org.slf4j.LoggerFactory @@ -25,8 +26,9 @@ import reactor.core.publisher.Flux class EthereumWsHead( private val ws: WsConnection, - forkChoice: ForkChoice -) : DefaultEthereumHead(forkChoice), Lifecycle { + forkChoice: ForkChoice, + blockValidator: BlockValidator +) : DefaultEthereumHead(forkChoice, blockValidator), Lifecycle { private val log = LoggerFactory.getLogger(EthereumWsHead::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt index 2850ee10..831c4e03 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt @@ -1,7 +1,9 @@ package io.emeraldpay.dshackle.upstream.ethereum.connectors +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.HttpFactory +import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice @@ -12,7 +14,8 @@ open class EthereumConnectorFactory( private val preferHttp: Boolean, private val wsFactory: EthereumWsFactory?, private val httpFactory: HttpFactory?, - private val forkChoice: ForkChoice + private val forkChoice: ForkChoice, + private val blockValidator: BlockValidator ) : ConnectorFactory { private val log = LoggerFactory.getLogger(EthereumConnectorFactory::class.java) @@ -25,11 +28,11 @@ open class EthereumConnectorFactory( override fun create(upstream: DefaultUpstream, validator: EthereumUpstreamValidator, chain: Chain): EthereumConnector { if (wsFactory != null && !preferHttp) { - return EthereumWsConnector(wsFactory, upstream, validator, chain, forkChoice) + return EthereumWsConnector(wsFactory, upstream, validator, chain, forkChoice, blockValidator) } if (httpFactory == null) { throw java.lang.IllegalArgumentException("Can't create rpc connector if no http factory set") } - return EthereumRpcConnector(httpFactory.create(upstream.getId(), chain), wsFactory, upstream.getId(), forkChoice) + return EthereumRpcConnector(httpFactory.create(upstream.getId(), chain), wsFactory, upstream.getId(), forkChoice, blockValidator) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt index 54816a74..c240d31a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt @@ -3,8 +3,10 @@ package io.emeraldpay.dshackle.upstream.ethereum.connectors import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.MergedHead +import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcHead import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead @@ -20,7 +22,8 @@ class EthereumRpcConnector( private val directReader: Reader, wsFactory: EthereumWsFactory?, id: String, - forkChoice: ForkChoice + forkChoice: ForkChoice, + blockValidator: BlockValidator ) : EthereumConnector, CachesEnabled { private val conn: WsConnection? private val head: Head @@ -33,14 +36,14 @@ class EthereumRpcConnector( if (wsFactory != null) { // do not set upstream to the WS, since it doesn't control the RPC upstream conn = wsFactory.create(null, null) - val wsHead = EthereumWsHead(conn, forkChoice) + val wsHead = EthereumWsHead(conn, forkChoice, blockValidator) // receive bew blocks through WebSockets, but also periodically verify with RPC in case if WS failed - val rpcHead = EthereumRpcHead(directReader, forkChoice, Duration.ofSeconds(60)) + val rpcHead = EthereumRpcHead(directReader, forkChoice, blockValidator, Duration.ofSeconds(60)) head = MergedHead(listOf(rpcHead, wsHead), forkChoice) } else { conn = null log.warn("Setting up connector for $id upstream with RPC-only access, less effective than WS+RPC") - head = EthereumRpcHead(directReader, forkChoice) + head = EthereumRpcHead(directReader, forkChoice, blockValidator) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt index ec670e7f..eb796104 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt @@ -1,12 +1,10 @@ package io.emeraldpay.dshackle.upstream.ethereum.connectors import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator -import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory -import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead -import io.emeraldpay.dshackle.upstream.ethereum.WsConnection +import io.emeraldpay.dshackle.upstream.ethereum.* import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse @@ -18,7 +16,8 @@ class EthereumWsConnector( upstream: DefaultUpstream, validator: EthereumUpstreamValidator, chain: Chain, - forkChoice: ForkChoice + forkChoice: ForkChoice, + blockValidator: BlockValidator ) : EthereumConnector { private val conn: WsConnection private val api: Reader @@ -26,7 +25,7 @@ class EthereumWsConnector( init { conn = wsFactory.create(upstream, validator) - head = EthereumWsHead(conn, forkChoice) + head = EthereumWsHead(conn, forkChoice, blockValidator) api = JsonRpcWsClient(conn) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy index b5760836..d86b035d 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy @@ -48,7 +48,7 @@ class FilteredApisSpec extends Specification { def httpFactory = Mock(HttpFactory) { create(_, _) >> TestingCommons.api().tap { it.id = "${i++}" } } - def connectorFactory = new EthereumConnectorFactory(false, null, httpFactory, new MostWorkForkChoice()) + def connectorFactory = new EthereumConnectorFactory(false, null, httpFactory, new MostWorkForkChoice(), BlockValidator.@Companion.ALWAYS_VALID) new EthereumRpcUpstream( "test", Chain.ETHEREUM, diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy index 94fe5fde..f0c3fc77 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.test.TestingCommons +import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.rpc.json.BlockJson @@ -31,7 +32,7 @@ import java.time.Instant class DefaultEthereumHeadSpec extends Specification { - DefaultEthereumHead head = new DefaultEthereumHead(new MostWorkForkChoice()) + DefaultEthereumHead head = new DefaultEthereumHead(new MostWorkForkChoice(), BlockValidator.@Companion.ALWAYS_VALID) ObjectMapper objectMapper = Global.objectMapper def blocks = (10L..20L).collect { i -> From da2a21efe87549ef9625b43120e1490dc812dfcf Mon Sep 17 00:00:00 2001 From: terminal Date: Mon, 5 Sep 2022 18:17:25 +0300 Subject: [PATCH 2/3] fix formatting --- .../emeraldpay/dshackle/startup/ConfiguredUpstreams.kt | 9 +++++++-- .../ethereum/connectors/EthereumConnectorFactory.kt | 1 - .../upstream/ethereum/connectors/EthereumRpcConnector.kt | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index ded6ea18..d59b3348 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -20,7 +20,12 @@ import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.BlockValidator +import io.emeraldpay.dshackle.upstream.CurrentMultistreamHolder +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.HttpRpcFactory +import io.emeraldpay.dshackle.upstream.MergedHead +import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcHead import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinZMQHead @@ -53,7 +58,7 @@ import javax.annotation.PostConstruct open class ConfiguredUpstreams( @Autowired private val currentUpstreams: CurrentMultistreamHolder, @Autowired private val fileResolver: FileResolver, - @Autowired private val config: UpstreamsConfig, + @Autowired private val config: UpstreamsConfig ) { private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt index 831c4e03..f7ef9b36 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt @@ -3,7 +3,6 @@ package io.emeraldpay.dshackle.upstream.ethereum.connectors import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.HttpFactory -import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt index c240d31a..8ee56bbe 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt @@ -6,7 +6,6 @@ import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.MergedHead -import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcHead import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead From 0f62f1a000bb10de6e501b50c59c509cfac65ff6 Mon Sep 17 00:00:00 2001 From: terminal Date: Mon, 5 Sep 2022 18:27:09 +0300 Subject: [PATCH 3/3] allow wildcard --- .editorconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 395a9a6c..23112292 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,2 +1,3 @@ [*.kt] -continuation_indent_size = 4 \ No newline at end of file +continuation_indent_size = 4 +disabled_rules=no-wildcard-imports \ No newline at end of file