From 08cbffe7d1640f12c6489760f767c20b040402ce Mon Sep 17 00:00:00 2001 From: a10zn8 Date: Tue, 31 Oct 2023 16:28:15 +0300 Subject: [PATCH] ethereum as generic upstream (#331) - merge all Ethereum implementations to Generic upstreams and multistreams - merge evm-pos and pow to ethereum --- .../org/drpc/chainsconfig/ChainsConfig.kt | 10 +- .../io/emeraldpay/dshackle/BlockchainType.kt | 12 +- .../dshackle/cache/CachesFactory.kt | 1 - .../emeraldpay/dshackle/config/CacheConfig.kt | 2 - .../dshackle/config/CacheConfigReader.kt | 10 +- .../dshackle/config/TokensConfig.kt | 2 +- .../config/context/MultistreamsConfig.kt | 56 +--- .../io/emeraldpay/dshackle/rpc/NativeCall.kt | 9 +- .../dshackle/rpc/NativeSubscribe.kt | 2 +- .../dshackle/startup/ConfiguredUpstreams.kt | 161 +++-------- .../dshackle/upstream/AbstractChainFees.kt | 131 --------- .../dshackle/upstream/CachingReader.kt | 14 +- .../dshackle/upstream/CallTargetsHolder.kt | 3 +- .../emeraldpay/dshackle/upstream/ChainFees.kt | 35 --- .../dshackle/upstream/EgressSubscription.kt | 2 +- .../dshackle/upstream/LabelsDetector.kt | 10 + .../dshackle/upstream/Multistream.kt | 79 +++--- .../dshackle/upstream/UpstreamValidator.kt | 25 ++ .../upstream/bitcoin/BitcoinMultistream.kt | 18 +- .../upstream/bitcoin/BitcoinReader.kt | 2 +- .../upstream/ethereum/ERC20Balance.kt | 75 ----- .../ethereum/EthereumCachingReader.kt | 3 +- .../ethereum/EthereumChainSpecific.kt | 75 +++++ .../upstream/ethereum/EthereumFees.kt | 87 ------ .../upstream/ethereum/EthereumLegacyFees.kt | 49 ---- .../ethereum/EthereumLikeRpcUpstream.kt | 206 -------------- .../upstream/ethereum/EthereumLikeUpstream.kt | 48 ---- .../upstream/ethereum/EthereumLocalReader.kt | 4 - .../upstream/ethereum/EthereumMultistream.kt | 215 -------------- .../upstream/ethereum/EthereumPriorityFees.kt | 59 ---- .../ethereum/EthereumUpstreamValidator.kt | 50 ++-- .../subscribe/EthereumLabelsDetector.kt | 5 +- .../ethereum_pos/EthereumPosMultiStream.kt | 222 --------------- .../upstream/generic/ChainSpecific.kt | 34 +++ .../upstream/generic/GenericMultistream.kt | 98 ++++++- .../upstream/generic/GenericUpstream.kt | 118 +++++++- .../generic/connectors/GenericConnector.kt | 4 +- .../generic/connectors/GenericRpcConnector.kt | 4 +- .../generic/connectors/GenericWsConnector.kt | 3 +- .../upstream/grpc/EthereumGrpcUpstream.kt | 223 --------------- ...GrpcUpstream.kt => GenericGrpcUpstream.kt} | 15 +- .../dshackle/upstream/grpc/GrpcUpstreams.kt | 67 ++--- .../starknet/StarknetChainSpecific.kt | 56 +++- .../config/CacheConfigReaderSpec.groovy | 10 - .../dshackle/rpc/NativeCallSpec.groovy | 7 +- .../dshackle/rpc/NativeSubscribeSpec.groovy | 8 +- .../dshackle/rpc/StreamHeadSpec.groovy | 12 +- .../test/EthereumRpcUpstreamMock.groovy | 87 ------ .../dshackle/test/GenericConnectorMock.groovy | 4 +- ...Mock.groovy => GenericUpstreamMock.groovy} | 28 +- .../test/MultistreamHolderMock.groovy | 32 ++- .../dshackle/test/TestingCommons.groovy | 53 ++-- .../upstream/AbstractChainFeesSpec.groovy | 253 ----------------- .../CurrentMultistreamHolderSpec.groovy | 21 +- .../dshackle/upstream/FilteredApisSpec.groovy | 18 +- .../dshackle/upstream/MultistreamSpec.groovy | 76 +++-- .../upstream/ethereum/ERC20BalanceSpec.groovy | 91 ------ .../EthereumEgressSubscriptionSpec.groovy | 23 +- .../ethereum/EthereumLegacyFeesSpec.groovy | 41 --- .../ethereum/EthereumLocalReaderSpec.groovy | 14 +- .../ethereum/EthereumPriorityFeesSpec.groovy | 150 ---------- .../EthereumUpstreamValidatorSpec.groovy | 83 +++--- .../ethereum/EthereumWsHeadSpec.groovy | 4 +- .../ethereum/WsConnectionImplRealSpec.groovy | 4 +- .../ethereum/WsConnectionImplSpec.groovy | 4 +- .../subscribe/ConnectBlockUpdatesSpec.groovy | 12 +- .../ethereum/subscribe/ConnectLogsSpec.groovy | 9 +- .../subscribe/ConnectNewHeadsSpec.groovy | 4 +- .../grpc/EthereumGrpcUpstreamSpec.groovy | 264 ------------------ 69 files changed, 826 insertions(+), 2790 deletions(-) delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/LabelsDetector.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeUpstream.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt delete mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt rename src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/{EthereumPosGrpcUpstream.kt => GenericGrpcUpstream.kt} (92%) delete mode 100644 src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy rename src/test/groovy/io/emeraldpay/dshackle/test/{EthereumPosRpcUpstreamMock.groovy => GenericUpstreamMock.groovy} (69%) delete mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy delete mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/ERC20BalanceSpec.groovy delete mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy delete mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy delete mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy diff --git a/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt b/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt index de74954e..e043a249 100644 --- a/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt +++ b/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt @@ -33,7 +33,10 @@ data class ChainsConfig(private val chains: List) : Iterable) : Iterable() { - companion object { - private val log = LoggerFactory.getLogger(CacheConfigReader::class.java) - } - override fun read(input: MappingNode?): CacheConfig? { return getMapping(input, "cache")?.let { node -> val config = CacheConfig() - getValueAsBool(node, "requests-cache-enabled")?.let { - config.requestsCacheEnabled = it - } getMapping(node, "redis")?.let { redisNode -> val redis = CacheConfig.Redis() val enabled = getValueAsBool(redisNode, "enabled") ?: true @@ -50,7 +42,7 @@ class CacheConfigReader : YamlConfigReader() { config.redis = redis } } - if (config.redis == null && config.requestsCacheEnabled) { + if (config.redis == null) { return null } config diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt index 51e3b222..4e8faade 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt @@ -41,7 +41,7 @@ class TokensConfig( type == null -> type address.isNullOrBlank() -> "address" blockchain != null && - (BlockchainType.from(blockchain!!) == BlockchainType.EVM_POS || BlockchainType.from(blockchain!!) == BlockchainType.EVM_POW) && + (BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM) && !Address.isValidAddress(address) -> "address" else -> null } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt index 0d7827ff..98c77954 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt @@ -2,16 +2,12 @@ package io.emeraldpay.dshackle.config.context import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.BlockchainType.BITCOIN -import io.emeraldpay.dshackle.BlockchainType.EVM_POS -import io.emeraldpay.dshackle.BlockchainType.EVM_POW -import io.emeraldpay.dshackle.BlockchainType.STARKNET import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.CachesFactory import io.emeraldpay.dshackle.upstream.CallTargetsHolder import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream +import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.ConfigurableListableBeanFactory @@ -31,14 +27,13 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) headScheduler: Scheduler, tracer: Tracer, ): List { - return Chain.values() + return Chain.entries .filterNot { it == Chain.UNSPECIFIED } .map { chain -> - when (BlockchainType.from(chain)) { - EVM_POS -> ethereumPosMultistream(chain, cachesFactory, headScheduler, tracer) - EVM_POW -> ethereumMultistream(chain, cachesFactory, headScheduler, tracer) - BITCOIN -> bitcoinMultistream(chain, cachesFactory, headScheduler) - STARKNET -> genericMultistream(chain, cachesFactory, headScheduler) + if (BlockchainType.from(chain) == BITCOIN) { + bitcoinMultistream(chain, cachesFactory, headScheduler) + } else { + genericMultistream(chain, cachesFactory, headScheduler, tracer) } } } @@ -47,47 +42,18 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) chain: Chain, cachesFactory: CachesFactory, headScheduler: Scheduler, + tracer: Tracer, ): Multistream { val name = "multi-$chain" + val cs = ChainSpecificRegistry.resolve(chain) return GenericMultistream( chain, CopyOnWriteArrayList(), cachesFactory.getCaches(chain), headScheduler, - ).also { register(it, name) } - } - - private fun ethereumMultistream( - chain: Chain, - cachesFactory: CachesFactory, - headScheduler: Scheduler, - tracer: Tracer, - ): EthereumMultistream { - val name = "multi-ethereum-$chain" - - return EthereumMultistream( - chain, - CopyOnWriteArrayList(), - cachesFactory.getCaches(chain), - headScheduler, - tracer, - ).also { register(it, name) } - } - - open fun ethereumPosMultistream( - chain: Chain, - cachesFactory: CachesFactory, - headScheduler: Scheduler, - tracer: Tracer, - ): EthereumPosMultiStream { - val name = "multi-ethereum-pos-$chain" - - return EthereumPosMultiStream( - chain, - CopyOnWriteArrayList(), - cachesFactory.getCaches(chain), - headScheduler, - tracer, + cs.makeCachingReaderBuilder(tracer), + cs::localReaderBuilder, + cs.subscriptionBuilder(headScheduler), ).also { register(it, name) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt index 536bdec9..3b3db842 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.BlockchainType -import io.emeraldpay.dshackle.BlockchainType.EVM_POS +import io.emeraldpay.dshackle.BlockchainType.ETHEREUM import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global.Companion.nullValue @@ -76,7 +76,6 @@ open class NativeCall( private val log = LoggerFactory.getLogger(NativeCall::class.java) private val objectMapper: ObjectMapper = Global.objectMapper - private val localRouterEnabled = config.cache?.requestsCacheEnabled ?: true private val passthrough = config.passthrough var rpcReaderFactory: RpcReaderFactory = RpcReaderFactory.default() @@ -85,7 +84,7 @@ open class NativeCall( @EventListener fun onUpstreamChangeEvent(event: UpstreamChangeEvent) { multistreamHolder.getUpstream(event.chain).let { up -> - if (BlockchainType.from(up.chain) == EVM_POS) { + if (BlockchainType.from(up.chain) == ETHEREUM) { ethereumCallSelectors.putIfAbsent( event.chain, EthereumCallSelector(up.caches), @@ -307,7 +306,7 @@ open class NativeCall( } // for ethereum the actual block needed for the call may be specified in the call parameters val callSpecificMatcher: Mono = - if (BlockchainType.from(upstream.chain) == BlockchainType.EVM_POS || BlockchainType.from(upstream.chain) == BlockchainType.EVM_POW) { + if (BlockchainType.from(upstream.chain) == ETHEREUM) { ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead(), passthrough) } else { null @@ -359,7 +358,7 @@ open class NativeCall( if (method in DefaultEthereumMethods.newFilterMethods) CreateFilterDecorator() else NoneResultDecorator() fun fetch(ctx: ValidCallContext): Mono { - return ctx.upstream.getLocalReader(localRouterEnabled) + return ctx.upstream.getLocalReader() .flatMap { api -> SpannedReader(api, tracer, LOCAL_READER) .read(JsonRpcRequest(ctx.payload.method, ctx.payload.params, ctx.nonce, ctx.forwardedSelector)) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt index 92ad5a6f..93eb16d3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt @@ -57,7 +57,7 @@ open class NativeSubscribe( fun start(request: BlockchainOuterClass.NativeSubscribeRequest): Publisher { val chain = Chain.byId(request.chainValue) - if (BlockchainType.from(chain) != BlockchainType.EVM_POS && BlockchainType.from(chain) != BlockchainType.EVM_POW) { + if (BlockchainType.from(chain) != BlockchainType.ETHEREUM) { return Mono.error(UnsupportedOperationException("Native subscribe is not supported for ${chain.chainCode}")) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index 2c15e345..6fff9716 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -20,9 +20,7 @@ import brave.grpc.GrpcTracing import com.google.common.annotations.VisibleForTesting import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.BlockchainType.BITCOIN -import io.emeraldpay.dshackle.BlockchainType.EVM_POS -import io.emeraldpay.dshackle.BlockchainType.EVM_POW -import io.emeraldpay.dshackle.BlockchainType.STARKNET +import io.emeraldpay.dshackle.BlockchainType.ETHEREUM import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.FileResolver @@ -52,13 +50,12 @@ import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock import io.emeraldpay.dshackle.upstream.bitcoin.ZMQServer import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumBlockValidator -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionFactory import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPoolFactory import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice +import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory.ConnectorMode.RPC_REQUESTS_WITH_MIXED_HEAD @@ -75,6 +72,7 @@ import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Component import reactor.core.scheduler.Scheduler import reactor.core.scheduler.Schedulers +import java.lang.IllegalStateException import java.net.URI import java.util.concurrent.Executor import java.util.concurrent.Executors @@ -138,24 +136,6 @@ open class ConfiguredUpstreams( .merge(up.options ?: ChainOptions.PartialOptions()) .buildOptions() val upstream = when (BlockchainType.from(chain)) { - EVM_POS -> { - buildEthereumPosUpstream( - up.nodeId, - up.cast(EthereumPosConnection::class.java), - chain, - options, - chainConfig, - ) - } - EVM_POW -> { - buildEthereumUpstream( - up.nodeId, - up.cast(RpcConnection::class.java), - chain, - options, - chainConfig, - ) - } BITCOIN -> { buildBitcoinUpstream( up.cast(BitcoinConnection::class.java), @@ -165,13 +145,29 @@ open class ConfiguredUpstreams( ) } - STARKNET -> { - buildStarknetUpstream( + ETHEREUM -> { + val posConn = up.cast(EthereumPosConnection::class.java) + + buildGenericUpstream( up.nodeId, - up.cast(RpcConnection::class.java), + up, + posConn.connection?.execution ?: throw IllegalStateException("Empty execution config"), chain, options, chainConfig, + posConn.connection?.upstreamRating ?: 0, + ) + } + + else -> { + buildGenericUpstream( + up.nodeId, + up, + up.connection as RpcConnection, + chain, + options, + chainConfig, + 0, ) } } @@ -224,31 +220,30 @@ open class ConfiguredUpstreams( } } - private fun buildStarknetUpstream( + private fun buildGenericUpstream( nodeId: Int?, - config: UpstreamsConfig.Upstream, + config: UpstreamsConfig.Upstream<*>, + connection: RpcConnection, chain: Chain, options: Options, chainConfig: ChainConfig, + nodeRating: Int, ): Upstream? { if (config.connection == null) { log.warn("Upstream doesn't have connection configuration") return null } - val connection = config.connection!! + val cs = ChainSpecificRegistry.resolve(chain) val connectorFactory = buildConnectorFactory( config.id!!, connection, chain, - NoChoiceWithPriorityForkChoice(0, config.id!!), + NoChoiceWithPriorityForkChoice(nodeRating, config.id!!), BlockValidator.ALWAYS_VALID, chainConfig, - ) - if (connectorFactory == null) { - return null - } + ) ?: return null val methods = buildMethods(config, chain) @@ -268,6 +263,9 @@ open class ConfiguredUpstreams( chainConfig, connectorFactory, eventPublisher, + cs::validator, + cs::labelDetector, + cs::subscriptionTopics, ) upstream.start() @@ -278,58 +276,6 @@ open class ConfiguredUpstreams( return upstream } - private fun buildEthereumPosUpstream( - nodeId: Int?, - config: UpstreamsConfig.Upstream, - chain: Chain, - options: Options, - chainConf: ChainConfig, - ): Upstream? { - val conn = config.connection!! - val execution = conn.execution - if (execution == null) { - log.warn("Upstream doesn't have execution layer configuration") - return null - } - val urls = ArrayList() - val connectorFactory = buildConnectorFactory( - config.id!!, - execution, - chain, - NoChoiceWithPriorityForkChoice(conn.upstreamRating, config.id!!), - BlockValidator.ALWAYS_VALID, - chainConf, - ) - val methods = buildMethods(config, chain) - if (connectorFactory == null) { - return null - } - - val hashUrl = conn.execution!!.let { - if (it.connectorMode == RPC_REQUESTS_WITH_MIXED_HEAD.name) it.rpc?.url ?: it.ws?.url else it.ws?.url ?: it.rpc?.url - } - val hash = getHash(nodeId, hashUrl!!) - val upstream = EthereumLikeRpcUpstream( - config.id!!, - hash, - chain, - options, - config.role, - methods, - QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(config.labels)), - connectorFactory, - chainConf, - true, - eventPublisher, - ) - upstream.start() - if (!upstream.isRunning) { - log.debug("Upstream ${upstream.getId()} is not running, it can't be added") - return null - } - return upstream - } - private fun buildBitcoinUpstream( config: UpstreamsConfig.Upstream, chain: Chain, @@ -373,45 +319,6 @@ open class ConfiguredUpstreams( return upstream } - private fun buildEthereumUpstream( - nodeId: Int?, - config: UpstreamsConfig.Upstream, - chain: Chain, - options: Options, - chainConf: ChainConfig, - ): Upstream? { - val conn = config.connection!! - val methods = buildMethods(config, chain) - - val connectorFactory = buildConnectorFactory( - config.id!!, - conn, - chain, - MostWorkForkChoice(), - EthereumBlockValidator(), - chainConf, - ) - if (connectorFactory == null) { - return null - } - - val hashUrl = if (conn.connectorMode == RPC_REQUESTS_WITH_MIXED_HEAD.name) conn.rpc?.url ?: conn.ws?.url else conn.ws?.url ?: conn.rpc?.url - val upstream = EthereumLikeRpcUpstream( - config.id!!, - getHash(nodeId, hashUrl!!), - chain, - options, config.role, - methods, - QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(config.labels)), - connectorFactory, - chainConf, - false, - eventPublisher, - ) - upstream.start() - return upstream - } - private fun buildGrpcUpstream( nodeId: Int?, config: UpstreamsConfig.Upstream, @@ -460,13 +367,13 @@ open class ConfiguredUpstreams( private fun buildHttpFactory(conn: HttpEndpoint?, urls: ArrayList? = null): HttpRpcFactory? { return conn?.let { endpoint -> - val tls = conn?.tls?.let { tls -> + val tls = conn.tls?.let { tls -> tls.ca?.let { ca -> fileResolver.resolve(ca).readBytes() } } urls?.add(endpoint.url) - HttpRpcFactory(endpoint.url.toString(), conn?.basicAuth, tls) + HttpRpcFactory(endpoint.url.toString(), conn.basicAuth, tls) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt deleted file mode 100644 index 11ec0dd2..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt +++ /dev/null @@ -1,131 +0,0 @@ -package io.emeraldpay.dshackle.upstream - -import com.github.benmanes.caffeine.cache.Caffeine -import io.emeraldpay.api.proto.BlockchainOuterClass -import org.slf4j.LoggerFactory -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import java.time.Duration -import java.util.EnumMap -import java.util.function.Function - -abstract class AbstractChainFees( - private val heightLimit: Int, - private val upstreams: Multistream, - extractTx: (B) -> List?, -) : ChainFees { - - companion object { - private val log = LoggerFactory.getLogger(AbstractChainFees::class.java) - } - - private val txSource = EnumMap>(ChainFees.Mode::class.java) - - init { - txSource[ChainFees.Mode.AVG_TOP] = TxAtTop(extractTx) - txSource[ChainFees.Mode.MIN_ALWAYS] = TxAtBottom(extractTx) - txSource[ChainFees.Mode.AVG_MIDDLE] = TxAtMiddle(extractTx) - txSource[ChainFees.Mode.AVG_LAST] = TxAtBottom(extractTx) - txSource[ChainFees.Mode.AVG_T5] = TxAtPos(extractTx, 5) - txSource[ChainFees.Mode.AVG_T20] = TxAtPos(extractTx, 20) - txSource[ChainFees.Mode.AVG_T50] = TxAtPos(extractTx, 50) - } - - override fun estimate(mode: ChainFees.Mode, blocks: Int): Mono { - return usingBlocks(blocks) - .flatMap { readFeesAt(it, mode) } - .transform(feeAggregation(mode)) - .next() - .map(getResponseBuilder()) - } - - // --- - - private val feeCache = Caffeine.newBuilder() - .expireAfterWrite(Duration.ofMinutes(60)) - .build, F>() - - fun usingBlocks(exp: Int): Flux { - val useBlocks = exp.coerceAtMost(heightLimit).coerceAtLeast(1) - - val height = upstreams.getHead().getCurrentHeight() - ?: return Mono.fromCallable { log.warn("Upstream is not ready. No current height") }.thenMany(Mono.empty()) // TODO or throw an exception to build a gRPC error? - val startBlock: Int = height.toInt() - useBlocks + 1 - if (startBlock < 0) { - log.warn("Blockchain doesn't have enough blocks. Height: $height") - return Flux.empty() - } - - return Flux.range(startBlock, useBlocks).map { it.toLong() } - } - - fun readFeesAt(height: Long, mode: ChainFees.Mode): Mono { - val current = feeCache.getIfPresent(Pair(height, mode)) - if (current != null) { - return Mono.just(current) - } - val txSelector = txSourceFor(mode) - return readFeesAt(height, txSelector).doOnNext { - // TODO it may be EMPTY for some blocks (ex. a no tx block), so nothing gets cached and goes to do the same call each time. so do cache empty values to avoid useless requests - feeCache.put(Pair(height, mode), it!!) - } - } - - open fun txSourceFor(mode: ChainFees.Mode): TxAt { - return txSource[mode] ?: throw IllegalStateException("No TS Source for mode $mode") - } - - abstract fun readFeesAt(height: Long, selector: TxAt): Mono - abstract fun feeAggregation(mode: ChainFees.Mode): Function, Mono> - abstract fun getResponseBuilder(): Function - - abstract class TxAt(private val extractTx: Function?>) { - fun get(block: B): TR? { - val txes = extractTx.apply(block) ?: return null - return get(txes) - } - - abstract fun get(transactions: List): TR? - } - - class TxAtPos(extractTx: Function?>, private val pos: Int) : TxAt(extractTx) { - - override fun get(transactions: List): TR? { - val index = pos.coerceAtMost(transactions.size - 1) - if (index < 0) { - return null - } - return transactions[transactions.size - index - 1] - } - } - - class TxAtTop(extractTx: Function?>) : TxAt(extractTx) { - override fun get(transactions: List): TR? { - if (transactions.isEmpty()) { - return null - } - return transactions[0] - } - } - - class TxAtBottom(extractTx: Function?>) : TxAt(extractTx) { - override fun get(transactions: List): TR? { - if (transactions.isEmpty()) { - return null - } - return transactions.last() - } - } - - class TxAtMiddle(extractTx: Function?>) : TxAt(extractTx) { - override fun get(transactions: List): TR? { - if (transactions.isEmpty()) { - return null - } - if (transactions.size == 1) { - return transactions[0] - } - return transactions[transactions.size / 2] - } - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingReader.kt index ce67e5ee..c9e0030f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingReader.kt @@ -1,3 +1,15 @@ package io.emeraldpay.dshackle.upstream -interface CachingReader +interface CachingReader : Lifecycle + +object NoopCachingReader : CachingReader { + override fun start() { + } + + override fun stop() { + } + + override fun isRunning(): Boolean { + return true + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt index 6149e646..10a8db83 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt @@ -18,9 +18,8 @@ class CallTargetsHolder { private fun setupDefaultMethods(chain: Chain): CallMethods { val created = when (BlockchainType.from(chain)) { - BlockchainType.EVM_POW -> DefaultEthereumMethods(chain) BlockchainType.BITCOIN -> DefaultBitcoinMethods() - BlockchainType.EVM_POS -> DefaultEthereumMethods(chain) + BlockchainType.ETHEREUM -> DefaultEthereumMethods(chain) BlockchainType.STARKNET -> DefaultStarknetMethods(chain) } callTargets[chain] = created diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt deleted file mode 100644 index 9cb86092..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt +++ /dev/null @@ -1,35 +0,0 @@ -package io.emeraldpay.dshackle.upstream - -import io.emeraldpay.api.proto.BlockchainOuterClass -import reactor.core.publisher.Mono - -interface ChainFees { - - companion object { - fun extractMode(req: BlockchainOuterClass.EstimateFeeRequest): Mode? { - return when (req.mode!!) { - BlockchainOuterClass.FeeEstimationMode.INVALID -> null - BlockchainOuterClass.FeeEstimationMode.AVG_LAST -> Mode.AVG_LAST - BlockchainOuterClass.FeeEstimationMode.AVG_T5 -> Mode.AVG_T5 - BlockchainOuterClass.FeeEstimationMode.AVG_T20 -> Mode.AVG_T20 - BlockchainOuterClass.FeeEstimationMode.AVG_T50 -> Mode.AVG_T50 - BlockchainOuterClass.FeeEstimationMode.MIN_ALWAYS -> Mode.MIN_ALWAYS - BlockchainOuterClass.FeeEstimationMode.AVG_MIDDLE -> Mode.AVG_MIDDLE - BlockchainOuterClass.FeeEstimationMode.AVG_TOP -> Mode.AVG_TOP - BlockchainOuterClass.FeeEstimationMode.UNRECOGNIZED -> null - } - } - } - - fun estimate(mode: Mode, blocks: Int): Mono - - enum class Mode { - AVG_LAST, - AVG_T5, - AVG_T20, - AVG_T50, - MIN_ALWAYS, - AVG_MIDDLE, - AVG_TOP, - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EgressSubscription.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EgressSubscription.kt index 4c83c4de..c46612a2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EgressSubscription.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EgressSubscription.kt @@ -22,7 +22,7 @@ interface EgressSubscription { fun subscribe(topic: String, params: Any?, matcher: Selector.Matcher): Flux } -class EmptyEgressSubscription : EgressSubscription { +object EmptyEgressSubscription : EgressSubscription { override fun getAvailableTopics(): List { return emptyList() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/LabelsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/LabelsDetector.kt new file mode 100644 index 00000000..d9884bd6 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/LabelsDetector.kt @@ -0,0 +1,10 @@ +package io.emeraldpay.dshackle.upstream + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.reader.JsonRpcReader +import reactor.core.publisher.Flux + +typealias LabelsDetectorBuilder = (Chain, JsonRpcReader) -> LabelsDetector? +interface LabelsDetector { + fun detectLabels(): Flux> +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt index 5552619f..431498f5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt @@ -51,10 +51,12 @@ import kotlin.concurrent.withLock */ abstract class Multistream( val chain: Chain, - private val upstreams: MutableList, val caches: Caches, ) : Upstream, Lifecycle, HasEgressSubscription { + abstract fun getUpstreams(): MutableList + abstract fun addUpstreamInternal(u: Upstream) + companion object { private const val metrics = "upstreams" } @@ -93,30 +95,6 @@ abstract class Multistream( .multicast() .directBestEffort() - init { - UpstreamAvailability.values().forEach { status -> - Metrics.gauge( - "$metrics.availability", - listOf(Tag.of("chain", chain.chainCode), Tag.of("status", status.name.lowercase())), - this, - ) { - upstreams.count { it.getStatus() == status }.toDouble() - } - } - - Metrics.gauge( - "$metrics.connected", - listOf(Tag.of("chain", chain.chainCode)), - this, - ) { - upstreams.size.toDouble() - } - - upstreams.forEach { up -> - monitorUpstream(up) - } - } - override fun getSubscriptionTopics(): List { return getEgressSubscription().getAvailableTopics() } @@ -153,29 +131,49 @@ abstract class Multistream( onUpstreamsUpdated() } + init { + UpstreamAvailability.entries.forEach { status -> + Metrics.gauge( + "$metrics.availability", + listOf(Tag.of("chain", chain.chainCode), Tag.of("status", status.name.lowercase())), + this, + ) { + getAll().count { it.getStatus() == status }.toDouble() + } + } + + Metrics.gauge( + "$metrics.connected", + listOf(Tag.of("chain", chain.chainCode)), + this, + ) { + getAll().size.toDouble() + } + } + /** * Get list of all underlying upstreams */ open fun getAll(): List { - return upstreams + return getUpstreams() } /** * Add an upstream */ fun addUpstream(upstream: Upstream): Boolean = - upstreams.none { + getUpstreams().none { it.getId() == upstream.getId() }.also { if (it) { - upstreams.add(upstream) + addUpstreamInternal(upstream) addHead(upstream) monitorUpstream(upstream) } } fun removeUpstream(id: String): Boolean = - upstreams.removeIf { up -> + getUpstreams().removeIf { up -> (up.getId() == id).also { if (it) { up.stop() @@ -196,13 +194,13 @@ abstract class Multistream( if (seq >= Int.MAX_VALUE / 2) { seq = 0 } - return FilteredApis(chain, upstreams, matcher, i) + return FilteredApis(chain, getUpstreams(), matcher, i) } /** * Finds an API that leverages caches and other optimizations/transformations of the request. */ - abstract fun getLocalReader(localEnabled: Boolean): Mono + abstract fun getLocalReader(): Mono override fun getIngressReader(): JsonRpcReader { throw NotImplementedError("Immediate direct API is not implemented for Aggregated Upstream") @@ -235,6 +233,7 @@ abstract class Multistream( lagObserver = null upstreams[0].setLag(0) } + upstreams.size > 1 -> if (lagObserver == null) lagObserver = makeLagObserver() } } @@ -389,17 +388,17 @@ abstract class Multistream( } catch (e: Exception) { log.warn("Head processing error: ${e.javaClass} ${e.message}") } - val statuses = upstreams.asSequence().map { it.getStatus() } + val statuses = getUpstreams().asSequence().map { it.getStatus() } .groupBy { it } .map { "${it.key.name}/${it.value.size}" } .joinToString(",") - val lag = upstreams.joinToString(", ") { + val lag = getUpstreams().joinToString(", ") { // by default, when no lag is available it uses Long.MAX_VALUE, and it doesn't make sense to print // status with such value. use NA (as Not Available) instead val value = it.getLag() value?.toString() ?: "NA" } - val weak = upstreams + val weak = getUpstreams() .filter { it.getStatus() != UpstreamAvailability.OK } .joinToString(", ") { it.getId() } @@ -424,6 +423,7 @@ abstract class Multistream( onUpstreamsUpdated() updateUpstreams.emitNext(event.upstream) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED } } + UpstreamChangeEvent.ChangeType.ADDED -> { if (!started) { start() @@ -441,6 +441,7 @@ abstract class Multistream( } } } + UpstreamChangeEvent.ChangeType.REMOVED -> { removeUpstream(event.upstream.getId()).takeIf { it }?.let { try { @@ -458,10 +459,10 @@ abstract class Multistream( } fun haveUpstreams(): Boolean = - upstreams.isNotEmpty() + getUpstreams().isNotEmpty() fun hasMatchingUpstream(matcher: Selector.LabelSelectorMatcher): Boolean { - return upstreams.any { matcher.matches(it) } + return getUpstreams().any { matcher.matches(it) } } fun subscribeAddedUpstreams(): Flux = @@ -469,12 +470,16 @@ abstract class Multistream( fun subscribeRemovedUpstreams(): Flux = removedUpstreams.asFlux() + fun subscribeUpdatedUpstreams(): Flux = updateUpstreams.asFlux() abstract fun makeLagObserver(): HeadLagObserver - open fun tryProxySubscribe(matcher: Selector.Matcher, request: BlockchainOuterClass.NativeSubscribeRequest): Flux? = null + open fun tryProxySubscribe( + matcher: Selector.Matcher, + request: BlockchainOuterClass.NativeSubscribeRequest, + ): Flux? = null abstract fun getCachingReader(): CachingReader? diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt new file mode 100644 index 00000000..995ad5e1 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt @@ -0,0 +1,25 @@ +package io.emeraldpay.dshackle.upstream + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig +import io.emeraldpay.dshackle.foundation.ChainOptions +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono + +typealias UpstreamValidatorBuilder = (Chain, Upstream, ChainOptions.Options, ChainConfig) -> UpstreamValidator? + +interface UpstreamValidator { + fun start(): Flux + + fun validateUpstreamSettings(): Mono + + fun validateUpstreamSettingsOnStartup(): ValidateUpstreamSettingsResult { + return validateUpstreamSettings().block() ?: ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR + } +} + +enum class ValidateUpstreamSettingsResult { + UPSTREAM_VALID, + UPSTREAM_SETTINGS_ERROR, + UPSTREAM_FATAL_SETTINGS_ERROR, +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt index 4e5ef579..03f2bcdb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt @@ -43,7 +43,7 @@ open class BitcoinMultistream( private val sourceUpstreams: MutableList, caches: Caches, private val headScheduler: Scheduler, -) : Multistream(chain, sourceUpstreams as MutableList, caches), Lifecycle { +) : Multistream(chain, caches), Lifecycle { private var head: Head = EmptyHead() private var esplora = sourceUpstreams.find { it.esploraClient != null }?.esploraClient @@ -51,6 +51,13 @@ open class BitcoinMultistream( private var addressActiveCheck: AddressActiveCheck? = null private var xpubAddresses: XpubAddresses? = null private var callRouter: LocalCallRouter = LocalCallRouter(DefaultBitcoinMethods(), reader) + override fun getUpstreams(): MutableList { + return sourceUpstreams + } + + override fun addUpstreamInternal(u: Upstream) { + sourceUpstreams.add(u as BitcoinUpstream) + } override fun init() { if (sourceUpstreams.size > 0) { @@ -59,11 +66,6 @@ open class BitcoinMultistream( super.init() } - open val upstreams: List - get() { - return sourceUpstreams - } - open fun getXpubAddresses(): XpubAddresses? { return xpubAddresses } @@ -103,7 +105,7 @@ open class BitcoinMultistream( .switchIfEmpty(Mono.error(Exception("No API available for $chain"))) } - override fun getLocalReader(localEnabled: Boolean): Mono { + override fun getLocalReader(): Mono { return Mono.just(callRouter) } @@ -146,7 +148,7 @@ open class BitcoinMultistream( } override fun getEgressSubscription(): EgressSubscription { - return EmptyEgressSubscription() + return EmptyEgressSubscription } override fun isRunning(): Boolean { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt index f7c99796..d076ea9b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt @@ -44,7 +44,7 @@ open class BitcoinReader( private val unspentReader: UnspentReader = if (esploraClient != null) { EsploraUnspentReader(esploraClient, head) - } else if (upstreams.upstreams.any { it.isGrpc() && it.getCapabilities().contains(Capability.BALANCE) }) { + } else if (upstreams.getUpstreams().any { it.isGrpc() && it.getCapabilities().contains(Capability.BALANCE) }) { RemoteUnspentReader(upstreams) } else { RpcUnspentReader(upstreams) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt deleted file mode 100644 index cf889279..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.upstream.ApiSource -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.etherjar.domain.Address -import io.emeraldpay.etherjar.erc20.ERC20Token -import io.emeraldpay.etherjar.hex.Hex32 -import io.emeraldpay.etherjar.hex.HexQuantity -import org.slf4j.LoggerFactory -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import java.math.BigInteger - -/** - * Query for a ERC20 token balance for an address - */ -open class ERC20Balance { - - companion object { - private val log = LoggerFactory.getLogger(ERC20Balance::class.java) - } - - open fun getBalance(upstreams: EthereumPosMultiStream, token: ERC20Token, address: Address): Mono { - return upstreams - // use only up-to-date upstreams - .getApiSource(Selector.HeightMatcher(upstreams.getHead().getCurrentHeight() ?: 0)) - .let { getBalance(it, token, address) } - } - - open fun getBalance(apis: ApiSource, token: ERC20Token, address: Address): Mono { - apis.request(1) - return Flux.from(apis) - .flatMap { - getBalance(it.cast(EthereumLikeRpcUpstream::class.java), token, address) - } - .doOnNext { - apis.resolve() - } - .next() - } - - open fun getBalance(upstream: EthereumLikeRpcUpstream, token: ERC20Token, address: Address): Mono { - return upstream - .getIngressReader() - .read(prepareEthCall(token, address, upstream.getHead())) - .flatMap(JsonRpcResponse::requireStringResult) - .map { Hex32.from(it).asQuantity().value } - } - - fun prepareEthCall(token: ERC20Token, target: Address, head: Head): JsonRpcRequest { - val call = token - .readBalanceOf(target) - .toJson() - val height = head.getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest" - return JsonRpcRequest("eth_call", listOf(call, height)) - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt index 4af3d69e..7a506650 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt @@ -34,7 +34,6 @@ import io.emeraldpay.dshackle.reader.RekeyingReader import io.emeraldpay.dshackle.reader.SpannedReader import io.emeraldpay.dshackle.reader.TransformingReader import io.emeraldpay.dshackle.upstream.CachingReader -import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumDirectReader.Result @@ -58,7 +57,7 @@ open class EthereumCachingReader( private val caches: Caches, callMethodsFactory: Factory, private val tracer: Tracer, -) : Lifecycle, CachingReader { +) : CachingReader { private val objectMapper: ObjectMapper = Global.objectMapper private val balanceCache = CurrentBlockCache() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt index 557c9316..f00f0eb2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt @@ -1,9 +1,31 @@ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.foundation.ChainOptions.Options +import io.emeraldpay.dshackle.reader.JsonRpcReader +import io.emeraldpay.dshackle.upstream.CachingReader +import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.EgressSubscription +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.LabelsDetector +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamValidator +import io.emeraldpay.dshackle.upstream.calls.CallMethods +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.AggregatedPendingTxes +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumLabelsDetector +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.NoPendingTxes +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource +import io.emeraldpay.dshackle.upstream.generic.CachingReaderBuilder import io.emeraldpay.dshackle.upstream.generic.ChainSpecific +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import org.springframework.cloud.sleuth.Tracer +import reactor.core.publisher.Mono +import reactor.core.scheduler.Scheduler object EthereumChainSpecific : ChainSpecific { override fun parseBlock(data: JsonRpcResponse, upstreamId: String): BlockContainer { @@ -11,4 +33,57 @@ object EthereumChainSpecific : ChainSpecific { } override fun latestBlockRequest() = JsonRpcRequest("eth_getBlockByNumber", listOf("latest", false)) + override fun localReaderBuilder( + cachingReader: CachingReader, + methods: CallMethods, + head: Head, + ): Mono { + return Mono.just(EthereumLocalReader(cachingReader as EthereumCachingReader, methods, head)) + } + + override fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription { + return { ms -> + val pendingTxes: PendingTxesSource = (ms.getAll()) + .filter { it is GenericUpstream } + .map { it as GenericUpstream } + .mapNotNull { + (it.getIngressSubscription() as EthereumIngressSubscription).getPendingTxes() + }.let { + if (it.isEmpty()) { + NoPendingTxes() + } else if (it.size == 1) { + it.first() + } else { + AggregatedPendingTxes(it) + } + } + EthereumEgressSubscription(ms, headScheduler, pendingTxes) + } + } + + override fun makeCachingReaderBuilder(tracer: Tracer): CachingReaderBuilder { + return { ms, caches, methodsFactory -> EthereumCachingReader(ms, caches, methodsFactory, tracer) } + } + + override fun validator( + chain: Chain, + upstream: Upstream, + options: Options, + config: ChainConfig, + ): UpstreamValidator? { + return EthereumUpstreamValidator(chain, upstream, options, config) + } + + override fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector? { + return EthereumLabelsDetector(reader, chain) + } + + override fun subscriptionTopics(upstream: GenericUpstream): List { + val subs = if (upstream.getCapabilities().contains(Capability.WS_HEAD)) { + listOf(EthereumEgressSubscription.METHOD_NEW_HEADS, EthereumEgressSubscription.METHOD_LOGS) + } else { + listOf() + } + return upstream.getIngressSubscription().getAvailableTopics().plus(subs).toSet().toList() + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt deleted file mode 100644 index 1c01e0b0..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.upstream.AbstractChainFees -import io.emeraldpay.dshackle.upstream.ChainFees -import io.emeraldpay.dshackle.upstream.Multistream -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot -import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionRefJson -import org.slf4j.LoggerFactory -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import reactor.util.function.Tuples -import java.util.function.Function - -abstract class EthereumFees( - upstreams: Multistream, - private val reader: EthereumCachingReader, - heightLimit: Int, -) : AbstractChainFees, TransactionRefJson, TransactionJsonSnapshot>(heightLimit, upstreams, extractTx), ChainFees { - - companion object { - private val log = LoggerFactory.getLogger(EthereumFees::class.java) - - private val extractTx = { block: BlockJson -> - block.transactions - } - } - - abstract fun extractFee(block: BlockJson, tx: TransactionJsonSnapshot): EthereumFee - - override fun readFeesAt(height: Long, selector: TxAt, TransactionRefJson>): Mono { - return reader.blocksByHeightParsed().read(height) - .flatMap { block -> - Mono.justOrEmpty(selector.get(block)) - .cast(TransactionRefJson::class.java) - .flatMap { reader.txByHash().read(it.hash) } - .map { tx -> extractFee(block, tx) } - } - } - - override fun feeAggregation(mode: ChainFees.Mode): Function, Mono> { - if (mode == ChainFees.Mode.MIN_ALWAYS) { - return Function { src -> - src.reduce { a, b -> - EthereumFee( - a.max.coerceAtLeast(b.max), - a.priority.coerceAtLeast(b.priority), - a.paid.coerceAtLeast(b.paid), - Wei.ZERO, - ) - } - } - } - return Function { src -> - src.map { Tuples.of(1, it) } - .reduce { a, b -> - Tuples.of(a.t1 + b.t1, a.t2.plus(b.t2)) - }.map { - EthereumFee(it.t2.max / it.t1, it.t2.priority / it.t1, it.t2.paid / it.t1, it.t2.base / it.t1) - } - } - } - - // --- - - data class EthereumFee(val max: Wei, val priority: Wei, val paid: Wei, val base: Wei) { - fun plus(o: EthereumFee): EthereumFee { - return EthereumFee(max + o.max, priority + o.priority, paid + o.paid, base + o.base) - } - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt deleted file mode 100644 index ab778050..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.api.proto.BlockchainOuterClass -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot -import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionRefJson -import org.slf4j.LoggerFactory -import java.util.function.Function - -class EthereumLegacyFees(upstreams: EthereumMultistream, reader: EthereumCachingReader, heightLimit: Int) : - EthereumFees(upstreams, reader, heightLimit) { - - companion object { - private val log = LoggerFactory.getLogger(EthereumLegacyFees::class.java) - } - - private val toGrpc: Function = Function { - BlockchainOuterClass.EstimateFeeResponse.newBuilder() - .setEthereumStd( - BlockchainOuterClass.EthereumStdFees.newBuilder() - .setFee(it.paid.amount.toString()), - ) - .build() - } - - override fun extractFee(block: BlockJson, tx: TransactionJsonSnapshot): EthereumFee { - return EthereumFee(tx.gasPrice, tx.gasPrice, tx.gasPrice, Wei.ZERO) - } - - override fun getResponseBuilder(): Function { - return toGrpc - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt deleted file mode 100644 index d2041cb5..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeRpcUpstream.kt +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright (c) 2020 EmeraldPay, Inc - * Copyright (c) 2019 ETCDEV GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.cache.Caches -import io.emeraldpay.dshackle.cache.CachesEnabled -import io.emeraldpay.dshackle.config.ChainsConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.foundation.ChainOptions -import io.emeraldpay.dshackle.reader.JsonRpcReader -import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.startup.UpstreamChangeEvent -import io.emeraldpay.dshackle.upstream.Capability -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.Upstream -import io.emeraldpay.dshackle.upstream.UpstreamAvailability -import io.emeraldpay.dshackle.upstream.calls.CallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_VALID -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumLabelsDetector -import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory -import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnector -import org.springframework.context.ApplicationEventPublisher -import org.springframework.context.Lifecycle -import reactor.core.Disposable -import reactor.core.publisher.Flux -import java.time.Duration -import java.util.concurrent.atomic.AtomicBoolean - -open class EthereumLikeRpcUpstream( - id: String, - hash: Byte, - val chain: Chain, - options: ChainOptions.Options, - role: UpstreamsConfig.UpstreamRole, - targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem?, - connectorFactory: ConnectorFactory, - chainConfig: ChainsConfig.ChainConfig, - skipEnhance: Boolean, - private val eventPublisher: ApplicationEventPublisher?, -) : EthereumLikeUpstream(id, hash, options, role, targets, node, chainConfig), Lifecycle, Upstream, CachesEnabled { - private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(chain, this, getOptions(), chainConfig.callLimitContract) - protected val connector: GenericConnector = connectorFactory.create(this, chain, skipEnhance) - private val labelsDetector = EthereumLabelsDetector(this.getIngressReader(), chain) - private var hasLiveSubscriptionHead: AtomicBoolean = AtomicBoolean(false) - - private var validatorSubscription: Disposable? = null - private var livenessSubscription: Disposable? = null - private var validationSettingsSubscription: Disposable? = null - - override fun getCapabilities(): Set { - return if (hasLiveSubscriptionHead.get()) { - setOf(Capability.RPC, Capability.BALANCE, Capability.WS_HEAD) - } else { - setOf(Capability.RPC, Capability.BALANCE) - } - } - - override fun setCaches(caches: Caches) { - if (connector is CachesEnabled) { - connector.setCaches(caches) - } - } - - override fun start() { - log.info("Configured for ${chain.chainName}") - connector.start() - val validSettingsResult = validator.validateUpstreamSettingsOnStartup() - when (validSettingsResult) { - UPSTREAM_FATAL_SETTINGS_ERROR -> { - connector.stop() - log.warn("Upstream ${getId()} couldn't start, invalid upstream settings") - return - } - UPSTREAM_SETTINGS_ERROR -> { - validateUpstreamSettings() - } - else -> { - upstreamStart() - labelsDetector.detectLabels() - .toStream() - .forEach { updateLabels(it) } - } - } - } - - private fun validateUpstreamSettings() { - validationSettingsSubscription = Flux.interval( - Duration.ofSeconds(10), - Duration.ofSeconds(20), - ).flatMap { - validator.validateUpstreamSettings() - }.subscribe { - when (it) { - UPSTREAM_FATAL_SETTINGS_ERROR -> { - connector.stop() - log.warn("Upstream ${getId()} couldn't start, invalid upstream settings") - disposeValidationSettingsSubscription() - } - UPSTREAM_VALID -> { - upstreamStart() - labelsDetector.detectLabels() - .subscribe { label -> updateLabels(label) } - eventPublisher?.publishEvent(UpstreamChangeEvent(chain, this, UpstreamChangeEvent.ChangeType.ADDED)) - disposeValidationSettingsSubscription() - } - else -> { - log.warn("Continue validation of upstream ${getId()}") - } - } - } - } - - private fun upstreamStart() { - if (getOptions().disableValidation) { - log.warn("Disable validation for upstream ${this.getId()}") - this.setLag(0) - this.setStatus(UpstreamAvailability.OK) - } else { - log.debug("Start validation for upstream ${this.getId()}") - validatorSubscription = validator.start() - .subscribe(this::setStatus) - } - livenessSubscription = connector.hasLiveSubscriptionHead().subscribe({ - hasLiveSubscriptionHead.set(it) - eventPublisher?.publishEvent(UpstreamChangeEvent(chain, this, UpstreamChangeEvent.ChangeType.UPDATED)) - }, { - log.debug("Error while checking live subscription for ${getId()}", it) - },) - } - - private fun updateLabels(label: Pair) { - log.info("Detected label ${label.first} with value ${label.second} for upstream ${getId()}") - node?.labels?.let { labels -> - labels[label.first] = label.second - } - } - - override fun getIngressSubscription(): EthereumIngressSubscription { - return connector.getIngressSubscription() - } - - override fun getSubscriptionTopics(): List { - val subs = if (getCapabilities().contains(Capability.WS_HEAD)) { - listOf(EthereumEgressSubscription.METHOD_NEW_HEADS, EthereumEgressSubscription.METHOD_LOGS) - } else { - listOf() - } - return getIngressSubscription().getAvailableTopics().plus(subs).toSet().toList() - } - - override fun getHead(): Head { - return connector.getHead() - } - - override fun stop() { - validatorSubscription?.dispose() - validatorSubscription = null - livenessSubscription?.dispose() - livenessSubscription = null - disposeValidationSettingsSubscription() - connector.stop() - } - - override fun isRunning(): Boolean { - return connector.isRunning() && validationSettingsSubscription == null - } - - override fun getIngressReader(): JsonRpcReader { - return connector.getIngressReader() - } - - override fun isGrpc(): Boolean { - return false - } - - @Suppress("UNCHECKED_CAST") - override fun cast(selfType: Class): T { - if (!selfType.isAssignableFrom(this.javaClass)) { - throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") - } - return this as T - } - - private fun disposeValidationSettingsSubscription() { - validationSettingsSubscription?.dispose() - validationSettingsSubscription = null - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeUpstream.kt deleted file mode 100644 index c02e5878..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeUpstream.kt +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (c) 2020 EmeraldPay, Inc - * Copyright (c) 2019 ETCDEV GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.config.ChainsConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.foundation.ChainOptions -import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.Capability -import io.emeraldpay.dshackle.upstream.DefaultUpstream -import io.emeraldpay.dshackle.upstream.calls.CallMethods - -abstract class EthereumLikeUpstream( - id: String, - hash: Byte, - options: ChainOptions.Options, - role: UpstreamsConfig.UpstreamRole, - targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem?, - val chainConfig: ChainsConfig.ChainConfig, -) : DefaultUpstream(id, hash, options, role, targets, node, chainConfig) { - - private val capabilities = setOf(Capability.RPC, Capability.BALANCE) - - override fun getCapabilities(): Set { - return capabilities - } - - override fun getLabels(): Collection { - return node?.let { listOf(it.labels) } ?: emptyList() - } - - abstract fun getIngressSubscription(): EthereumIngressSubscription -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt index 03b51881..26212358 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt @@ -41,7 +41,6 @@ class EthereumLocalReader( private val reader: EthereumCachingReader, private val methods: CallMethods, private val head: Head, - private val localEnabled: Boolean, ) : JsonRpcReader { override fun read(key: JsonRpcRequest): Mono { @@ -49,9 +48,6 @@ class EthereumLocalReader( return Mono.just(methods.executeHardcoded(key.method)) .map { JsonRpcResponse(it, null) } } - if (!localEnabled) { - return Mono.empty() - } if (!methods.isCallable(key.method)) { return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt deleted file mode 100644 index 79986fa5..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Copyright (c) 2020 EmeraldPay, Inc - * Copyright (c) 2020 ETCDEV GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.api.proto.BlockchainOuterClass -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.cache.Caches -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.reader.JsonRpcReader -import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.DistanceExtractor -import io.emeraldpay.dshackle.upstream.DynamicMergedHead -import io.emeraldpay.dshackle.upstream.EgressSubscription -import io.emeraldpay.dshackle.upstream.EmptyHead -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.HeadLagObserver -import io.emeraldpay.dshackle.upstream.Lifecycle -import io.emeraldpay.dshackle.upstream.MergedHead -import io.emeraldpay.dshackle.upstream.Multistream -import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.Upstream -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.AggregatedPendingTxes -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.NoPendingTxes -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource -import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice -import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice -import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream -import io.emeraldpay.etherjar.domain.BlockHash -import org.springframework.cloud.sleuth.Tracer -import org.springframework.util.ConcurrentReferenceHashMap -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import reactor.core.scheduler.Scheduler - -@Suppress("UNCHECKED_CAST") -open class EthereumMultistream( - chain: Chain, - val upstreams: MutableList, - caches: Caches, - private val headScheduler: Scheduler, - tracer: Tracer, -) : Multistream(chain, upstreams as MutableList, caches) { - - private var head: DynamicMergedHead = DynamicMergedHead( - PriorityForkChoice(), - "ETH Multistream of ${chain.chainCode}", - headScheduler, - ) - - private val filteredHeads: MutableMap = - ConcurrentReferenceHashMap(16, ConcurrentReferenceHashMap.ReferenceType.WEAK) - - private val reader: EthereumCachingReader = EthereumCachingReader(this, this.caches, getMethodsFactory(), tracer) - private var subscribe = EthereumEgressSubscription(this, headScheduler, NoPendingTxes()) - - init { - this.init() - } - - override fun init() { - if (upstreams.size > 0) { - upstreams.forEach { addHead(it) } - } - super.init() - } - - override fun onUpstreamsUpdated() { - super.onUpstreamsUpdated() - - val pendingTxes: PendingTxesSource = upstreams - .mapNotNull { - it.getIngressSubscription().getPendingTxes() - }.let { - if (it.isEmpty()) { - NoPendingTxes() - } else if (it.size == 1) { - it.first() - } else { - AggregatedPendingTxes(it) - } - } - subscribe = EthereumEgressSubscription(this, headScheduler, pendingTxes) - } - - override fun start() { - super.start() - head.start() - onHeadUpdated(head) - reader.start() - } - - override fun stop() { - super.stop() - reader.stop() - filteredHeads.clear() - } - - override fun addHead(upstream: Upstream) { - val newHead = upstream.getHead() - if (newHead is Lifecycle && !newHead.isRunning()) { - newHead.start() - } - head.addHead(upstream) - } - - override fun removeHead(upstreamId: String) { - head.removeHead(upstreamId) - } - - override fun makeLagObserver(): HeadLagObserver = - HeadLagObserver(head, upstreams, DistanceExtractor::extractPowDistance, headScheduler, 6).apply { - start() - } - - override fun isRunning(): Boolean { - return super.isRunning() || reader.isRunning() - } - - override fun getCachingReader(): EthereumCachingReader { - return reader - } - - override fun getHead(): Head { - return head - } - - override fun tryProxySubscribe( - matcher: Selector.Matcher, - request: BlockchainOuterClass.NativeSubscribeRequest, - ): Flux? = - upstreams.filter { - matcher.matches(it) - }.takeIf { ups -> - ups.size == 1 && ups.all { it.isGrpc() } - }?.map { - it as GrpcUpstream - }?.map { - it.getBlockchainApi().nativeSubscribe(request) - }?.let { - Flux.merge(it) - } - - override fun getLabels(): Collection { - return upstreams.flatMap { it.getLabels() } - } - - @Suppress("UNCHECKED_CAST") - override fun cast(selfType: Class): T { - if (!selfType.isAssignableFrom(this.javaClass)) { - throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") - } - return this as T - } - - override fun getEgressSubscription(): EgressSubscription { - return subscribe - } - - override fun getLocalReader(localEnabled: Boolean): Mono { - return Mono.just(EthereumLocalReader(reader, getMethods(), getHead(), localEnabled)) - } - - override fun getHead(mather: Selector.Matcher): Head = - filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ -> - upstreams.filter { mather.matches(it) } - .apply { - log.debug("Found $size upstreams matching [${mather.describeInternal()}]") - }.let { - val selected = it.map { it.getHead() } - when (it.size) { - 0 -> EmptyHead() - 1 -> selected.first() - else -> MergedHead(selected, MostWorkForkChoice(), headScheduler, "Eth head ${it.map { it.getId() }}").apply { - start() - } - } - } - } - - override fun getEnrichedHead(mather: Selector.Matcher): Head = - filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ -> - upstreams.filter { mather.matches(it) } - .apply { - log.debug("Found $size upstreams matching [${mather.describeInternal()}]") - }.let { - val selected = it.map { source -> source.getHead() } - EnrichedMergedHead( - selected, - getHead(), - headScheduler, - object : - Reader { - override fun read(key: BlockHash): Mono { - return reader.blocksByHashAsCont().read(key).map { res -> res.data } - } - }, - ) - } - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt deleted file mode 100644 index d5488be7..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.api.proto.BlockchainOuterClass -import io.emeraldpay.dshackle.upstream.Multistream -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot -import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionRefJson -import org.slf4j.LoggerFactory -import java.util.function.Function - -class EthereumPriorityFees(upstreams: Multistream, reader: EthereumCachingReader, heightLimit: Int) : - EthereumFees(upstreams, reader, heightLimit) { - - companion object { - private val log = LoggerFactory.getLogger(EthereumPriorityFees::class.java) - } - - private val toGrpc: Function = - Function { - BlockchainOuterClass.EstimateFeeResponse.newBuilder() - .setEthereumExtended( - BlockchainOuterClass.EthereumExtFees.newBuilder() - .setMax(it.max.amount.toString()) - .setPriority(it.priority.amount.toString()) - .setExpect(it.paid.amount.toString()), - ) - .build() - } - - override fun extractFee(block: BlockJson, tx: TransactionJsonSnapshot): EthereumFee { - val baseFee = block.baseFeePerGas ?: Wei.ZERO - if (tx.type == 2) { - // an EIP-1559 Transaction provides Max and Priority fee - val paid = (baseFee + tx.maxPriorityFeePerGas).coerceAtMost(tx.maxFeePerGas) - return EthereumFee(tx.maxFeePerGas, tx.maxPriorityFeePerGas, paid, baseFee) - } - return EthereumFee(tx.gasPrice, (tx.gasPrice - baseFee).coerceAtLeast(Wei.ZERO), tx.gasPrice, baseFee) - } - - override fun getResponseBuilder(): Function { - return toGrpc - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt index b41f9322..d07a47e2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt @@ -20,12 +20,12 @@ import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.UpstreamAvailability -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR -import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_VALID +import io.emeraldpay.dshackle.upstream.UpstreamValidator +import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.etherjar.domain.Address @@ -47,8 +47,8 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( private val chain: Chain, private val upstream: Upstream, private val options: ChainOptions.Options, - private val callLimitContract: String? = null, -) { + private val config: ChainConfig, +) : UpstreamValidator { companion object { private val log = LoggerFactory.getLogger(EthereumUpstreamValidator::class.java) val scheduler = @@ -127,7 +127,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( .onErrorReturn(UpstreamAvailability.UNAVAILABLE) } - fun start(): Flux { + override fun start(): Flux { return Flux.interval( Duration.ZERO, Duration.ofSeconds(options.validationInterval.toLong()), @@ -140,13 +140,9 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( } } - fun validateUpstreamSettingsOnStartup(): ValidateUpstreamSettingsResult { - return validateUpstreamSettings().block() ?: UPSTREAM_FATAL_SETTINGS_ERROR - } - - fun validateUpstreamSettings(): Mono { + override fun validateUpstreamSettings(): Mono { if (options.disableUpstreamValidation) { - return Mono.just(UPSTREAM_VALID) + return Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_VALID) } return Mono.zip( validateChain(), @@ -159,7 +155,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( private fun validateChain(): Mono { if (!options.validateChain) { - return Mono.just(UPSTREAM_VALID) + return Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_VALID) } return Mono.zip( chainId(), @@ -178,20 +174,20 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( } if (isChainValid) { - UPSTREAM_VALID + ValidateUpstreamSettingsResult.UPSTREAM_VALID } else { - UPSTREAM_FATAL_SETTINGS_ERROR + ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR } } .onErrorResume { log.error("Error during chain validation", it) - Mono.just(UPSTREAM_SETTINGS_ERROR) + Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR) } } private fun validateCallLimit(): Mono { - if (!options.validateCallLimit || callLimitContract == null) { - return Mono.just(UPSTREAM_VALID) + if (!options.validateCallLimit || config.callLimitContract == null) { + return Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_VALID) } return upstream.getIngressReader() .read( @@ -199,7 +195,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( "eth_call", listOf( TransactionCallJson( - Address.from(callLimitContract), + Address.from(config.callLimitContract), // calling contract with param 200_000, meaning it will generate 200k symbols or response // f4240 + metadata — ~1 million HexData.from("0xd8a26e3a00000000000000000000000000000000000000000000000000000000000f4240"), @@ -209,7 +205,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( ), ) .flatMap(JsonRpcResponse::requireResult) - .map { UPSTREAM_VALID } + .map { ValidateUpstreamSettingsResult.UPSTREAM_VALID } .onErrorResume { if (it.message != null && it.message!!.contains("rpc.returndata.limit")) { log.warn( @@ -217,7 +213,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( "You need to set up your return limit to at least 1_100_000. " + "Erigon config example: https://github.com/ledgerwatch/erigon/blob/d014da4dc039ea97caf04ed29feb2af92b7b129d/cmd/utils/flags.go#L369", ) - Mono.just(UPSTREAM_FATAL_SETTINGS_ERROR) + Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR) } else { Mono.error(it) } @@ -233,7 +229,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( "message ${ctx.exception().message}", ) } - .onErrorReturn(UPSTREAM_SETTINGS_ERROR) + .onErrorReturn(ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR) } private fun validateOldBlocks(): Mono { @@ -257,11 +253,11 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( "Node ${upstream.getId()} probably is synced incorrectly, it is not possible to get old blocks", ) } - UPSTREAM_VALID + ValidateUpstreamSettingsResult.UPSTREAM_VALID } .onErrorResume { log.warn("Error during old blocks validation", it) - Mono.just(UPSTREAM_VALID) + Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_VALID) } } @@ -290,10 +286,4 @@ open class EthereumUpstreamValidator @JvmOverloads constructor( .doOnError { log.error("Error during execution 'net_version' - ${it.message} for ${upstream.getId()}") } .flatMap(JsonRpcResponse::requireStringResult) } - - enum class ValidateUpstreamSettingsResult { - UPSTREAM_VALID, - UPSTREAM_SETTINGS_ERROR, - UPSTREAM_FATAL_SETTINGS_ERROR, - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumLabelsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumLabelsDetector.kt index 9bf33c49..29067ced 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumLabelsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumLabelsDetector.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.module.kotlin.readValue import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global.Companion.objectMapper import io.emeraldpay.dshackle.reader.JsonRpcReader +import io.emeraldpay.dshackle.upstream.LabelsDetector import io.emeraldpay.dshackle.upstream.ethereum.EthereumArchiveBlockNumberReader import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse @@ -14,10 +15,10 @@ import reactor.core.publisher.Mono class EthereumLabelsDetector( private val reader: JsonRpcReader, private val chain: Chain, -) { +) : LabelsDetector { private val blockNumberReader = EthereumArchiveBlockNumberReader(reader) - fun detectLabels(): Flux> { + override fun detectLabels(): Flux> { return Flux.merge( detectNodeType(), detectArchiveNode(), diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt deleted file mode 100644 index 46b14e47..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Copyright (c) 2020 EmeraldPay, Inc - * Copyright (c) 2020 ETCDEV GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.api.proto.BlockchainOuterClass -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.cache.Caches -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.reader.JsonRpcReader -import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.DistanceExtractor -import io.emeraldpay.dshackle.upstream.DynamicMergedHead -import io.emeraldpay.dshackle.upstream.EmptyHead -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.HeadLagObserver -import io.emeraldpay.dshackle.upstream.Lifecycle -import io.emeraldpay.dshackle.upstream.MergedHead -import io.emeraldpay.dshackle.upstream.Multistream -import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.Upstream -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.AggregatedPendingTxes -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.NoPendingTxes -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource -import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice -import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream -import io.emeraldpay.etherjar.domain.BlockHash -import org.springframework.cloud.sleuth.Tracer -import org.springframework.util.ConcurrentReferenceHashMap -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import reactor.core.scheduler.Scheduler - -@Suppress("UNCHECKED_CAST") -open class EthereumPosMultiStream( - chain: Chain, - val upstreams: MutableList, - caches: Caches, - private val headScheduler: Scheduler, - tracer: Tracer, -) : Multistream(chain, upstreams as MutableList, caches) { - - private var head: DynamicMergedHead = DynamicMergedHead( - PriorityForkChoice(), - "ETH Pos Multistream of ${chain.chainCode}", - headScheduler, - ) - - private val reader: EthereumCachingReader = EthereumCachingReader(this, this.caches, getMethodsFactory(), tracer) - private var subscribe = EthereumEgressSubscription(this, headScheduler, NoPendingTxes()) - private val filteredHeads: MutableMap = - ConcurrentReferenceHashMap(16, ConcurrentReferenceHashMap.ReferenceType.WEAK) - - init { - this.init() - } - - override fun init() { - if (upstreams.size > 0) { - upstreams.forEach { addHead(it) } - } - super.init() - } - - override fun start() { - super.start() - head.start() - onHeadUpdated(head) - reader.start() - } - - override fun stop() { - super.stop() - reader.stop() - filteredHeads.clear() - } - - override fun addHead(upstream: Upstream) { - val newHead = upstream.getHead() - if (newHead is Lifecycle && !newHead.isRunning()) { - newHead.start() - } - head.addHead(upstream) - } - - override fun removeHead(upstreamId: String) { - head.removeHead(upstreamId) - } - - override fun isRunning(): Boolean { - return super.isRunning() || reader.isRunning() - } - - override fun makeLagObserver(): HeadLagObserver = - HeadLagObserver(head, upstreams, DistanceExtractor::extractPriorityDistance, headScheduler, 6).apply { - start() - } - - override fun getCachingReader(): EthereumCachingReader { - return reader - } - - override fun getHead(): Head { - return head - } - - override fun tryProxySubscribe( - matcher: Selector.Matcher, - request: BlockchainOuterClass.NativeSubscribeRequest, - ): Flux? = - upstreams.filter { - matcher.matches(it) - }.takeIf { ups -> - ups.size == 1 && ups.all { it.isGrpc() } - }?.map { - it as GrpcUpstream - }?.map { - it.proxySubscribe(request) - }?.let { - Flux.merge(it) - } - - override fun getLabels(): Collection { - return upstreams.flatMap { it.getLabels() } - } - - @Suppress("UNCHECKED_CAST") - override fun cast(selfType: Class): T { - if (!selfType.isAssignableFrom(this.javaClass)) { - throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") - } - return this as T - } - - override fun getLocalReader(localEnabled: Boolean): Mono { - return Mono.just(EthereumLocalReader(reader, getMethods(), getHead(), localEnabled)) - } - - override fun getEgressSubscription(): EthereumEgressSubscription { - return subscribe - } - - override fun getHead(mather: Selector.Matcher): Head = - if (mather == Selector.empty || mather == Selector.anyLabel) { - head - } else { - filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ -> - upstreams.filter { mather.matches(it) } - .apply { - log.debug("Found $size upstreams matching [${mather.describeInternal()}]") - } - .let { - val selected = it.map { it.getHead() } - when (it.size) { - 0 -> EmptyHead() - 1 -> selected.first() - else -> MergedHead( - selected, - PriorityForkChoice(), - headScheduler, - "ETH head for ${it.map { it.getId() }}", - ).apply { - start() - } - } - } - } - } - - override fun getEnrichedHead(mather: Selector.Matcher): Head = - filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ -> - upstreams.filter { mather.matches(it) } - .apply { - log.debug("Found $size upstreams matching [${mather.describeInternal()}]") - }.let { - val selected = it.map { source -> source.getHead() } - EnrichedMergedHead( - selected, - getHead(), - headScheduler, - object : - Reader { - override fun read(key: BlockHash): Mono { - return reader.blocksByHashAsCont().read(key).map { res -> res.data } - } - }, - ) - } - } - - override fun onUpstreamsUpdated() { - super.onUpstreamsUpdated() - - val pendingTxes: PendingTxesSource = upstreams - .mapNotNull { - it.getIngressSubscription().getPendingTxes() - }.let { - if (it.isEmpty()) { - NoPendingTxes() - } else if (it.size == 1) { - it.first() - } else { - AggregatedPendingTxes(it) - } - } - subscribe = EthereumEgressSubscription(this, headScheduler, pendingTxes) - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt index 90e90e13..fd69eb60 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt @@ -3,19 +3,53 @@ package io.emeraldpay.dshackle.upstream.generic import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.BlockchainType.STARKNET import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.cache.Caches +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.foundation.ChainOptions +import io.emeraldpay.dshackle.reader.JsonRpcReader +import io.emeraldpay.dshackle.upstream.CachingReader +import io.emeraldpay.dshackle.upstream.EgressSubscription +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.LabelsDetector +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamValidator +import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific +import org.apache.commons.collections4.Factory +import org.springframework.cloud.sleuth.Tracer +import reactor.core.publisher.Mono +import reactor.core.scheduler.Scheduler + +typealias SubscriptionBuilder = (Multistream) -> EgressSubscription +typealias LocalReaderBuilder = (CachingReader, CallMethods, Head) -> Mono +typealias CachingReaderBuilder = (Multistream, Caches, Factory) -> CachingReader interface ChainSpecific { fun parseBlock(data: JsonRpcResponse, upstreamId: String): BlockContainer fun latestBlockRequest(): JsonRpcRequest + + fun localReaderBuilder(cachingReader: CachingReader, methods: CallMethods, head: Head): Mono + + fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription + + fun makeCachingReaderBuilder(tracer: Tracer): CachingReaderBuilder + + fun validator(chain: Chain, upstream: Upstream, options: ChainOptions.Options, config: ChainConfig): UpstreamValidator? + + fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector? + + fun subscriptionTopics(upstream: GenericUpstream): List } object ChainSpecificRegistry { + + @JvmStatic fun resolve(chain: Chain): ChainSpecific { if (BlockchainType.from(chain) == STARKNET) { return StarknetChainSpecific diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericMultistream.kt index 3e351ce2..7fb8f013 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericMultistream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.generic +import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig @@ -24,24 +25,42 @@ import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.DistanceExtractor import io.emeraldpay.dshackle.upstream.DynamicMergedHead import io.emeraldpay.dshackle.upstream.EgressSubscription -import io.emeraldpay.dshackle.upstream.EmptyEgressSubscription +import io.emeraldpay.dshackle.upstream.EmptyHead import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.HeadLagObserver import io.emeraldpay.dshackle.upstream.Lifecycle +import io.emeraldpay.dshackle.upstream.MergedHead import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector.Matcher import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice +import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream +import org.springframework.util.ConcurrentReferenceHashMap +import org.springframework.util.ConcurrentReferenceHashMap.ReferenceType.WEAK +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.core.scheduler.Scheduler -@Suppress("UNCHECKED_CAST") open class GenericMultistream( chain: Chain, - val upstreams: MutableList, + private val upstreams: MutableList, caches: Caches, private val headScheduler: Scheduler, -) : Multistream(chain, upstreams as MutableList, caches) { + cachingReaderBuilder: CachingReaderBuilder, + private val localReaderBuilder: LocalReaderBuilder, + private val subscriptionBuilder: SubscriptionBuilder, +) : Multistream(chain, caches) { + + private val cachingReader = cachingReaderBuilder(this, caches, getMethodsFactory()) + + override fun getUpstreams(): MutableList { + return upstreams + } + + override fun addUpstreamInternal(u: Upstream) { + upstreams.add(u as GenericUpstream) + } private var head: DynamicMergedHead = DynamicMergedHead( PriorityForkChoice(), @@ -53,6 +72,8 @@ open class GenericMultistream( this.init() } + private var subscription: EgressSubscription = subscriptionBuilder(this) + override fun init() { if (upstreams.size > 0) { upstreams.forEach { addHead(it) } @@ -60,10 +81,20 @@ open class GenericMultistream( super.init() } + private val filteredHeads: MutableMap = + ConcurrentReferenceHashMap(16, WEAK) + override fun start() { super.start() head.start() onHeadUpdated(head) + cachingReader.start() + } + + override fun stop() { + super.stop() + cachingReader.stop() + filteredHeads.clear() } override fun addHead(upstream: Upstream) { @@ -78,17 +109,45 @@ open class GenericMultistream( head.removeHead(upstreamId) } + override fun isRunning(): Boolean { + return super.isRunning() || cachingReader.isRunning() + } + override fun makeLagObserver(): HeadLagObserver = HeadLagObserver(head, upstreams, DistanceExtractor::extractPriorityDistance, headScheduler, 6).apply { start() } override fun getCachingReader(): CachingReader? { - return null + return cachingReader } override fun getHead(mather: Matcher): Head { - return getHead() + if (mather == Selector.empty || mather == Selector.anyLabel) { + return head + } else { + return filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ -> + upstreams.filter { mather.matches(it) } + .apply { + log.debug("Found $size upstreams matching [${mather.describeInternal()}]") + } + .let { + val selected = it.map { it.getHead() } + when (it.size) { + 0 -> EmptyHead() + 1 -> selected.first() + else -> MergedHead( + selected, + PriorityForkChoice(), + headScheduler, + "Head for ${it.map { it.getId() }}", + ).apply { + start() + } + } + } + } + } } override fun getHead(): Head { @@ -111,11 +170,32 @@ open class GenericMultistream( return this as T } - override fun getLocalReader(localEnabled: Boolean): Mono { - return Mono.just(LocalReader(getMethods())) + override fun getLocalReader(): Mono { + return localReaderBuilder(cachingReader, getMethods(), getHead()) } override fun getEgressSubscription(): EgressSubscription { - return EmptyEgressSubscription() + return subscription } + + override fun onUpstreamsUpdated() { + super.onUpstreamsUpdated() + subscription = subscriptionBuilder(this) + } + + override fun tryProxySubscribe( + matcher: Matcher, + request: BlockchainOuterClass.NativeSubscribeRequest, + ): Flux? = + upstreams.filter { + matcher.matches(it) + }.takeIf { ups -> + ups.size == 1 && ups.all { it.isGrpc() } + }?.map { + it as GrpcUpstream + }?.map { + it.proxySubscribe(request) + }?.let { + Flux.merge(it) + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt index b849206f..3be3e9d9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt @@ -8,21 +8,27 @@ import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.UpstreamChangeEvent -import io.emeraldpay.dshackle.startup.UpstreamChangeEvent.ChangeType.UPDATED import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.IngressSubscription +import io.emeraldpay.dshackle.upstream.LabelsDetectorBuilder import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.dshackle.upstream.UpstreamValidator +import io.emeraldpay.dshackle.upstream.UpstreamValidatorBuilder +import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnector import org.springframework.context.ApplicationEventPublisher import org.springframework.context.Lifecycle import reactor.core.Disposable +import reactor.core.publisher.Flux +import java.time.Duration import java.util.concurrent.atomic.AtomicBoolean -class GenericUpstream( +open class GenericUpstream( id: String, val chain: Chain, hash: Byte, @@ -30,14 +36,23 @@ class GenericUpstream( role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, private val node: QuorumForLabels.QuorumItem?, - val chainConfig: ChainsConfig.ChainConfig, + chainConfig: ChainsConfig.ChainConfig, connectorFactory: ConnectorFactory, private val eventPublisher: ApplicationEventPublisher?, + validatorBuilder: UpstreamValidatorBuilder, + labelsDetectorBuilder: LabelsDetectorBuilder, + private val subscriptionTopics: (GenericUpstream) -> List, ) : DefaultUpstream(id, hash, null, UpstreamAvailability.OK, options, role, targets, node, chainConfig), Lifecycle { + private val validator: UpstreamValidator? = validatorBuilder(chain, this, getOptions(), chainConfig) + private var validatorSubscription: Disposable? = null + private var validationSettingsSubscription: Disposable? = null + private val hasLiveSubscriptionHead: AtomicBoolean = AtomicBoolean(false) - private val connector: GenericConnector = connectorFactory.create(this, chain, true) + protected val connector: GenericConnector = connectorFactory.create(this, chain, true) private var livenessSubscription: Disposable? = null + private val labelsDetector = labelsDetectorBuilder(chain, this.getIngressReader()) + override fun getHead(): Head { return connector.getHead() } @@ -51,10 +66,7 @@ class GenericUpstream( } override fun getSubscriptionTopics(): List { - // should be implemented in next iterations - // starknet doesn't have any subscriptions at all - // polkadot serves subscriptions like separate json-rpc methods - return emptyList() + return subscriptionTopics(this) } // outdated, looks like applicable only for bitcoin and our ws_head trick @@ -83,19 +95,107 @@ class GenericUpstream( log.info("Configured for ${chain.chainName}") connector.start() + if (validator != null) { + val validSettingsResult = validator.validateUpstreamSettingsOnStartup() + when (validSettingsResult) { + ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR -> { + connector.stop() + log.warn("Upstream ${getId()} couldn't start, invalid upstream settings") + return + } + ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR -> { + validateUpstreamSettings() + } + else -> { + upstreamStart() + } + } + } else { + upstreamStart() + } + } + + private fun validateUpstreamSettings() { + if (validator != null) { + validationSettingsSubscription = Flux.interval( + Duration.ofSeconds(10), + Duration.ofSeconds(20), + ).flatMap { + validator.validateUpstreamSettings() + }.subscribe { + when (it) { + ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR -> { + connector.stop() + disposeValidationSettingsSubscription() + } + + ValidateUpstreamSettingsResult.UPSTREAM_VALID -> { + upstreamStart() + eventPublisher?.publishEvent( + UpstreamChangeEvent( + chain, + this, + UpstreamChangeEvent.ChangeType.ADDED, + ), + ) + disposeValidationSettingsSubscription() + } + + else -> { + log.warn("Continue validation of upstream ${getId()}") + } + } + } + } + } + + private fun detectLabels() { + labelsDetector?.detectLabels()?.subscribe { label -> updateLabels(label) } + } + + private fun upstreamStart() { + if (getOptions().disableValidation) { + log.warn("Disable validation for upstream ${this.getId()}") + this.setLag(0) + this.setStatus(UpstreamAvailability.OK) + } else { + log.debug("Start validation for upstream ${this.getId()}") + validatorSubscription = validator?.start() + ?.subscribe(this::setStatus) + } livenessSubscription = connector.hasLiveSubscriptionHead().subscribe({ hasLiveSubscriptionHead.set(it) - eventPublisher?.publishEvent(UpstreamChangeEvent(chain, this, UPDATED)) + eventPublisher?.publishEvent(UpstreamChangeEvent(chain, this, UpstreamChangeEvent.ChangeType.UPDATED)) }, { log.debug("Error while checking live subscription for ${getId()}", it) },) + detectLabels() } override fun stop() { + validatorSubscription?.dispose() + validatorSubscription = null livenessSubscription?.dispose() livenessSubscription = null + disposeValidationSettingsSubscription() connector.stop() } + private fun disposeValidationSettingsSubscription() { + validationSettingsSubscription?.dispose() + validationSettingsSubscription = null + } + + private fun updateLabels(label: Pair) { + log.info("Detected label ${label.first} with value ${label.second} for upstream ${getId()}") + node?.labels?.let { labels -> + labels[label.first] = label.second + } + } + + fun getIngressSubscription(): IngressSubscription { + return connector.getIngressSubscription() + } + override fun isRunning() = connector.isRunning() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt index ca68bf18..0486bb4c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt @@ -2,8 +2,8 @@ package io.emeraldpay.dshackle.upstream.generic.connectors import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.Lifecycle -import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription import reactor.core.publisher.Flux interface GenericConnector : Lifecycle { @@ -13,5 +13,5 @@ interface GenericConnector : Lifecycle { fun getIngressReader(): JsonRpcReader - fun getIngressSubscription(): EthereumIngressSubscription + fun getIngressSubscription(): IngressSubscription } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt index 43ee5ea0..6b2732f9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt @@ -7,9 +7,9 @@ import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.MergedHead -import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidator import io.emeraldpay.dshackle.upstream.ethereum.NoEthereumIngressSubscription @@ -145,7 +145,7 @@ class GenericRpcConnector( return directReader } - override fun getIngressSubscription(): EthereumIngressSubscription { + override fun getIngressSubscription(): IngressSubscription { return NoEthereumIngressSubscription.DEFAULT } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt index 3bd33a2c..2caa57b9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt @@ -4,6 +4,7 @@ import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidator @@ -74,7 +75,7 @@ class GenericWsConnector( return reader } - override fun getIngressSubscription(): EthereumIngressSubscription { + override fun getIngressSubscription(): IngressSubscription { return subscriptions } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt deleted file mode 100644 index eb6fdeb3..00000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Copyright (c) 2020 EmeraldPay, Inc - * Copyright (c) 2019 ETCDEV GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.grpc - -import io.emeraldpay.api.proto.BlockchainOuterClass -import io.emeraldpay.api.proto.ReactorBlockchainGrpc.ReactorBlockchainStub -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.Defaults -import io.emeraldpay.dshackle.config.ChainsConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.data.BlockId -import io.emeraldpay.dshackle.foundation.ChainOptions -import io.emeraldpay.dshackle.reader.JsonRpcReader -import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.BuildInfo -import io.emeraldpay.dshackle.upstream.Capability -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.Lifecycle -import io.emeraldpay.dshackle.upstream.Upstream -import io.emeraldpay.dshackle.upstream.UpstreamAvailability -import io.emeraldpay.dshackle.upstream.calls.CallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeUpstream -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumDshackleIngressSubscription -import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.etherjar.domain.BlockHash -import io.emeraldpay.etherjar.rpc.RpcException -import org.reactivestreams.Publisher -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import reactor.core.scheduler.Scheduler -import java.math.BigInteger -import java.time.Instant -import java.util.Locale -import java.util.concurrent.TimeoutException -import java.util.function.Function - -open class EthereumGrpcUpstream( - private val parentId: String, - hash: Byte, - role: UpstreamsConfig.UpstreamRole, - private val chain: Chain, - private val remote: ReactorBlockchainStub, - client: JsonRpcGrpcClient, - overrideLabels: UpstreamsConfig.Labels?, - chainConfig: ChainsConfig.ChainConfig, - headScheduler: Scheduler, -) : EthereumLikeUpstream( - "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", - hash, - ChainOptions.PartialOptions.getDefaults().buildOptions(), - role, - null, - null, - chainConfig, -), - GrpcUpstream, - Lifecycle { - - private val blockConverter: Function = Function { value -> - val parentHash = - if (value.parentBlockId.isBlank()) { - null - } else { - BlockId.from(BlockHash.from("0x" + value.parentBlockId)) - } - val block = BlockContainer( - value.height, - BlockId.from(BlockHash.from("0x" + value.blockId)), - BigInteger(1, value.weight.toByteArray()), - Instant.ofEpochMilli(value.timestamp), - false, - null, - null, - parentHash, - ) - block - } - - override fun getSubscriptionTopics(): List { - return subscriptionTopics - } - - private val reloadBlock: Function> = Function { existingBlock -> - // head comes without transaction data - // need to download transactions for the block - defaultReader.read(JsonRpcRequest("eth_getBlockByHash", listOf(existingBlock.hash.toHexWithPrefix(), false))) - .flatMap(JsonRpcResponse::requireResult) - .map { - BlockContainer.fromEthereumJson(it, getId()) - } - .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) - .doOnError { t -> - setStatus(UpstreamAvailability.UNAVAILABLE) - val msg = "Failed to download block data for chain $chain on $parentId" - if (t is RpcException || t is TimeoutException) { - log.warn("$msg. Message: ${t.message}") - } else { - log.error(msg, t) - } - } - } - - private val upstreamStatus = GrpcUpstreamStatus(overrideLabels) - private val grpcHead = GrpcHead( - getId(), - chain, - this, - remote, - blockConverter, - reloadBlock, - MostWorkForkChoice(), - headScheduler, - ) - private var capabilities: Set = emptySet() - private val buildInfo: BuildInfo = BuildInfo() - private var subscriptionTopics = listOf() - - private val defaultReader: JsonRpcReader = client.getReader() - var timeout = Defaults.timeout - private val ethereumSubscriptions = EthereumDshackleIngressSubscription(chain, remote) - - override fun getBlockchainApi(): ReactorBlockchainStub { - return remote - } - - override fun proxySubscribe(request: BlockchainOuterClass.NativeSubscribeRequest): Flux = - remote.nativeSubscribe(request) - - override fun start() { - } - - override fun isRunning(): Boolean { - return true - } - - override fun stop() { - } - - override fun getBuildInfo(): BuildInfo { - return buildInfo - } - - override fun update(conf: BlockchainOuterClass.DescribeChain, buildInfo: BlockchainOuterClass.BuildInfo): Boolean { - val newBuildInfo = BuildInfo.extract(buildInfo) - val buildInfoChanged = this.buildInfo.update(newBuildInfo) - val newCapabilities = RemoteCapabilities.extract(conf) - val upstreamStatusChanged = (upstreamStatus.update(conf) || (newCapabilities != capabilities)).also { - capabilities = newCapabilities - } - conf.status?.let { status -> onStatus(status) } - val subsChanged = (conf.supportedSubscriptionsList != subscriptionTopics).also { - subscriptionTopics = conf.supportedSubscriptionsList - } - return buildInfoChanged || upstreamStatusChanged || subsChanged - } - - override fun getQuorumByLabel(): QuorumForLabels { - return upstreamStatus.getNodes() - } - - // ------------------------------------------------------------------------------------------ - - override fun getLabels(): Collection { - return upstreamStatus.getLabels() - } - - override fun getIngressSubscription(): EthereumIngressSubscription { - return ethereumSubscriptions - } - - override fun getMethods(): CallMethods { - return upstreamStatus.getCallMethods() - } - - override fun isAvailable(): Boolean { - return super.isAvailable() && grpcHead.getCurrent() != null && getQuorumByLabel().getAll().any { - it.quorum > 0 - } - } - - override fun getHead(): Head { - return grpcHead - } - - override fun getIngressReader(): JsonRpcReader { - return defaultReader - } - - @Suppress("UNCHECKED_CAST") - override fun cast(selfType: Class): T { - if (!selfType.isAssignableFrom(this.javaClass)) { - throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") - } - return this as T - } - - override fun getCapabilities(): Set { - return capabilities - } - - override fun isGrpc(): Boolean { - return true - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GenericGrpcUpstream.kt similarity index 92% rename from src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt rename to src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GenericGrpcUpstream.kt index f5710a58..56f29183 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GenericGrpcUpstream.kt @@ -28,13 +28,11 @@ import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.BuildInfo import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.calls.CallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeUpstream -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.EthereumDshackleIngressSubscription import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient import io.emeraldpay.etherjar.domain.BlockHash @@ -45,7 +43,7 @@ import java.time.Instant import java.util.Locale import java.util.function.Function -open class EthereumPosGrpcUpstream( +open class GenericGrpcUpstream( parentId: String, hash: Byte, role: UpstreamsConfig.UpstreamRole, @@ -56,7 +54,7 @@ open class EthereumPosGrpcUpstream( overrideLabels: UpstreamsConfig.Labels?, chainConfig: ChainsConfig.ChainConfig, headScheduler: Scheduler, -) : EthereumLikeUpstream( +) : DefaultUpstream( "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", hash, ChainOptions.PartialOptions.getDefaults().buildOptions(), @@ -103,7 +101,8 @@ open class EthereumPosGrpcUpstream( private val buildInfo: BuildInfo = BuildInfo() private val defaultReader: JsonRpcReader = client.getReader() - private val ethereumSubscriptions = EthereumDshackleIngressSubscription(chain, remote) + + // private val ethereumSubscriptions = EthereumDshackleIngressSubscription(chain, remote) private var subscriptionTopics = listOf() override fun start() { @@ -155,10 +154,6 @@ open class EthereumPosGrpcUpstream( return upstreamStatus.getLabels() } - override fun getIngressSubscription(): EthereumIngressSubscription { - return ethereumSubscriptions - } - override fun getMethods(): CallMethods { return upstreamStatus.getCallMethods() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt index 83e8f1b7..1cf32a48 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt @@ -25,6 +25,7 @@ import io.emeraldpay.api.proto.Common.ChainRef.UNRECOGNIZED import io.emeraldpay.api.proto.ReactorAuthGrpc import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.BlockchainType +import io.emeraldpay.dshackle.BlockchainType.BITCOIN import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.FileResolver @@ -34,7 +35,6 @@ import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.upstream.DefaultUpstream -import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.grpc.auth.AuthException import io.emeraldpay.dshackle.upstream.grpc.auth.ClientAuthenticationInterceptor @@ -246,42 +246,37 @@ class GrpcUpstreams( return sslContext.build() } - private val creators: Map DefaultUpstream> = mapOf( - BlockchainType.EVM_POW to { chain, rpcClient -> - EthereumGrpcUpstream( - id, - hash, - role, - chain, - client, - rpcClient, - labels, - chainsConfig.resolve(chain.chainName), - headScheduler, - ) - }, - BlockchainType.EVM_POS to { chain, rpcClient -> - EthereumPosGrpcUpstream( - id, - hash, - role, - chain, - client, - rpcClient, - nodeRating, - labels, - chainsConfig.resolve(chain.chainName), - headScheduler, - ) - }, - BlockchainType.BITCOIN to { chain, rpcClient -> - BitcoinGrpcUpstream(id, role, chain, client, rpcClient, labels, chainsConfig.resolve(chain.chainCode), headScheduler) - }, - ) - private fun getOrCreate(chain: Chain): UpstreamChangeEvent { val metrics = makeMetrics(chain) - val creator = creators.getValue(BlockchainType.from(chain)) + val creator = if (BlockchainType.from(chain) != BITCOIN) { + { ch: Chain, rpcClient: JsonRpcGrpcClient -> + GenericGrpcUpstream( + id, + hash, + role, + ch, + client, + rpcClient, + nodeRating, + labels, + chainsConfig.resolve(chain.chainName), + headScheduler, + ) + } + } else { + { ch: Chain, rpcClient: JsonRpcGrpcClient -> + BitcoinGrpcUpstream( + id, + role, + chain, + client, + rpcClient, + labels, + chainsConfig.resolve(chain.chainCode), + headScheduler, + ) + } + } return getOrCreate(chain, metrics, creator) } @@ -315,7 +310,7 @@ class GrpcUpstreams( val rpcClient = JsonRpcGrpcClient(client, chain, metrics) val created = creator(chain, rpcClient) known[chain] = created - if (created is Lifecycle) created.start() + created.start() UpstreamChangeEvent(chain, created, UpstreamChangeEvent.ChangeType.ADDED) } else { UpstreamChangeEvent(chain, current, UpstreamChangeEvent.ChangeType.REVALIDATED) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt index 45679d69..96ddafe0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt @@ -2,12 +2,32 @@ package io.emeraldpay.dshackle.upstream.starknet import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty +import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.foundation.ChainOptions.Options +import io.emeraldpay.dshackle.reader.JsonRpcReader +import io.emeraldpay.dshackle.upstream.CachingReader +import io.emeraldpay.dshackle.upstream.EgressSubscription +import io.emeraldpay.dshackle.upstream.EmptyEgressSubscription +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.LabelsDetector +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.NoopCachingReader +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamValidator +import io.emeraldpay.dshackle.upstream.calls.CallMethods +import io.emeraldpay.dshackle.upstream.generic.CachingReaderBuilder import io.emeraldpay.dshackle.upstream.generic.ChainSpecific +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream +import io.emeraldpay.dshackle.upstream.generic.LocalReader import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import org.springframework.cloud.sleuth.Tracer +import reactor.core.publisher.Mono +import reactor.core.scheduler.Scheduler import java.math.BigInteger import java.time.Instant @@ -30,7 +50,41 @@ object StarknetChainSpecific : ChainSpecific { ) } - override fun latestBlockRequest(): JsonRpcRequest = JsonRpcRequest("starknet_getBlockWithTxHashes", listOf("latest")) + override fun latestBlockRequest(): JsonRpcRequest = + JsonRpcRequest("starknet_getBlockWithTxHashes", listOf("latest")) + + override fun localReaderBuilder( + cachingReader: CachingReader, + methods: CallMethods, + head: Head, + ): Mono { + return Mono.just(LocalReader(methods)) + } + + override fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription { + return { _ -> EmptyEgressSubscription } + } + + override fun makeCachingReaderBuilder(tracer: Tracer): CachingReaderBuilder { + return { _, _, _ -> NoopCachingReader } + } + + override fun validator( + chain: Chain, + upstream: Upstream, + options: Options, + config: ChainConfig, + ): UpstreamValidator? { + return null + } + + override fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector? { + return null + } + + override fun subscriptionTopics(upstream: GenericUpstream): List { + return emptyList() + } } @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/CacheConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/CacheConfigReaderSpec.groovy index 9f6d962e..99305cf9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/CacheConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/CacheConfigReaderSpec.groovy @@ -47,14 +47,4 @@ class CacheConfigReaderSpec extends Specification { //later may be not null if we support something else besides Redis act == null } - - def "Local read disabled"() { - setup: - def config = this.class.getClassLoader().getResourceAsStream("cache-local-router-disabled.yaml") - when: - def act = reader.read(config) - - then: - !act.requestsCacheEnabled - } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy index 3f5bcec2..8ff0659c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy @@ -54,7 +54,7 @@ class NativeCallSpec extends Specification { ObjectMapper objectMapper = Global.objectMapper - def nativeCall(MultistreamHolder upstreams = null, ResponseSigner signer = null, Boolean enableCache = true, Boolean passthrough = false) { + def nativeCall(MultistreamHolder upstreams = null, ResponseSigner signer = null, Boolean passthrough = false) { if (upstreams == null) { upstreams = Stub(MultistreamHolder) @@ -65,7 +65,6 @@ class NativeCallSpec extends Specification { def config = new MainConfig() def cacheConfig = new CacheConfig() - cacheConfig.requestsCacheEnabled = enableCache config.cache = cacheConfig config.passthrough = passthrough @@ -77,7 +76,7 @@ class NativeCallSpec extends Specification { 1 * read(new JsonRpcRequest("eth_test", [])) >> Mono.just(new JsonRpcResponse("1".bytes, null)) } def upstream = Mock(Multistream) { - 1 * getLocalReader(_) >> Mono.just(routedApi) + 1 * getLocalReader() >> Mono.just(routedApi) } def nativeCall = nativeCall() @@ -98,7 +97,7 @@ class NativeCallSpec extends Specification { 1 * read(new JsonRpcRequest("eth_test", [])) >> Mono.error(new RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Test message")) } def upstream = Mock(Multistream) { - 1 * getLocalReader(_) >> Mono.just(routedApi) + 1 * getLocalReader() >> Mono.just(routedApi) } def nativeCall = nativeCall() diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy index 1ac4713c..2a0887b7 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy @@ -21,7 +21,7 @@ import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.test.MultistreamHolderMock import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.ethereum.EthereumEgressSubscription -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream +import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import io.emeraldpay.dshackle.upstream.signature.NoSigner import reactor.core.publisher.Flux import reactor.test.StepVerifier @@ -42,7 +42,7 @@ class NativeSubscribeSpec extends Specification { def subscribe = Mock(EthereumEgressSubscription) { 1 * it.subscribe("newHeads", null, _ as Selector.AnyLabelMatcher) >> Flux.just("{}") } - def up = Mock(EthereumPosMultiStream) { + def up = Mock(GenericMultistream) { 1 * it.tryProxySubscribe(_ as Selector.AnyLabelMatcher, call) >> null 1 * it.getEgressSubscription() >> subscribe } @@ -81,7 +81,7 @@ class NativeSubscribeSpec extends Specification { ok }, _ as Selector.AnyLabelMatcher) >> Flux.just("{}") } - def up = Mock(EthereumPosMultiStream) { + def up = Mock(GenericMultistream) { 1 * it.tryProxySubscribe(_ as Selector.AnyLabelMatcher, call) >> null 1 * it.getEgressSubscription() >> subscribe } @@ -104,7 +104,7 @@ class NativeSubscribeSpec extends Specification { .setChainValue(Chain.ETHEREUM__MAINNET.id) .setMethod("newHeads") .build() - def up = Mock(EthereumPosMultiStream) { + def up = Mock(GenericMultistream) { 1 * it.tryProxySubscribe(_ as Selector.AnyLabelMatcher, call) >> Flux.just("{}") 0 * it.getEgressSubscription() } diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy index 3ca52a22..75db54b0 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy @@ -16,17 +16,15 @@ */ package io.emeraldpay.dshackle.rpc -import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream +import io.emeraldpay.dshackle.test.GenericUpstreamMock import io.emeraldpay.dshackle.test.MultistreamHolderMock import io.emeraldpay.dshackle.test.TestingCommons -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.rpc.json.TransactionRefJson @@ -39,11 +37,9 @@ import java.time.Instant class StreamHeadSpec extends Specification { - ObjectMapper objectMapper = Global.objectMapper - def "Errors on unavailable chain"() { setup: - def upstreams = new MultistreamHolderMock(Chain.ETHEREUM__MAINNET, Stub(EthereumLikeRpcUpstream)) + def upstreams = new MultistreamHolderMock(Chain.ETHEREUM__MAINNET, Stub(GenericUpstream)) def streamHead = new StreamHead(upstreams) when: def flux = streamHead.add( @@ -80,7 +76,7 @@ class StreamHeadSpec extends Specification { .build() } - def upstream = new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def upstream = new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) def upstreams = new MultistreamHolderMock(Chain.ETHEREUM__MAINNET, upstream) def streamHead = new StreamHead(upstreams) when: diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy deleted file mode 100644 index 19db1801..00000000 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright (c) 2019 ETCDEV GmbH - * Copyright (c) 2020 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.test - -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.config.ChainsConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.UpstreamAvailability -import io.emeraldpay.dshackle.upstream.calls.* -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import org.jetbrains.annotations.NotNull -import io.emeraldpay.dshackle.foundation.ChainOptions -import org.reactivestreams.Publisher - -class EthereumRpcUpstreamMock extends EthereumLikeRpcUpstream { - EthereumHeadMock ethereumHeadMock - - - static CallMethods allMethods() { - new AggregatedCallMethods([ - new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET), - new DefaultBitcoinMethods(), - new DirectCallMethods(["eth_test"]) - ]) - } - - EthereumRpcUpstreamMock(@NotNull Chain chain, @NotNull Reader api) { - this(chain, api, allMethods()) - } - - EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api) { - this(id, chain, api, allMethods()) - } - - EthereumRpcUpstreamMock(@NotNull Chain chain, @NotNull Reader api, CallMethods methods) { - this("test", chain, api, methods) - } - - EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods) { - super(id, id.hashCode().byteValue(), chain, - ChainOptions.PartialOptions.getDefaults().buildOptions(), - UpstreamsConfig.UpstreamRole.PRIMARY, - methods, - new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()), - new ConnectorFactoryMock(api, new EthereumHeadMock()), - ChainsConfig.ChainConfig.default(), - false, - null - ) - this.ethereumHeadMock = this.getHead() as EthereumHeadMock - setLag(0) - setStatus(UpstreamAvailability.OK) - start() - } - - void nextBlock(BlockContainer block) { - this.ethereumHeadMock.nextBlock(block) - } - - void setBlocks(Publisher blocks) { - this.ethereumHeadMock.predefined = blocks - } - - @Override - String toString() { - return "Upstream mock ${getId()}" - } -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy index 6612c4e9..281aae18 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy @@ -2,7 +2,7 @@ package io.emeraldpay.dshackle.test import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.ethereum.EthereumIngressSubscription +import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.ethereum.NoEthereumIngressSubscription import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnector import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest @@ -47,7 +47,7 @@ class GenericConnectorMock implements GenericConnector { } @Override - EthereumIngressSubscription getIngressSubscription() { + IngressSubscription getIngressSubscription() { return NoEthereumIngressSubscription.DEFAULT } } \ No newline at end of file diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy similarity index 69% rename from src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy rename to src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy index d5558e68..abf68804 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy @@ -24,14 +24,14 @@ import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.calls.* -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import org.jetbrains.annotations.NotNull import org.reactivestreams.Publisher import io.emeraldpay.dshackle.foundation.ChainOptions -class EthereumPosRpcUpstreamMock extends EthereumLikeRpcUpstream { +class GenericUpstreamMock extends GenericUpstream { EthereumHeadMock ethereumHeadMock static CallMethods allMethods() { @@ -42,36 +42,40 @@ class EthereumPosRpcUpstreamMock extends EthereumLikeRpcUpstream { ]) } - EthereumPosRpcUpstreamMock(@NotNull Chain chain, @NotNull Reader api) { + GenericUpstreamMock(@NotNull Chain chain, @NotNull Reader api) { this(chain, api, allMethods()) } - EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, Map labels) { + GenericUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, Map labels) { this(id, chain, api, allMethods(), labels) } - EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api) { + GenericUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api) { this(id, chain, api, allMethods()) } - EthereumPosRpcUpstreamMock(@NotNull Chain chain, @NotNull Reader api, CallMethods methods) { + GenericUpstreamMock(@NotNull Chain chain, @NotNull Reader api, CallMethods methods) { this("test", chain, api, methods) } - EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods) { + GenericUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods) { this(id, chain, api, methods, Collections.emptyMap()) } - EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods, Map labels) { - super(id, (byte)id.hashCode(), chain, + GenericUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods, Map labels) { + super(id, + chain, + (byte)id.hashCode(), getOpts(), UpstreamsConfig.UpstreamRole.PRIMARY, methods, new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(labels)), - new ConnectorFactoryMock(api, new EthereumHeadMock()), ChainConfig.default(), - true, - null + new ConnectorFactoryMock(api, new EthereumHeadMock()), + null, + io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.&validator, + io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.&labelDetector, + io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.&subscriptionTopics, ) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy index 88b7e624..883e1f20 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy @@ -20,13 +20,13 @@ import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.generic.* import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumCachingReader -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream +import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific import org.jetbrains.annotations.NotNull import org.springframework.cloud.sleuth.brave.bridge.BraveTracer import reactor.core.scheduler.Schedulers @@ -42,13 +42,16 @@ class MultistreamHolderMock implements MultistreamHolder { Multistream addUpstream(@NotNull Chain chain, @NotNull Upstream up) { if (!upstreams.containsKey(chain)) { - if (BlockchainType.from(chain) == BlockchainType.EVM_POS) { - if (up instanceof EthereumPosMultiStream) { + if (BlockchainType.from(chain) == BlockchainType.ETHEREUM) { + if (up instanceof GenericMultistream) { upstreams[chain] = up - } else if (up instanceof EthereumLikeRpcUpstream) { - upstreams[chain] = new EthereumPosMultiStream( - chain, [up as EthereumLikeRpcUpstream], Caches.default(), - Schedulers.boundedElastic(), TestingCommons.tracerMock() + } else if (up instanceof GenericUpstream) { + upstreams[chain] = new GenericMultistream( + chain, [up as GenericUpstream], Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()) ) } else { throw new IllegalArgumentException("Unsupported upstream type ${up.class}") @@ -90,21 +93,24 @@ class MultistreamHolderMock implements MultistreamHolder { return upstreams.values().toList() } - static class EthereumMultistreamMock extends EthereumPosMultiStream { + static class EthereumMultistreamMock extends GenericMultistream { EthereumCachingReader customReader = null CallMethods customMethods = null Head customHead = null - EthereumMultistreamMock(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { - super(chain, upstreams, caches, Schedulers.boundedElastic(), new BraveTracer(null, null, null)) + EthereumMultistreamMock(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { + super(chain, upstreams, caches, Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(new BraveTracer(null, null, null)), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) } - EthereumMultistreamMock(@NotNull Chain chain, @NotNull List upstreams) { + EthereumMultistreamMock(@NotNull Chain chain, @NotNull List upstreams) { this(chain, upstreams, Caches.default()) } - EthereumMultistreamMock(@NotNull Chain chain, @NotNull EthereumLikeRpcUpstream upstream) { + EthereumMultistreamMock(@NotNull Chain chain, @NotNull GenericUpstream upstream) { this(chain, [upstream]) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy index 9619b918..80c73e39 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy @@ -28,10 +28,10 @@ import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream +import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson import io.emeraldpay.etherjar.domain.TransactionId @@ -54,44 +54,49 @@ class TestingCommons { return new TracerMock(null, null, null) } - static EthereumPosRpcUpstreamMock upstream() { - return new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, api()) + static GenericUpstreamMock upstream() { + return new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, api()) } - static EthereumPosRpcUpstreamMock upstream(String id) { - return new EthereumPosRpcUpstreamMock(id, Chain.ETHEREUM__MAINNET, api()) + static GenericUpstreamMock upstream(String id) { + return new GenericUpstreamMock(id, Chain.ETHEREUM__MAINNET, api()) } - static EthereumPosRpcUpstreamMock upstream(String id, String provider) { - return new EthereumPosRpcUpstreamMock(id, Chain.ETHEREUM__MAINNET, api(), Collections.singletonMap("provider", provider)) + static GenericUpstreamMock upstream(String id, String provider) { + return new GenericUpstreamMock(id, Chain.ETHEREUM__MAINNET, api(), Collections.singletonMap("provider", provider)) } - static EthereumPosRpcUpstreamMock upstream(String id, Reader api) { - return new EthereumPosRpcUpstreamMock(id, Chain.ETHEREUM__MAINNET, api) + static GenericUpstreamMock upstream(String id, Reader api) { + return new GenericUpstreamMock(id, Chain.ETHEREUM__MAINNET, api) } - static EthereumPosRpcUpstreamMock upstream(Reader api) { - return new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, api) + static GenericUpstreamMock upstream(Reader api) { + return new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, api) } - static EthereumPosRpcUpstreamMock upstream(Reader api, String method) { + static GenericUpstreamMock upstream(Reader api, String method) { return upstream(api, [method]) } - static EthereumPosRpcUpstreamMock upstream(Reader api, List methods) { - return new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, api, new DirectCallMethods(methods)) + static GenericUpstreamMock upstream(Reader api, List methods) { + return new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, api, new DirectCallMethods(methods)) } - static EthereumPosRpcUpstreamMock upstream(Reader api, CallMethods callMethods) { - return new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, api, callMethods) + static GenericUpstreamMock upstream(Reader api, CallMethods callMethods) { + return new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, api, callMethods) } static Multistream multistream(Reader api) { return multistream(upstream(api)) } - static Multistream multistream(EthereumPosRpcUpstreamMock up) { - return new EthereumPosMultiStream(Chain.ETHEREUM__MAINNET, [up], Caches.default(), Schedulers.boundedElastic(), tracerMock()).tap { + static Multistream multistream(GenericUpstreamMock up) { + return new GenericMultistream(Chain.ETHEREUM__MAINNET, [up], Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()), + ).tap { start() } } @@ -112,11 +117,17 @@ class TestingCommons { } static Multistream multistreamWithoutUpstreams(Chain chain) { - return new EthereumPosMultiStream(chain, [], emptyCaches().getCaches(chain), Schedulers.boundedElastic(), tracerMock()) + return new GenericMultistream(chain, [], emptyCaches().getCaches(chain), Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) } static Multistream multistreamClassicWithoutUpstreams(Chain chain) { - return new EthereumMultistream(chain, [], emptyCaches().getCaches(chain), Schedulers.boundedElastic(), tracerMock()) + return new GenericMultistream(chain, [], emptyCaches().getCaches(chain), Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) } static FileResolver fileResolver() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy deleted file mode 100644 index 37db9b97..00000000 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy +++ /dev/null @@ -1,253 +0,0 @@ -package io.emeraldpay.dshackle.upstream - -import kotlin.jvm.functions.Function1 -import org.jetbrains.annotations.NotNull -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import spock.lang.Specification - -import java.time.Duration -import java.util.function.Function - -class AbstractChainFeesSpec extends Specification { - - Function, List> extractTx = { it } - - def "Iterates N last blocks"() { - setup: - def ups = Mock(Multistream) { - 1 * getHead() >> Mock(Head) { - 1 * it.getCurrentHeight() >> 234 - } - } - def fees = new TestChainFees(5, ups, Stub(Function1)) - when: - def act = fees.usingBlocks(3).collectList().block(Duration.ofSeconds(1)).toSorted() - then: - act == [232L, 233L, 234L] - } - - def "Limits iteration to configured"() { - setup: - def ups = Mock(Multistream) { - 1 * getHead() >> Mock(Head) { - 1 * it.getCurrentHeight() >> 234 - } - } - def fees = new TestChainFees(3, ups, Stub(Function1)) - when: - def act = fees.usingBlocks(5).collectList().block(Duration.ofSeconds(1)).toSorted() - then: - act == [232L, 233L, 234L] - } - - def "TxAtPos 0 for empty list"() { - setup: - def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 0) - - when: - def act = txAt.get([]) - - then: - act == null - } - - def "TxAtPos 0 for single item list"() { - setup: - def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 0) - - when: - def act = txAt.get(["t_1"]) - - then: - act == "t_1" - } - - def "TxAtPos 0 for many item list"() { - setup: - def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 0) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_3" - } - - def "TxAtPos 1 for many item list"() { - setup: - def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 1) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_2" - } - - def "TxAtPos 2 for many item list"() { - setup: - def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 2) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_1" - } - - def "TxAtPos 5 for 3 item list"() { - setup: - def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 5) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_1" - } - - def "TxAtBottom for empty list"() { - setup: - def txAt = new AbstractChainFees.TxAtBottom, String>(extractTx) - - when: - def act = txAt.get([]) - - then: - act == null - } - - def "TxAtBottom for single item list"() { - setup: - def txAt = new AbstractChainFees.TxAtBottom, String>(extractTx) - - when: - def act = txAt.get(["t_1"]) - - then: - act == "t_1" - } - - def "TxAtBottom for multi item list"() { - setup: - def txAt = new AbstractChainFees.TxAtBottom, String>(extractTx) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_3" - } - - def "TxAtTop for empty list"() { - setup: - def txAt = new AbstractChainFees.TxAtTop, String>(extractTx) - - when: - def act = txAt.get([]) - - then: - act == null - } - - def "TxAtTop for single item list"() { - setup: - def txAt = new AbstractChainFees.TxAtTop, String>(extractTx) - - when: - def act = txAt.get(["t_1"]) - - then: - act == "t_1" - } - - def "TxAtTop for multi item list"() { - setup: - def txAt = new AbstractChainFees.TxAtTop, String>(extractTx) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_1" - } - - def "TxAtMiddle for empty list"() { - setup: - def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) - - when: - def act = txAt.get([]) - - then: - act == null - } - - def "TxAtMiddle for single item list"() { - setup: - def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) - - when: - def act = txAt.get(["t_1"]) - - then: - act == "t_1" - } - - def "TxAtMiddle for 3 item list"() { - setup: - def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) - - when: - def act = txAt.get(["t_1", "t_2", "t_3"]) - - then: - act == "t_2" - } - - def "TxAtMiddle for 4 item list"() { - setup: - def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) - - when: - def act = txAt.get(["t_1", "t_2", "t_3", "t_4"]) - - then: - act == "t_2" || act == "t_3" - } - - def "TxAtMiddle for 5 item list"() { - setup: - def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) - - when: - def act = txAt.get(["t_1", "t_2", "t_3", "t_4", "t_5"]) - - then: - act == "t_3" - } - - class TestChainFees extends AbstractChainFees { - - TestChainFees(int heightLimit, @NotNull Multistream upstreams, @NotNull Function1 extractTx) { - super(heightLimit, upstreams, extractTx) - } - - @Override - Mono readFeesAt(long height, @NotNull TxAt selector) { - return null - } - - @Override - Function feeAggregation(@NotNull Mode mode) { - return null - } - - @Override - Function getResponseBuilder() { - return null - } - } - -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolderSpec.groovy index 1ec7f3fe..ebc03b62 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolderSpec.groovy @@ -17,8 +17,7 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.startup.UpstreamChangeEvent -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock -import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock +import io.emeraldpay.dshackle.test.GenericUpstreamMock import io.emeraldpay.dshackle.test.TestingCommons import spock.lang.Specification @@ -27,7 +26,7 @@ class CurrentMultistreamHolderSpec extends Specification { def "add upstream"() { setup: def current = new CurrentMultistreamHolder(TestingCommons.defaultMultistreams()) - def up = new EthereumPosRpcUpstreamMock("test", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up = new GenericUpstreamMock("test", Chain.ETHEREUM__MAINNET, TestingCommons.api()) when: current.getUpstream(Chain.ETHEREUM__MAINNET).onUpstreamChange(new UpstreamChangeEvent(Chain.ETHEREUM__MAINNET, up, UpstreamChangeEvent.ChangeType.ADDED)) then: @@ -38,9 +37,9 @@ class CurrentMultistreamHolderSpec extends Specification { def "add multiple upstreams"() { setup: def current = new CurrentMultistreamHolder(TestingCommons.defaultMultistreams()) - def up1 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) - def up2 = new EthereumRpcUpstreamMock("test2", Chain.ETHEREUM_CLASSIC__MAINNET, TestingCommons.api()) - def up3 = new EthereumPosRpcUpstreamMock("test3", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up1 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up2 = new GenericUpstreamMock("test2", Chain.ETHEREUM_CLASSIC__MAINNET, TestingCommons.api()) + def up3 = new GenericUpstreamMock("test3", Chain.ETHEREUM__MAINNET, TestingCommons.api()) when: current.getUpstream(Chain.ETHEREUM__MAINNET).onUpstreamChange(new UpstreamChangeEvent(Chain.ETHEREUM__MAINNET, up1, UpstreamChangeEvent.ChangeType.ADDED)) current.getUpstream(Chain.ETHEREUM_CLASSIC__MAINNET).onUpstreamChange(new UpstreamChangeEvent(Chain.ETHEREUM_CLASSIC__MAINNET, up2, UpstreamChangeEvent.ChangeType.ADDED)) @@ -55,10 +54,10 @@ class CurrentMultistreamHolderSpec extends Specification { def "remove upstream"() { setup: def current = new CurrentMultistreamHolder(TestingCommons.defaultMultistreams()) - def up1 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) - def up2 = new EthereumRpcUpstreamMock("test2", Chain.ETHEREUM_CLASSIC__MAINNET, TestingCommons.api()) - def up3 = new EthereumPosRpcUpstreamMock("test3", Chain.ETHEREUM__MAINNET, TestingCommons.api()) - def up1_del = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up1 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up2 = new GenericUpstreamMock("test2", Chain.ETHEREUM_CLASSIC__MAINNET, TestingCommons.api()) + def up3 = new GenericUpstreamMock("test3", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up1_del = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) when: current.getUpstream(Chain.ETHEREUM__MAINNET).onUpstreamChange(new UpstreamChangeEvent(Chain.ETHEREUM__MAINNET, up1, UpstreamChangeEvent.ChangeType.ADDED)) current.getUpstream(Chain.ETHEREUM_CLASSIC__MAINNET).onUpstreamChange(new UpstreamChangeEvent(Chain.ETHEREUM_CLASSIC__MAINNET, up2, UpstreamChangeEvent.ChangeType.ADDED)) @@ -73,7 +72,7 @@ class CurrentMultistreamHolderSpec extends Specification { def "available after adding"() { setup: def current = new CurrentMultistreamHolder(TestingCommons.defaultMultistreams()) - def up1 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) + def up1 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api()) when: def act = current.isAvailable(Chain.ETHEREUM__MAINNET) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy index dd8812ef..780c849e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy @@ -24,7 +24,7 @@ import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.test.EthereumApiStub import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice import reactor.core.scheduler.Schedulers @@ -44,13 +44,15 @@ class FilteredApisSpec extends Specification { def "Verifies labels"() { setup: def i = 0 - List upstreams = [ + def cs = io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE + List upstreams = [ [test: "foo"], [test: "bar"], [test: "foo", test2: "baz"], [test: "foo"], [test: "baz"] ].collect { + def httpFactory = Mock(HttpFactory) { create(_, _) >> Stub(JsonRpcHttpReader) } @@ -64,18 +66,20 @@ class FilteredApisSpec extends Specification { Schedulers.boundedElastic(), Duration.ofSeconds(12) ) - new EthereumLikeRpcUpstream( + new GenericUpstream( "test", - (byte) 123, Chain.ETHEREUM__MAINNET, + (byte) 123, new ChainOptions.PartialOptions().buildOptions(), UpstreamsConfig.UpstreamRole.PRIMARY, ethereumTargets, new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)), - connectorFactory, ChainsConfig.ChainConfig.default(), - false, - null + connectorFactory, + null, + cs.&validator, + cs.&labelDetector, + cs.&subscriptionTopics, ) } def matcher = new Selector.LabelMatcher("test", ["foo"]) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy index 09328b41..d3410349 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy @@ -25,15 +25,17 @@ import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.quorum.AlwaysQuorum import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.startup.UpstreamChangeEvent -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock +import io.emeraldpay.dshackle.test.GenericUpstreamMock import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeRpcUpstream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream +import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream +import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.grpc.EthereumPosGrpcUpstream +import io.emeraldpay.dshackle.upstream.grpc.GenericGrpcUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.jetbrains.annotations.NotNull @@ -51,9 +53,13 @@ class MultistreamSpec extends Specification { def "Aggregates methods"() { setup: - def up1 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) - def up2 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test2", "eth_test3"])) - def aggr = new EthereumPosMultiStream(Chain.ETHEREUM__MAINNET, [up1, up2], Caches.default(), Schedulers.boundedElastic(), TestingCommons.tracerMock()) + def up1 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) + def up2 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test2", "eth_test3"])) + def aggr = new GenericMultistream(Chain.ETHEREUM__MAINNET, [up1, up2], Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) when: aggr.onUpstreamsUpdated() def act = aggr.getMethods() @@ -184,7 +190,11 @@ class MultistreamSpec extends Specification { def up1 = TestingCommons.upstream("test-1", "internal") def up2 = TestingCommons.upstream("test-2", "external") def up3 = TestingCommons.upstream("test-3", "external") - def multistream = new EthereumPosMultiStream(Chain.ETHEREUM__MAINNET, [up1, up2, up3], Caches.default(), Schedulers.boundedElastic(), TestingCommons.tracerMock()) + def multistream = new GenericMultistream(Chain.ETHEREUM__MAINNET, [up1, up2, up3], Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) expect: multistream.getHead(new Selector.LabelMatcher("provider", ["internal"])).is(up1.ethereumHeadMock) @@ -205,14 +215,14 @@ class MultistreamSpec extends Specification { .setMethod("newHeads") .build() - def up1 = Mock(EthereumPosGrpcUpstream) { + def up1 = Mock(GenericGrpcUpstream) { 1 * isGrpc() >> true - 1 * getId() >> "internal" + _ * getId() >> "internal" 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] 1 * proxySubscribe(call) >> Flux.just("{}") } - def up2 = Mock(EthereumPosGrpcUpstream) { - 1 * getId() >> "external" + def up2 = Mock(GenericUpstream) { + _ * getId() >> "external" 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "external"))] } def multiStream = new TestEthereumPosMultistream(Chain.ETHEREUM__MAINNET, [up1, up2], Caches.default()) @@ -235,9 +245,9 @@ class MultistreamSpec extends Specification { .setMethod("newHeads") .build() - def up2 = Mock(EthereumPosGrpcUpstream) { + def up2 = Mock(GenericUpstream) { 1 * isGrpc() >> false - 1 * getId() >> "2" + _ * getId() >> "2" 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] } def multiStream = new TestEthereumPosMultistream(Chain.ETHEREUM__MAINNET, [up2], Caches.default()) @@ -251,9 +261,13 @@ class MultistreamSpec extends Specification { def "Change ms methods based on upstream availability"() { setup: - def up1 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2", "eth_test3"])) - def up2 = new EthereumPosRpcUpstreamMock("test2", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) - def ms = new EthereumPosMultiStream(Chain.ETHEREUM__MAINNET, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), TestingCommons.tracerMock()) + def up1 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2", "eth_test3"])) + def up2 = new GenericUpstreamMock("test2", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) + def ms = new GenericMultistream(Chain.ETHEREUM__MAINNET, new ArrayList(), Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) when: ms.onUpstreamChange( new UpstreamChangeEvent(Chain.ETHEREUM__MAINNET, up1, UpstreamChangeEvent.ChangeType.ADDED) @@ -278,9 +292,13 @@ class MultistreamSpec extends Specification { def "Filter older blocks on multistream head"() { setup: - def up1 = new EthereumPosRpcUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2", "eth_test3"])) - def up2 = new EthereumPosRpcUpstreamMock("test2", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) - def ms = new EthereumPosMultiStream(Chain.ETHEREUM__MAINNET, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), TestingCommons.tracerMock()) + def up1 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2", "eth_test3"])) + def up2 = new GenericUpstreamMock("test2", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) + def ms = new GenericMultistream(Chain.ETHEREUM__MAINNET, new ArrayList(), Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) def head1 = createBlock(250, "0x0d050c785de17179f935b9b93aca09c442964cc59972c71ae68e74731448401b") def head2 = createBlock(270, "0x0d050c785de17179f935b9b93aca09c442964cc59972c71ae68e74731448402b") def head3 = createBlock(100, "0x0d050c785de17179f935b9b93aca09c442964cc59972c71ae68e74731448412b") @@ -311,7 +329,11 @@ class MultistreamSpec extends Specification { def up1 = TestingCommons.upstream("test-1", "internal") def up2 = TestingCommons.upstream("test-2", "external") def up3 = TestingCommons.upstream("test-3", "external") - def multistream = new EthereumPosMultiStream(Chain.ETHEREUM__MAINNET, [up1, up2, up3], Caches.default(), Schedulers.boundedElastic(), TestingCommons.tracerMock()) + def multistream = new GenericMultistream(Chain.ETHEREUM__MAINNET, [up1, up2, up3], Caches.default(), + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) def observer = multistream.lagObserver multistream.onUpstreamsUpdated() @@ -335,15 +357,19 @@ class MultistreamSpec extends Specification { .build() } - class TestEthereumPosMultistream extends EthereumPosMultiStream { + class TestEthereumPosMultistream extends GenericMultistream { - TestEthereumPosMultistream(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { - super(chain, upstreams, caches, Schedulers.boundedElastic(), TestingCommons.tracerMock()) + TestEthereumPosMultistream(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { + super(chain, upstreams, caches, + Schedulers.boundedElastic(), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.&localReaderBuilder, + StarknetChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) } @NotNull @Override - Mono> getLocalReader(boolean localEnabled) { + Mono> getLocalReader() { return null } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/ERC20BalanceSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/ERC20BalanceSpec.groovy deleted file mode 100644 index 2c684a7d..00000000 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/ERC20BalanceSpec.groovy +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock -import io.emeraldpay.dshackle.test.ReaderMock -import io.emeraldpay.dshackle.upstream.ApiSource -import io.emeraldpay.dshackle.upstream.FilteredApis -import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.etherjar.domain.Address -import io.emeraldpay.etherjar.erc20.ERC20Token -import io.emeraldpay.etherjar.hex.HexData -import io.emeraldpay.etherjar.rpc.json.TransactionCallJson -import spock.lang.Specification - -import java.time.Duration - -class ERC20BalanceSpec extends Specification { - - def "Gets balance from upstream"() { - setup: - ReaderMock api = new ReaderMock() - .with( - new JsonRpcRequest("eth_call", [ - new TransactionCallJson().tap { json -> - json.setTo(Address.from("0x54EedeAC495271d0F6B175474E89094C44Da98b9")) - json.setData(HexData.from("0x70a0823100000000000000000000000016c15c65ad00b6dfbcc2cb8a7b6c2d0103a3883b")) - }, - "latest" - ]), - JsonRpcResponse.ok('"0x0000000000000000000000000000000000000000000000000000001f28d72868"') - ) - - EthereumLikeRpcUpstream upstream = new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, api) - ERC20Token token = new ERC20Token(Address.from("0x54EedeAC495271d0F6B175474E89094C44Da98b9")) - ERC20Balance query = new ERC20Balance() - - when: - def act = query.getBalance(upstream, token, Address.from("0x16c15c65ad00b6dfbcc2cb8a7b6c2d0103a3883b")) - .block(Duration.ofSeconds(1)) - - then: - act.toLong() == 0x1f28d72868 - } - - def "Gets balance from api source"() { - setup: - ReaderMock api = new ReaderMock() - .with( - new JsonRpcRequest("eth_call", [ - new TransactionCallJson().tap { json -> - json.setTo(Address.from("0x54EedeAC495271d0F6B175474E89094C44Da98b9")) - json.setData(HexData.from("0x70a0823100000000000000000000000016c15c65ad00b6dfbcc2cb8a7b6c2d0103a3883b")) - }, - "latest" - ]), - JsonRpcResponse.ok('"0x0000000000000000000000000000000000000000000000000000001f28d72868"') - ) - - EthereumLikeRpcUpstream upstream = new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, api) - ERC20Token token = new ERC20Token(Address.from("0x54EedeAC495271d0F6B175474E89094C44Da98b9")) - ERC20Balance query = new ERC20Balance() - - ApiSource apiSource = new FilteredApis( - Chain.ETHEREUM__MAINNET, [upstream], Selector.empty - ) - - when: - def act = query.getBalance(apiSource, token, Address.from("0x16c15c65ad00b6dfbcc2cb8a7b6c2d0103a3883b")) - .block(Duration.ofSeconds(1)) - - then: - act.toLong() == 0x1f28d72868 - } -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy index 33c8f06f..ce51e881 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.test.TestingCommons +import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.hex.Hex32 @@ -28,7 +29,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read empty logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([:]) @@ -39,7 +40,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read single address logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ address: "0x829bd824b016326a401d083b33d092293333a830" @@ -64,7 +65,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "ignores invalid address for logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ address: "829bd824b016326a401d083b33d092293333a830" @@ -77,7 +78,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read multi address logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ address: ["0x829bd824b016326a401d083b33d092293333a830", "0x401d083b33d092293333a83829bd824b016326a0"] @@ -93,7 +94,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read single topic logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ topics: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" @@ -118,7 +119,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read invalid topic for request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ topics: [ @@ -136,7 +137,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read multi topic logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ topics: [ @@ -155,7 +156,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def "read full logs request"() { setup: - def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe = new EthereumEgressSubscription(TestingCommons.emptyMultistream() as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) when: def act = ethereumSubscribe.readLogsRequest([ address: "0x298d492e8c1d909d3f63bc4a36c66c64acb3d695", @@ -180,7 +181,7 @@ class EthereumEgressSubscriptionSpec extends Specification { def up1 = TestingCommons.upstream("test") up1.getConnectorMock().setLiveness(Flux.just(false)) - def ethereumSubscribe1 = new EthereumEgressSubscription(TestingCommons.multistream(up1) as EthereumPosMultiStream, Schedulers.boundedElastic(), null) + def ethereumSubscribe1 = new EthereumEgressSubscription(TestingCommons.multistream(up1) as GenericMultistream, Schedulers.boundedElastic(), null) then: ethereumSubscribe1.getAvailableTopics() == [] when: @@ -188,7 +189,7 @@ class EthereumEgressSubscriptionSpec extends Specification { up2.getConnectorMock().setLiveness(Flux.just(true)) up2.stop() up2.start() - def ethereumSubscribe2 = new EthereumEgressSubscription(TestingCommons.multistream(up2) as EthereumPosMultiStream, Schedulers.boundedElastic(), null) + def ethereumSubscribe2 = new EthereumEgressSubscription(TestingCommons.multistream(up2) as GenericMultistream, Schedulers.boundedElastic(), null) then: ethereumSubscribe2.getAvailableTopics().toSet() == [EthereumEgressSubscription.METHOD_LOGS, EthereumEgressSubscription.METHOD_NEW_HEADS].toSet() when: @@ -196,7 +197,7 @@ class EthereumEgressSubscriptionSpec extends Specification { up3.getConnectorMock().setLiveness(Flux.just(true)) up3.stop() up3.start() - def ethereumSubscribe3 = new EthereumEgressSubscription(TestingCommons.multistream(up3) as EthereumPosMultiStream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + def ethereumSubscribe3 = new EthereumEgressSubscription(TestingCommons.multistream(up3) as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) then: ethereumSubscribe3.getAvailableTopics().toSet() == [EthereumEgressSubscription.METHOD_LOGS, EthereumEgressSubscription.METHOD_NEW_HEADS, EthereumEgressSubscription.METHOD_PENDING_TXES].toSet() diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy deleted file mode 100644 index a93a66f9..00000000 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot -import io.emeraldpay.etherjar.domain.Wei -import spock.lang.Specification - -class EthereumLegacyFeesSpec extends Specification { - - def "Extract fee from"() { - setup: - def block = new BlockJson() - // 0x75cc01873a9818bf426a8b23d83450bf18530a822fd4fe9e86a416a5554176a6 - def tx = new TransactionJsonSnapshot().tap { - it.gasPrice = Wei.ofUnits(8, Wei.Unit.GWEI) - } - - def fees = new EthereumLegacyFees(Stub(EthereumMultistream), Stub(EthereumCachingReader), 10) - when: - def act = fees.extractFee(block, tx) - then: - act.priority == Wei.ofUnits(8, Wei.Unit.GWEI) - act.max == Wei.ofUnits(8, Wei.Unit.GWEI) - act.base == Wei.ZERO - } -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy index 7f37ce85..0807ce58 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy @@ -30,8 +30,7 @@ class EthereumLocalReaderSpec extends Specification { TestingCommons.tracerMock() ), methods, - new EmptyHead(), - true + new EmptyHead() ) when: def act = router.read(new JsonRpcRequest("eth_coinbase", [])).block(Duration.ofSeconds(1)) @@ -50,8 +49,7 @@ class EthereumLocalReaderSpec extends Specification { TestingCommons.tracerMock() ), methods, - new EmptyHead(), - true + new EmptyHead() ) when: def act = router.read(new JsonRpcRequest("eth_getTransactionByHash", ["test"], 10)) @@ -75,7 +73,7 @@ class EthereumLocalReaderSpec extends Specification { } } def methods = new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) - def router = new EthereumLocalReader(reader, methods, head, true) + def router = new EthereumLocalReader(reader, methods, head) when: def act = router.getBlockByNumber(["latest", false]) @@ -103,7 +101,7 @@ class EthereumLocalReaderSpec extends Specification { } } def methods = new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) - def router = new EthereumLocalReader(reader, methods, head, true) + def router = new EthereumLocalReader(reader, methods, head) when: def act = router.getBlockByNumber(["earliest", false]) @@ -131,7 +129,7 @@ class EthereumLocalReaderSpec extends Specification { } } def methods = new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) - def router = new EthereumLocalReader(reader, methods, head, true) + def router = new EthereumLocalReader(reader, methods, head) when: def act = router.getBlockByNumber(["0x123ef", false]) @@ -155,7 +153,7 @@ class EthereumLocalReaderSpec extends Specification { _ * blocksByHeightAsCont() >> new EmptyReader<>() } def methods = new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) - def router = new EthereumLocalReader(reader, methods, head, true) + def router = new EthereumLocalReader(reader, methods, head) when: def act = router.getBlockByNumber(["0x0", true]) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy deleted file mode 100644 index 5deb8f7d..00000000 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Copyright (c) 2021 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.ethereum - -import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.ChainFees -import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot -import io.emeraldpay.etherjar.domain.TransactionId -import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionRefJson -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import spock.lang.Specification - -import java.time.Duration - -class EthereumPriorityFeesSpec extends Specification { - - def "Extract fee from EIP1559 tx"() { - setup: - // 13756007 - def block = new BlockJson().tap { - it.baseFeePerGas = new Wei(104197355513) - } - // 0x5da50f35a51e56ecd4313417b1c30f9c088222f3f8763701effe14f3dd18b6cc - def tx = new TransactionJsonSnapshot().tap { - it.type = 2 - it.maxFeePerGas = Wei.ofUnits(999, Wei.Unit.GWEI) - it.maxPriorityFeePerGas = Wei.ofUnits(5.0001, Wei.Unit.GWEI) - } - - def fees = new EthereumPriorityFees(Stub(EthereumMultistream), Stub(EthereumCachingReader), 10) - when: - def act = fees.extractFee(block, tx) - then: - act.priority == Wei.ofUnits(5.0001, Wei.Unit.GWEI) - act.max == Wei.ofUnits(999, Wei.Unit.GWEI) - act.base == new Wei(104197355513) - } - - def "Extract fee from legacy tx"() { - setup: - // 13756007 - def block = new BlockJson().tap { - it.baseFeePerGas = new Wei(104197355513) - } - // 0x1f507982bef0f11a8304287d41f228b5f1dda1114a446ee781c3d95ef4a7b891 - def tx = new TransactionJsonSnapshot().tap { - it.type = 0 - // 109.564020111 Gwei - it.gasPrice = Wei.from("0x198286458f") - } - - def fees = new EthereumPriorityFees(Stub(EthereumMultistream), Stub(EthereumCachingReader), 10) - when: - def act = fees.extractFee(block, tx) - then: - // as difference between base minimum and actually paid - act.priority == Wei.ofUnits(5.366664598, Wei.Unit.GWEI) - act.max == Wei.from("0x198286458f") - act.base == new Wei(104197355513) - } - - def "Calculates average fee"() { - setup: - def inputs = [ - new EthereumFees.EthereumFee(Wei.ofEthers(1), Wei.ofEthers(0.5), Wei.ofEthers(0.75), Wei.ofEthers(0.5)), - new EthereumFees.EthereumFee(Wei.ofEthers(0.75), Wei.ofEthers(0.5), Wei.ofEthers(0.75), Wei.ofEthers(0.5)), - new EthereumFees.EthereumFee(Wei.ofEthers(0.6), Wei.ofEthers(0.2), Wei.ofEthers(0.6), Wei.ofEthers(0.5)), - ] - def fees = new EthereumPriorityFees(Stub(EthereumMultistream), Stub(EthereumCachingReader), 10) - when: - def act = Flux.fromIterable(inputs) - .transform(fees.feeAggregation(ChainFees.Mode.AVG_LAST)) - .next().block(Duration.ofSeconds(1)) - then: - act.priority == Wei.ofEthers(0.4) // 0.5 + 0.5 + 0.2 - act.paid == Wei.ofEthers(0.7) // 0.75 + 0.75 + 0.6 - } - - def "Estimate"() { - setup: - def block1 = new BlockJson().tap { - it.baseFeePerGas = new Wei(92633661632) - it.transactions = [ - new TransactionRefJson(TransactionId.from("0x00000000fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), - new TransactionRefJson(TransactionId.from("0x11111111fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), - new TransactionRefJson(TransactionId.from("0x22222222fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), - ] - } - def block2 = new BlockJson().tap { - it.baseFeePerGas = new Wei(104197355513) - it.transactions = [ - new TransactionRefJson(TransactionId.from("0x33333333fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), - new TransactionRefJson(TransactionId.from("0x44444444fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), - new TransactionRefJson(TransactionId.from("0x55555555fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), - ] - } - def tx1 = new TransactionJsonSnapshot().tap { - it.type = 2 - it.maxFeePerGas = Wei.ofUnits(150, Wei.Unit.GWEI) - it.maxPriorityFeePerGas = Wei.ofUnits(3, Wei.Unit.GWEI) - } - def tx2 = new TransactionJsonSnapshot().tap { - it.type = 2 - it.maxFeePerGas = Wei.ofUnits(200, Wei.Unit.GWEI) - it.maxPriorityFeePerGas = Wei.ofUnits(6, Wei.Unit.GWEI) - } - - def ups = Mock(EthereumMultistream) { - 1 * getHead() >> Mock(Head) { - 1 * getCurrentHeight() >> 13756007 - } - } - def reader = Mock(EthereumCachingReader) { - _ * it.blocksByHeightParsed() >> Mock(Reader) { - 1 * it.read(13756006) >> Mono.just(block1) - 1 * it.read(13756007) >> Mono.just(block2) - } - _ * it.txByHash() >> Mock(Reader) { - 1 * it.read(TransactionId.from("0x22222222fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")) >> Mono.just(tx1) - 1 * it.read(TransactionId.from("0x55555555fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")) >> Mono.just(tx2) - } - } - def fees = new EthereumPriorityFees(ups, reader, 10) - when: - def act = fees.estimate(ChainFees.Mode.AVG_LAST, 2).block(Duration.ofSeconds(1)) - - then: - act.hasEthereumExtended() - act.ethereumExtended.priority == "4500000000" - act.ethereumExtended.max == "175000000000" - act.ethereumExtended.expect == "102915508572" - } -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidatorSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidatorSpec.groovy index aa847c09..a65f23d8 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidatorSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidatorSpec.groovy @@ -24,6 +24,7 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.hex.HexData import io.emeraldpay.etherjar.rpc.RpcResponseError @@ -37,16 +38,20 @@ import java.time.Duration import static io.emeraldpay.dshackle.Chain.ETHEREUM__MAINNET import static io.emeraldpay.dshackle.Chain.OPTIMISM__MAINNET import static io.emeraldpay.dshackle.upstream.UpstreamAvailability.* -import static io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR -import static io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR -import static io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator.ValidateUpstreamSettingsResult.UPSTREAM_VALID +import static io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR +import static io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR +import static io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult.UPSTREAM_VALID import static java.util.Collections.emptyList +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig class EthereumUpstreamValidatorSpec extends Specification { + def conf = ChainConfig.defaultWithContract("0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") + def "Resolve to final availability"() { setup: - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, Stub(EthereumLikeUpstream), ChainOptions.PartialOptions.getDefaults().buildOptions()) + + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, Stub(Upstream), ChainOptions.PartialOptions.getDefaults().buildOptions(), conf) expect: validator.resolve(Tuples.of(sync, peers)) == exp where: @@ -67,8 +72,8 @@ class EthereumUpstreamValidatorSpec extends Specification { def options = ChainOptions.PartialOptions.getDefaults().tap { it.validateSyncing = false }.buildOptions() - def up = Mock(EthereumLikeUpstream) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def up = Mock(Upstream) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validateSyncing().block(Duration.ofSeconds(1)) @@ -87,7 +92,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("eth_syncing", [], false) } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validateSyncing().block(Duration.ofSeconds(1)) @@ -100,7 +105,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def options = ChainOptions.PartialOptions.getDefaults().tap { it.validateSyncing = true }.buildOptions() - def up = Mock(EthereumLikeUpstream) { + def up = Mock(Upstream) { 2 * getIngressReader() >> Mock(Reader) { reader -> 2 * reader.read(_) >>> [ Mono.just(new JsonRpcResponse('true'.getBytes(), null)), @@ -112,7 +117,7 @@ class EthereumUpstreamValidatorSpec extends Specification { 1 * head.onSyncingNode(false) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validateSyncing().block(Duration.ofSeconds(1)) @@ -132,7 +137,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("eth_syncing", [], [startingBlock: 100, currentBlock: 50]) } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validateSyncing().block(Duration.ofSeconds(1)) @@ -150,7 +155,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("eth_syncing", [], new RpcResponseError(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unavailable")) } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validateSyncing().block(Duration.ofSeconds(1)) @@ -164,8 +169,8 @@ class EthereumUpstreamValidatorSpec extends Specification { it.validatePeers = false it.minPeers = 10 }.buildOptions() - def up = Mock(EthereumLikeUpstream) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def up = Mock(Upstream) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validatePeers().block(Duration.ofSeconds(1)) @@ -180,8 +185,8 @@ class EthereumUpstreamValidatorSpec extends Specification { it.validatePeers = true it.minPeers = 0 }.buildOptions() - def up = Mock(EthereumLikeUpstream) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def up = Mock(Upstream) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validatePeers().block(Duration.ofSeconds(1)) @@ -201,7 +206,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("net_peerCount", [], "0x5") } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validatePeers().block(Duration.ofSeconds(1)) @@ -220,7 +225,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("net_peerCount", [], "0xa") } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validatePeers().block(Duration.ofSeconds(1)) @@ -239,7 +244,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("net_peerCount", [], "0xff") } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validatePeers().block(Duration.ofSeconds(1)) @@ -258,7 +263,7 @@ class EthereumUpstreamValidatorSpec extends Specification { answer("net_peerCount", [], new RpcResponseError(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unavailable")) } ) - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validatePeers().block(Duration.ofSeconds(1)) @@ -272,7 +277,7 @@ class EthereumUpstreamValidatorSpec extends Specification { it.validateCalllimit = false it.validateChain = false }.buildOptions() - def up = Mock(EthereumLikeUpstream) { + def up = Mock(Upstream) { 2 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_blockNumber", [])) >> Mono.just(new JsonRpcResponse('"0x10ff9be"'.getBytes(), null)) @@ -280,7 +285,7 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options) + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) when: def act = validator.validateUpstreamSettingsOnStartup() @@ -293,7 +298,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def options = ChainOptions.PartialOptions.getDefaults().tap { it.validateChain = false }.buildOptions() - def up = Mock(EthereumLikeRpcUpstream) { + def up = Mock(Upstream) { 3 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_call", [new TransactionCallJson( Address.from("0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96"), @@ -304,8 +309,8 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") - + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) +//"0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96 when: def act = validator.validateUpstreamSettingsOnStartup() then: @@ -317,7 +322,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def options = ChainOptions.PartialOptions.getDefaults().tap { it.validateChain = false }.buildOptions() - def up = Mock(EthereumLikeRpcUpstream) { + def up = Mock(Upstream) { 3 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_call", [new TransactionCallJson( Address.from("0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96"), @@ -328,8 +333,8 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") - + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) +// "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96" when: def act = validator.validateUpstreamSettingsOnStartup() then: @@ -341,7 +346,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def options = ChainOptions.PartialOptions.getDefaults().tap { it.validateCalllimit = false }.buildOptions() - def up = Mock(EthereumLikeRpcUpstream) { + def up = Mock(Upstream) { 4 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_chainId", emptyList())) >> Mono.just(new JsonRpcResponse('"0x1"'.getBytes(), null)) 1 * read(new JsonRpcRequest("net_version", emptyList())) >> Mono.just(new JsonRpcResponse('"1"'.getBytes(), null)) @@ -350,8 +355,8 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") - + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) +// "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96" when: def act = validator.validateUpstreamSettingsOnStartup() then: @@ -363,7 +368,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def options = ChainOptions.PartialOptions.getDefaults().tap { it.validateCalllimit = false }.buildOptions() - def up = Mock(EthereumLikeRpcUpstream) { + def up = Mock(Upstream) { 4 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_chainId", emptyList())) >> Mono.just(new JsonRpcResponse('"0x1"'.getBytes(), null)) 1 * read(new JsonRpcRequest("net_version", emptyList())) >> Mono.just(new JsonRpcResponse('"1"'.getBytes(), null)) @@ -372,8 +377,8 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(OPTIMISM__MAINNET, up, options, "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") - + def validator = new EthereumUpstreamValidator(OPTIMISM__MAINNET, up, options, conf) +// "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96" when: def act = validator.validateUpstreamSettingsOnStartup() then: @@ -383,7 +388,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def "Upstream is valid if all setting are valid"() { setup: def options = ChainOptions.PartialOptions.getDefaults().buildOptions() - def up = Mock(EthereumLikeRpcUpstream) { + def up = Mock(Upstream) { 5 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_chainId", emptyList())) >> Mono.just(new JsonRpcResponse('"0x1"'.getBytes(), null)) 1 * read(new JsonRpcRequest("net_version", emptyList())) >> Mono.just(new JsonRpcResponse('"1"'.getBytes(), null)) @@ -396,8 +401,8 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") - + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) +// "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96" when: def act = validator.validateUpstreamSettingsOnStartup() then: @@ -407,7 +412,7 @@ class EthereumUpstreamValidatorSpec extends Specification { def "Upstream is not valid if there are errors"() { setup: def options = ChainOptions.PartialOptions.getDefaults().buildOptions() - def up = Mock(EthereumLikeRpcUpstream) { + def up = Mock(Upstream) { 5 * getIngressReader() >> Mock(Reader) { 1 * read(new JsonRpcRequest("eth_chainId", emptyList())) >> Mono.just(new JsonRpcResponse(null, new JsonRpcError(1, "Too long"))) 1 * read(new JsonRpcRequest("net_version", emptyList())) >> Mono.just(new JsonRpcResponse(null, new JsonRpcError(1, "Too long"))) @@ -420,8 +425,8 @@ class EthereumUpstreamValidatorSpec extends Specification { Mono.just(new JsonRpcResponse('"result"'.getBytes(), null)) } } - def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96") - + def validator = new EthereumUpstreamValidator(ETHEREUM__MAINNET, up, options, conf) +// "0x32268860cAAc2948Ab5DdC7b20db5a420467Cf96" when: def act = validator.validateUpstreamSettingsOnStartup() then: diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHeadSpec.groovy index 014cb7d4..1cd1bfc5 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHeadSpec.groovy @@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock +import io.emeraldpay.dshackle.test.GenericUpstreamMock import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream @@ -41,7 +41,7 @@ import java.time.temporal.ChronoUnit class EthereumWsHeadSpec extends Specification { BlockHash parent = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200") - DefaultUpstream upstream = new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) + DefaultUpstream upstream = new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) def "Fetch block"() { setup: diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy index 8f9c2722..0d1d35da 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy @@ -1,7 +1,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock +import io.emeraldpay.dshackle.test.GenericUpstreamMock import io.emeraldpay.dshackle.test.MockWSServer import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.DefaultUpstream @@ -15,7 +15,7 @@ import spock.lang.Specification import java.time.Duration class WsConnectionImplRealSpec extends Specification { - DefaultUpstream upstream = new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) + DefaultUpstream upstream = new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) static SLEEP = 500 diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy index 5f13d5c3..c46f61d3 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy @@ -17,7 +17,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global -import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock +import io.emeraldpay.dshackle.test.GenericUpstreamMock import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest @@ -32,7 +32,7 @@ import spock.lang.Specification import java.time.Duration class WsConnectionImplSpec extends Specification { - DefaultUpstream upstream = new EthereumPosRpcUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) + DefaultUpstream upstream = new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, TestingCommons.api()) def "Makes a RPC call"() { setup: diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy index 3e567eec..ef1ce1b9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy @@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream +import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson @@ -38,7 +38,7 @@ class ConnectBlockUpdatesSpec extends Specification { def "Extracts updates"() { setup: - def connectBlockUpdates = new ConnectBlockUpdates(Stub(EthereumMultistream), Schedulers.boundedElastic()) + def connectBlockUpdates = new ConnectBlockUpdates(Stub(GenericMultistream), Schedulers.boundedElastic()) def block = BlockContainer.from(new BlockJson().tap { hash = BlockHash.from("0xe5be2159b2b7daf6b126babdcbaa349da668b92d6b8c7db1350fd527fec4885c") number = 13412871 @@ -72,7 +72,7 @@ class ConnectBlockUpdatesSpec extends Specification { def "Produce DROP updates for replaced block"() { setup: - def connectBlockUpdates = new ConnectBlockUpdates(Stub(EthereumMultistream), Schedulers.boundedElastic()) + def connectBlockUpdates = new ConnectBlockUpdates(Stub(GenericMultistream), Schedulers.boundedElastic()) def block = BlockContainer.from(new BlockJson().tap { hash = BlockHash.from("0xe5be2159b2b7daf6b126babdcbaa349da668b92d6b8c7db1350fd527fec4885c") number = 13412871 @@ -106,7 +106,7 @@ class ConnectBlockUpdatesSpec extends Specification { def "Gets prev version if available"() { setup: - def connectBlockUpdates = new ConnectBlockUpdates(Stub(EthereumMultistream), Schedulers.boundedElastic()) + def connectBlockUpdates = new ConnectBlockUpdates(Stub(GenericMultistream), Schedulers.boundedElastic()) def block1 = BlockContainer.from(new BlockJson().tap { hash = BlockHash.from("0xe5be2159b2b7daf6b126babdcbaa349da668b92d6b8c7db1350fd527fec4885c") number = 13412871 @@ -170,7 +170,7 @@ class ConnectBlockUpdatesSpec extends Specification { def "Marks old txes as dropped before producing a new version of same block"() { setup: - def connectBlockUpdates = new ConnectBlockUpdates(Stub(EthereumMultistream), Schedulers.boundedElastic()) + def connectBlockUpdates = new ConnectBlockUpdates(Stub(GenericMultistream), Schedulers.boundedElastic()) def block1 = BlockContainer.from(new BlockJson().tap { hash = BlockHash.from("0xe5be2159b2b7daf6b126babdcbaa349da668b92d6b8c7db1350fd527fec4885c") number = 13412871 @@ -232,7 +232,7 @@ class ConnectBlockUpdatesSpec extends Specification { def head = Mock(Head) { 1 * getFlux() >> Flux.never() } - def up = Mock(EthereumMultistream) { + def up = Mock(GenericMultistream) { 1 * getEnrichedHead(Selector.empty) >> head } def connectBlockUpdates = new ConnectBlockUpdates(up, Schedulers.boundedElastic()) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy index 0de35e85..86bca59b 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy @@ -16,7 +16,6 @@ package io.emeraldpay.dshackle.upstream.ethereum.subscribe import io.emeraldpay.dshackle.test.TestingCommons -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.domain.BlockHash @@ -91,7 +90,7 @@ class ConnectLogsSpec extends Specification { def "Filter is empty"() { setup: - def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic()) + def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream(), Schedulers.boundedElastic()) when: def input = Flux.fromIterable([ log1, log2, log3, log4 @@ -109,7 +108,7 @@ class ConnectLogsSpec extends Specification { def "Filter by address"() { setup: - def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic()) + def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream(), Schedulers.boundedElastic()) when: def input = Flux.fromIterable([ log1, log2 @@ -124,7 +123,7 @@ class ConnectLogsSpec extends Specification { def "Filter by topic"() { setup: - def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic()) + def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream(), Schedulers.boundedElastic()) when: def input = Flux.fromIterable([ log1, log2, log3, log4 @@ -141,7 +140,7 @@ class ConnectLogsSpec extends Specification { def "Filter by address and topic"() { setup: - def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream() as EthereumPosMultiStream, Schedulers.boundedElastic()) + def connectLogs = new ConnectLogs(TestingCommons.emptyMultistream(), Schedulers.boundedElastic()) when: def input = Flux.fromIterable([ log1, log2, log3, log4 diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeadsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeadsSpec.groovy index 998640c3..c7c3d09a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeadsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeadsSpec.groovy @@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.upstream.ethereum.subscribe import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream +import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import reactor.core.publisher.Flux import reactor.core.scheduler.Schedulers import reactor.test.StepVerifier @@ -18,7 +18,7 @@ class ConnectNewHeadsSpec extends Specification { TestingCommons.blockForEthereum(100) ]) } - def up = Mock(EthereumMultistream) { + def up = Mock(GenericMultistream) { 1 * getHead(Selector.empty) >> head } ConnectNewHeads connectNewHeads = new ConnectNewHeads(up, Schedulers.boundedElastic()) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy deleted file mode 100644 index c181fbe1..00000000 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy +++ /dev/null @@ -1,264 +0,0 @@ -/** - * Copyright (c) 2019 ETCDEV GmbH - * Copyright (c) 2020 EmeraldPay, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.emeraldpay.dshackle.upstream.grpc - -import com.fasterxml.jackson.databind.ObjectMapper -import com.google.protobuf.ByteString -import io.emeraldpay.api.proto.BlockchainGrpc -import io.emeraldpay.api.proto.BlockchainOuterClass -import io.emeraldpay.api.proto.Common -import io.emeraldpay.dshackle.Chain -import io.emeraldpay.dshackle.Global -import io.emeraldpay.dshackle.config.ChainsConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig -import io.emeraldpay.dshackle.data.BlockId -import io.emeraldpay.dshackle.test.MockGrpcServer -import io.emeraldpay.dshackle.test.TestingCommons -import io.emeraldpay.dshackle.upstream.BuildInfo -import io.emeraldpay.dshackle.upstream.UpstreamAvailability -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient -import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics -import io.emeraldpay.etherjar.domain.BlockHash -import io.grpc.stub.StreamObserver -import io.micrometer.core.instrument.Counter -import io.micrometer.core.instrument.Timer -import reactor.core.scheduler.Schedulers -import spock.lang.Specification - -import java.time.Duration -import java.time.Instant -import java.util.concurrent.CompletableFuture - -class EthereumGrpcUpstreamSpec extends Specification { - - MockGrpcServer mockServer = new MockGrpcServer() - ObjectMapper objectMapper = Global.objectMapper - RpcMetrics metrics = new RpcMetrics( - Timer.builder("test1").register(TestingCommons.meterRegistry), - Counter.builder("test2").register(TestingCommons.meterRegistry) - ) - BlockHash parent = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - - def hash = (byte)123 - def buildInfo = new BuildInfo("v0.0.1-test") - - def "Subscribe to head"() { - setup: - def callData = [:] - def chain = Chain.ETHEREUM__MAINNET - def api = TestingCommons.api() - def block1 = new BlockJson().with { - it.number = 650246 - it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) - it.timestamp = Instant.now() - it.parentHash = parent - return it - } - api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) - def client = mockServer.clientForServer(new BlockchainGrpc.BlockchainImplBase() { - @Override - void nativeCall(BlockchainOuterClass.NativeCallRequest request, StreamObserver responseObserver) { - api.nativeCall(request, responseObserver) - } - - @Override - void subscribeHead(Common.Chain request, StreamObserver responseObserver) { - callData.chain = request.getTypeValue() - responseObserver.onNext( - BlockchainOuterClass.ChainHead.newBuilder() - .setBlockId(block1.hash.toHex().substring(2)) - .setHeight(block1.number) - .setParentBlockId(parent.toHex().substring(2)) - .setWeight(ByteString.copyFrom(block1.totalDifficulty.toByteArray())) - .build() - ) - } - }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null, ChainsConfig.ChainConfig.default(), Schedulers.boundedElastic()) - upstream.setLag(0) - upstream.update( - BlockchainOuterClass.DescribeChain.newBuilder() - .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) - .addAllSupportedMethods(["eth_getBlockByHash"]) - .build(), - BlockchainOuterClass.BuildInfo.newBuilder() - .setVersion(buildInfo.version) - .build(), - ) - when: - new Thread({ Thread.sleep(50); upstream.head.start() }).start() - def h = upstream.head.getFlux().next().block(Duration.ofSeconds(1)) - then: - callData.chain == Chain.ETHEREUM__MAINNET.id - upstream.status == UpstreamAvailability.OK - upstream.getBuildInfo() == buildInfo - h.hash == BlockId.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - } - - def "Follows difficulty, ignores less difficult"() { - setup: - def api = TestingCommons.api() - def block1 = new BlockJson().with { - it.number = 650246 - it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) - it.timestamp = Instant.now() - it.parentHash = parent - return it - } - def block2 = new BlockJson().with { - it.number = 650247 - it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") - it.totalDifficulty = new BigInteger("35bbde5595de6455", 16) - it.timestamp = Instant.now() - it.parentHash = parent - return it - } - api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) - api.answer("eth_getBlockByHash", [block2.hash.toHex(), false], block2) - def client = mockServer.clientForServer(new BlockchainGrpc.BlockchainImplBase() { - @Override - void nativeCall(BlockchainOuterClass.NativeCallRequest request, StreamObserver responseObserver) { - api.nativeCall(request, responseObserver) - } - - @Override - void subscribeHead(Common.Chain request, StreamObserver responseObserver) { - new Thread({ - responseObserver.onNext( - BlockchainOuterClass.ChainHead.newBuilder() - .setBlockId(block1.hash.toHex().substring(2)) - .setHeight(block1.number) - .setParentBlockId(parent.toHex().substring(2)) - .setWeight(ByteString.copyFrom(block1.totalDifficulty.toByteArray())) - .build() - ) - Thread.sleep(100) - responseObserver.onNext( - BlockchainOuterClass.ChainHead.newBuilder() - .setBlockId(block2.hash.toHex().substring(2)) - .setHeight(block2.number) - .setParentBlockId(parent.toHex().substring(2)) - .setWeight(ByteString.copyFrom(block2.totalDifficulty.toByteArray())) - .build() - ) - }).start() - } - }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM__MAINNET, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM__MAINNET, metrics), null, ChainsConfig.ChainConfig.default(), Schedulers.boundedElastic()) - upstream.setLag(0) - upstream.update( - BlockchainOuterClass.DescribeChain.newBuilder() - .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) - .addAllSupportedMethods(["eth_getBlockByHash"]) - .build(), - BlockchainOuterClass.BuildInfo.newBuilder() - .setVersion(buildInfo.version) - .build(), - ) - when: - new Thread({ Thread.sleep(50); upstream.head.start() }).start() - def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block(Duration.ofSeconds(2)) - then: - upstream.status == UpstreamAvailability.OK - upstream.getBuildInfo() == buildInfo - h.hash == BlockId.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - h.height == 650246 - } - - def "Follows difficulty"() { - setup: - def callData = [:] - def finished = new CompletableFuture() - def chain = Chain.ETHEREUM__MAINNET - def api = TestingCommons.api() - def block1 = new BlockJson().with { - it.number = 650246 - it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) - it.timestamp = Instant.now() - it.parentHash = parent - return it - } - def block2 = new BlockJson().with { - it.number = 650247 - it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") - it.totalDifficulty = new BigInteger("35bbde5595de6457", 16) - it.timestamp = Instant.now() - it.parentHash = parent - return it - } - api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) - api.answer("eth_getBlockByHash", [block2.hash.toHex(), false], block2) - def client = mockServer.clientForServer(new BlockchainGrpc.BlockchainImplBase() { - @Override - void nativeCall(BlockchainOuterClass.NativeCallRequest request, StreamObserver responseObserver) { - api.nativeCall(request, responseObserver) - } - - @Override - void subscribeHead(Common.Chain request, StreamObserver responseObserver) { - responseObserver.onNext( - BlockchainOuterClass.ChainHead.newBuilder() - .setBlockId(block1.hash.toHex().substring(2)) - .setHeight(block1.number) - .setParentBlockId(parent.toHex().substring(2)) - .setWeight(ByteString.copyFrom(block1.totalDifficulty.toByteArray())) - .build() - ) - responseObserver.onNext( - BlockchainOuterClass.ChainHead.newBuilder() - .setBlockId(block2.hash.toHex().substring(2)) - .setHeight(block2.number) - .setParentBlockId(parent.toHex().substring(2)) - .setWeight(ByteString.copyFrom(block2.totalDifficulty.toByteArray())) - .build() - ) - finished.complete(true) - } - }) - def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null, ChainsConfig.ChainConfig.default(), Schedulers.boundedElastic()) - upstream.setLag(0) - upstream.update( - BlockchainOuterClass.DescribeChain.newBuilder() - .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) - .addAllSupportedMethods(["eth_getBlockByHash"]) - .build(), - BlockchainOuterClass.BuildInfo.newBuilder() - .setVersion(buildInfo.version) - .build(), - ) - when: - new Thread({ Thread.sleep(50); upstream.head.start() }).start() - finished.get() - def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block(Duration.ofSeconds(2)) - then: - upstream.status == UpstreamAvailability.OK - upstream.getBuildInfo() == buildInfo - h.hash == BlockId.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") - h.height == 650247 - } - - private BlockchainOuterClass.DescribeChain describe(List methods) { - return BlockchainOuterClass.DescribeChain.newBuilder() - .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) - .addAllSupportedMethods(methods) - .build() - } -}