diff --git a/dshackle-cli/grpc b/dshackle-cli/grpc index 0d32b45d..d21e88f5 160000 --- a/dshackle-cli/grpc +++ b/dshackle-cli/grpc @@ -1 +1 @@ -Subproject commit 0d32b45d007417c901ca704c9324b5e6c7167f4f +Subproject commit d21e88f56f5a31fb5619cf9cf36c8649fa998535 diff --git a/emerald-java-client b/emerald-java-client index a3dde262..521524b6 160000 --- a/emerald-java-client +++ b/emerald-java-client @@ -1 +1 @@ -Subproject commit a3dde262ffab10749b8f14532765e0d2054fbd98 +Subproject commit 521524b657a9a55dd8fa86950f14aeeac1a0eff8 diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt index ac411dbb..52404ccb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt @@ -31,34 +31,40 @@ class BlockContainer( json: ByteArray?, val parsed: Any?, val transactions: List = emptyList(), - val nodeRating: Int = 0 + val nodeRating: Int = 0, + val upstreamId: String = "" ) : SourceContainer(json, parsed) { companion object { @JvmStatic - fun from(block: BlockJson<*>, raw: ByteArray): BlockContainer { - val hasTransactions = block.transactions?.filterIsInstance()?.count() ?: 0 > 0 + fun from(block: BlockJson<*>, raw: ByteArray, upstreamId: String): BlockContainer { + val hasTransactions = !block.transactions?.filterIsInstance().isNullOrEmpty() return BlockContainer( - block.number, - BlockId.from(block), - block.totalDifficulty, - block.timestamp, - hasTransactions, - raw, - block, - block.transactions?.map { TxId.from(it.hash) } ?: emptyList() + height = block.number, + hash = BlockId.from(block), + difficulty = block.totalDifficulty, + timestamp = block.timestamp, + full = hasTransactions, + json = raw, + parsed = block, + transactions = block.transactions?.map { TxId.from(it.hash) } ?: emptyList(), + upstreamId = upstreamId ) } @JvmStatic fun from(block: BlockJson<*>): BlockContainer { - return from(block, Global.objectMapper.writeValueAsBytes(block)) + return from(block, "unknown") + } + @JvmStatic + fun from(block: BlockJson<*>, upstream: String): BlockContainer { + return from(block, Global.objectMapper.writeValueAsBytes(block), upstream) } @JvmStatic - fun fromEthereumJson(raw: ByteArray): BlockContainer { + fun fromEthereumJson(raw: ByteArray, upstream: String): BlockContainer { val block = Global.objectMapper.readValue(raw, BlockJson::class.java) - return from(block, raw) + return from(block, raw, upstream) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt index 4a8911b9..67991ebf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt @@ -141,7 +141,7 @@ class QuorumRpcReader( src.map { val signature = response.providedSignature ?: if (key.nonce != null) { - signer?.sign(key.nonce, response.getResult(), api) + signer?.sign(key.nonce, response.getResult(), api.getId()) } else { null } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt index 6867a786..a2e5db89 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt @@ -220,7 +220,7 @@ open class NativeCall( .flatMap(JsonRpcResponse::requireResult) .map { if (ctx.nonce != null) { - CallResult.ok(ctx.id, ctx.nonce, it, signer.sign(ctx.nonce, it, ctx.upstream)) + CallResult.ok(ctx.id, ctx.nonce, it, signer.sign(ctx.nonce, it, ctx.upstream.getId())) } else { CallResult.ok(ctx.id, null, it, null) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt index 1169b03b..f57b0fd7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt @@ -17,11 +17,14 @@ package io.emeraldpay.dshackle.rpc import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.api.proto.BlockchainOuterClass.NativeSubscribeReplyItem import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.HasUpstream +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain import io.grpc.Status @@ -35,7 +38,8 @@ import reactor.core.publisher.Mono @Service open class NativeSubscribe( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder, + @Autowired private val signer: ResponseSigner ) { companion object { @@ -44,33 +48,46 @@ open class NativeSubscribe( private val objectMapper = Global.objectMapper - fun nativeSubscribe(request: Mono): Flux { + fun nativeSubscribe(request: Mono): Flux { return request .flatMapMany(this@NativeSubscribe::start) .map(this@NativeSubscribe::convertToProto) .onErrorMap(this@NativeSubscribe::convertToStatus) } - fun start(request: BlockchainOuterClass.NativeSubscribeRequest): Publisher { + fun start(request: BlockchainOuterClass.NativeSubscribeRequest): Publisher { val chain = Chain.byId(request.chainValue) if (BlockchainType.from(chain) != BlockchainType.ETHEREUM_POS && BlockchainType.from(chain) != BlockchainType.ETHEREUM) { return Mono.error(UnsupportedOperationException("Native subscribe is not supported for ${chain.chainCode}")) } - val method = request.method - val params: Any? = request.payload?.takeIf { !it.isEmpty }?.let { - objectMapper.readValue(it.newInput(), Map::class.java) - } + + val nonce = request.nonce.takeIf { it != 0L } val matcher = Selector.convertToMatcher(request.selector) - return subscribe(chain, method, params, matcher) + + /** + * Try to proxy request subscription directly to the upstream dshackle instance. + * If not possible - performs subscription logic on the current instance + * @see EthereumLikeMultistream.tryProxy + */ + val publisher = getUpstream(chain)?.tryProxy(matcher, request) ?: run { + val method = request.method + val params: Any? = request.payload?.takeIf { !it.isEmpty }?.let { + objectMapper.readValue(it.newInput(), Map::class.java) + } + subscribe(chain, method, params, matcher) + } + return publisher.map { ResponseHolder(it, nonce) } } fun convertToStatus(t: Throwable) = when (t) { is SilentException.UnsupportedBlockchain -> StatusException( Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}") ) + is UnsupportedOperationException -> StatusException( Status.UNIMPLEMENTED.withDescription(t.message) ) + else -> { log.warn("Unhandled error", t) StatusException( @@ -79,17 +96,54 @@ open class NativeSubscribe( } } - open fun subscribe(chain: Chain, method: String, params: Any?, matcher: Selector.Matcher): Flux { - val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain)) - return (up as EthereumLikeMultistream) - .getSubscribe() - .subscribe(method, params, matcher) + open fun subscribe(chain: Chain, method: String, params: Any?, matcher: Selector.Matcher): Flux = + getUpstream(chain)?.getSubscribe()?.subscribe(method, params, matcher) + ?: Flux.error(SilentException.UnsupportedBlockchain(chain)) + + private fun getUpstream(chain: Chain): EthereumLikeMultistream? = + multistreamHolder.getUpstream(chain) + ?.let { it as EthereumLikeMultistream } + + fun convertToProto(holder: ResponseHolder): NativeSubscribeReplyItem { + if (holder.response is NativeSubscribeReplyItem) { + return holder.response + } + val result = objectMapper.writeValueAsBytes(holder.response) + val builder = NativeSubscribeReplyItem.newBuilder() + .setPayload(ByteString.copyFrom(result)) + + holder.nonce?.also { nonce -> + holder.getSource()?.let { + signer.sign(nonce, result, it) + }?.let { + buildSignature(nonce, it) + }?.also { + builder.signature = it + } + } + + return builder.build() } - fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem { - val result = objectMapper.writeValueAsBytes(value) - return BlockchainOuterClass.NativeSubscribeReplyItem.newBuilder() - .setPayload(ByteString.copyFrom(result)) - .build() + fun buildSignature( + nonce: Long, + signature: ResponseSigner.Signature + ): BlockchainOuterClass.NativeCallReplySignature { + val msg = BlockchainOuterClass.NativeCallReplySignature.newBuilder() + msg.signature = ByteString.copyFrom(signature.value) + msg.keyId = signature.keyId + msg.upstreamId = signature.upstreamId + msg.nonce = nonce + return msg.build() + } + + data class ResponseHolder( + val response: Any, + val nonce: Long? + ) { + fun getSource(): String? = + if (response is HasUpstream) { + response.upstreamId.takeIf { it != "unknown" } + } else null } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt index 0c87de88..1518f579 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream +import com.google.common.annotations.VisibleForTesting import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice @@ -61,4 +62,8 @@ class MergedHead( } } } + + @VisibleForTesting + private fun getSources() = + sources } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt index eee280c0..66b2f129 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Mono open class DefaultEthereumHead( + private val upstreamId: String, forkChoice: ForkChoice, blockValidator: BlockValidator ) : Head, AbstractHead(forkChoice, blockValidator) { @@ -57,7 +58,7 @@ open class DefaultEthereumHead( .timeout(Defaults.timeout, Mono.error(Exception("Block data not received"))) } .map { - BlockContainer.fromEthereumJson(it.getResult()) + BlockContainer.fromEthereumJson(it.getResult(), upstreamId) } .onErrorResume { err -> log.debug("Failed to fetch latest block: ${err.message}") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt index 06ec5921..15bc299c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt @@ -119,7 +119,7 @@ class EthereumDirectReader( if (block == null) { Mono.empty() } else { - Mono.just(BlockContainer.from(block, blockbytes)) + Mono.just(BlockContainer.from(block, blockbytes, "unknown")) } } .doOnNext { block -> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt index 33f596fa..d049fe43 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt @@ -1,12 +1,23 @@ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Upstream +import reactor.core.publisher.Flux interface EthereumLikeMultistream : Upstream { fun getReader(): EthereumReader fun getSubscribe(): EthereumSubscribe fun getHead(mather: Selector.Matcher): Head + + /** + * Tries to proxy the native subscribe request to the managed upstreams if + * - any of them matches the matcher criteria + * - all of matching above are gRPC ones + * in this case the upstream dshackle instances can sign the results and they will just proxied as is with original signs + * Otherwise return null + */ + fun tryProxy(matcher: Selector.Matcher, request: BlockchainOuterClass.NativeSubscribeRequest): Flux? } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt index 3605976b..fc6426d3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader @@ -27,12 +28,14 @@ import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice +import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import org.springframework.util.ConcurrentReferenceHashMap +import reactor.core.publisher.Flux import reactor.core.publisher.Mono @Suppress("UNCHECKED_CAST") @@ -93,6 +96,22 @@ open class EthereumMultistream( return head!! } + override fun tryProxy( + matcher: Selector.Matcher, + request: BlockchainOuterClass.NativeSubscribeRequest + ): Flux? = + upstreams.filter { + matcher.matches(it) + }.takeIf { ups -> + ups.all { it.isGrpc() } + }?.map { + it as GrpcUpstream + }?.map { + it.getBlockchainApi().nativeSubscribe(request) + }?.let { + Flux.merge(it) + } + override fun setHead(head: Head) { this.head = head } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt index bb1fb38d..d299354f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt @@ -33,9 +33,10 @@ import java.util.concurrent.Executors class EthereumRpcHead( private val api: Reader, forkChoice: ForkChoice, + upstreamId: String, blockValidator: BlockValidator, private val interval: Duration = Duration.ofSeconds(10), -) : DefaultEthereumHead(forkChoice, blockValidator), Lifecycle { +) : DefaultEthereumHead(upstreamId, forkChoice, blockValidator), Lifecycle { companion object { val scheduler = diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt index 00def0bf..fee1674d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt @@ -62,7 +62,7 @@ class EthereumWsFactory( require(upstream == null || upstream.getId() == id) { "Creating instance for different upstream. ${upstream?.getId()} != id" } - return WsConnection(uri, origin, basicAuth, metrics, upstream, validator).also { ws -> + return WsConnection(id, uri, origin, basicAuth, metrics, upstream, validator).also { ws -> config?.frameSize?.let { ws.frameSize = it } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt index d008530f..e34ef409 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt @@ -26,9 +26,10 @@ import reactor.core.publisher.Flux class EthereumWsHead( private val ws: WsConnection, + upstreamId: String, forkChoice: ForkChoice, blockValidator: BlockValidator -) : DefaultEthereumHead(forkChoice, blockValidator), Lifecycle { +) : DefaultEthereumHead(upstreamId, forkChoice, blockValidator), Lifecycle { private val log = LoggerFactory.getLogger(EthereumWsHead::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnection.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnection.kt index a1c95ff8..b2b6bec2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnection.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnection.kt @@ -65,6 +65,7 @@ import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger open class WsConnection( + private val id: String, private val uri: URI, private val origin: URI, private val basicAuth: AuthConfig.ClientBasicAuth?, @@ -352,7 +353,7 @@ open class WsConnection( } } .flatMap(JsonRpcResponse::requireResult) - .map { BlockContainer.fromEthereumJson(it) } + .map { BlockContainer.fromEthereumJson(it, id) } .subscribeOn(Schedulers.boundedElastic()) .timeout(Defaults.timeoutInternal, Mono.empty()) }.repeatWhenEmpty { n -> @@ -368,7 +369,7 @@ open class WsConnection( .then() } else { Mono.fromCallable { - blocks.tryEmitNext(BlockContainer.from(block)) + blocks.tryEmitNext(BlockContainer.from(block, id)) }.then() } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt index f7ef9b36..f23efb6f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt @@ -16,7 +16,9 @@ open class EthereumConnectorFactory( private val forkChoice: ForkChoice, private val blockValidator: BlockValidator ) : ConnectorFactory { - private val log = LoggerFactory.getLogger(EthereumConnectorFactory::class.java) + companion object { + private val log = LoggerFactory.getLogger(EthereumConnectorFactory::class.java) + } override fun isValid(): Boolean { if (preferHttp && httpFactory == null) { @@ -27,7 +29,7 @@ open class EthereumConnectorFactory( override fun create(upstream: DefaultUpstream, validator: EthereumUpstreamValidator, chain: Chain): EthereumConnector { if (wsFactory != null && !preferHttp) { - return EthereumWsConnector(wsFactory, upstream, validator, chain, forkChoice, blockValidator) + return EthereumWsConnector(wsFactory, upstream, validator, forkChoice, blockValidator) } if (httpFactory == null) { throw java.lang.IllegalArgumentException("Can't create rpc connector if no http factory set") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt index 8ee56bbe..2b46a499 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt @@ -35,14 +35,14 @@ class EthereumRpcConnector( if (wsFactory != null) { // do not set upstream to the WS, since it doesn't control the RPC upstream conn = wsFactory.create(null, null) - val wsHead = EthereumWsHead(conn, forkChoice, blockValidator) + val wsHead = EthereumWsHead(conn, id, forkChoice, blockValidator) // receive bew blocks through WebSockets, but also periodically verify with RPC in case if WS failed - val rpcHead = EthereumRpcHead(directReader, forkChoice, blockValidator, Duration.ofSeconds(60)) + val rpcHead = EthereumRpcHead(directReader, forkChoice, id, blockValidator, Duration.ofSeconds(60)) head = MergedHead(listOf(rpcHead, wsHead), forkChoice) } else { conn = null log.warn("Setting up connector for $id upstream with RPC-only access, less effective than WS+RPC") - head = EthereumRpcHead(directReader, forkChoice, blockValidator) + head = EthereumRpcHead(directReader, forkChoice, id, blockValidator) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt index eb796104..0a248e2c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt @@ -4,18 +4,19 @@ import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head -import io.emeraldpay.dshackle.upstream.ethereum.* +import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator +import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory +import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead +import io.emeraldpay.dshackle.upstream.ethereum.WsConnection import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient -import io.emeraldpay.grpc.Chain class EthereumWsConnector( wsFactory: EthereumWsFactory, upstream: DefaultUpstream, validator: EthereumUpstreamValidator, - chain: Chain, forkChoice: ForkChoice, blockValidator: BlockValidator ) : EthereumConnector { @@ -25,7 +26,7 @@ class EthereumWsConnector( init { conn = wsFactory.create(upstream, validator) - head = EthereumWsHead(conn, forkChoice, blockValidator) + head = EthereumWsHead(conn, upstream.getId(), forkChoice, blockValidator) api = JsonRpcWsClient(conn) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt index 7d635b62..87ed40cf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt @@ -71,7 +71,7 @@ class ConnectBlockUpdates( val prev = findPrevious(block) remember(block) val removed = if (prev != null) { - whenReplaced(prev) + whenReplaced(prev, block.upstreamId) } else { Flux.empty() } @@ -103,13 +103,14 @@ class ConnectBlockUpdates( /** * Produce updates for transactions when a block is replaces with a different one on the same height. */ - fun whenReplaced(prev: BlockContainer): Flux { + fun whenReplaced(prev: BlockContainer, source: String): Flux { return Flux.fromIterable(prev.transactions).map { Update( prev.hash, prev.height, UpdateType.DROP, - it + it, + source ) } } @@ -121,7 +122,8 @@ class ConnectBlockUpdates( block.hash, block.height, UpdateType.NEW, - it + it, + block.upstreamId ) } } @@ -131,6 +133,7 @@ class ConnectBlockUpdates( val blockNumber: Long, val type: UpdateType, val transactionId: TxId, + val upstreamId: String ) enum class UpdateType { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt index 44da804d..a7c11ba6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt @@ -94,7 +94,9 @@ class ProduceLogs( txlog.topics, txlog.transactionHash, txlog.transactionIndex, - false + false, + update.upstreamId + ) } oldMessages.put(LogReference(update.blockHash, update.transactionId), messages) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt index 3b9f6162..68fa7d22 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream.ethereum.subscribe import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHeadMessage import io.emeraldpay.etherjar.rpc.json.BlockJson @@ -41,13 +42,7 @@ class ProduceNewHeads( fun start(): Flux { return head.getFlux() .map { - if (it.parsed != null) { - it.parsed as BlockJson - } else { - objectMapper.readValue(it.json, BlockJson::class.java) - } - } - .map { block -> + val block = extractBlock(it) NewHeadMessage( block.number, block.hash, @@ -58,8 +53,16 @@ class ProduceNewHeads( block.gasUsed, block.logsBloom, block.miner, - block.baseFeePerGas?.amount + block.baseFeePerGas?.amount, + it.upstreamId ) } } + + private fun extractBlock(blockContainer: BlockContainer): BlockJson = + if (blockContainer.parsed != null) { + blockContainer.parsed as BlockJson + } else { + objectMapper.readValue(blockContainer.json, BlockJson::class.java) + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/HasUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/HasUpstream.kt new file mode 100644 index 00000000..6c3a2a8c --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/HasUpstream.kt @@ -0,0 +1,5 @@ +package io.emeraldpay.dshackle.upstream.ethereum.subscribe.json + +interface HasUpstream { + val upstreamId: String +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt index f0ef7d53..88c77530 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum.subscribe.json +import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonSerialize import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.domain.BlockHash @@ -40,5 +41,7 @@ data class LogMessage( val transactionHash: TransactionId, @get:JsonSerialize(using = NumberAsHexSerializer::class) val transactionIndex: Long, - val removed: Boolean -) + val removed: Boolean, + @get:JsonIgnore + override val upstreamId: String +) : HasUpstream diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt index 90eec052..5dd5b175 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum.subscribe.json +import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.databind.annotation.JsonSerialize import io.emeraldpay.etherjar.domain.Address @@ -50,5 +51,7 @@ data class NewHeadMessage( val miner: Address, @get:JsonSerialize(using = NumberAsHexSerializer::class) @get:JsonInclude(JsonInclude.Include.NON_NULL) - val baseFeePerGas: BigInteger? -) + val baseFeePerGas: BigInteger?, + @get:JsonIgnore + override val upstreamId: String +) : HasUpstream diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt index 60326bc3..5bcce942 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader @@ -27,12 +28,14 @@ import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice +import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import org.springframework.util.ConcurrentReferenceHashMap +import reactor.core.publisher.Flux import reactor.core.publisher.Mono @Suppress("UNCHECKED_CAST") @@ -88,6 +91,19 @@ open class EthereumPosMultiStream( return head!! } + override fun tryProxy(matcher: Selector.Matcher, request: BlockchainOuterClass.NativeSubscribeRequest): Flux? = + upstreams.filter { + matcher.matches(it) + }.takeIf { ups -> + ups.all { it.isGrpc() } + }?.map { + it as GrpcUpstream + }?.map { + it.proxySubscribe(request) + }?.let { + Flux.merge(it) + } + override fun setHead(head: Head) { this.head = head } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt index 0117efe0..a539a21a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt @@ -38,6 +38,7 @@ import io.emeraldpay.grpc.Chain import org.reactivestreams.Publisher import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.math.BigInteger import java.time.Instant @@ -105,6 +106,9 @@ class BitcoinGrpcUpstream( return remote } + override fun proxySubscribe(request: BlockchainOuterClass.NativeSubscribeRequest): Flux = + remote.nativeSubscribe(request) + override fun getHead(): Head { return grpcHead } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt index 7f79bf95..ab4084c5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt @@ -41,6 +41,7 @@ import io.emeraldpay.grpc.Chain import org.reactivestreams.Publisher import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.math.BigInteger import java.time.Instant @@ -84,7 +85,7 @@ open class EthereumGrpcUpstream( defaultReader.read(JsonRpcRequest("eth_getBlockByHash", listOf(existingBlock.hash.toHexWithPrefix(), false))) .flatMap(JsonRpcResponse::requireResult) .map { - BlockContainer.fromEthereumJson(it) + BlockContainer.fromEthereumJson(it, getId()) } .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) .doOnError { t -> @@ -110,6 +111,9 @@ open class EthereumGrpcUpstream( return remote } + override fun proxySubscribe(request: BlockchainOuterClass.NativeSubscribeRequest): Flux = + remote.nativeSubscribe(request) + override fun start() { } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt index 256f0a4f..43acab2f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt @@ -41,6 +41,7 @@ import io.emeraldpay.grpc.Chain import org.reactivestreams.Publisher import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.math.BigInteger import java.time.Instant @@ -84,7 +85,7 @@ open class EthereumPosGrpcUpstream( defaultReader.read(JsonRpcRequest("eth_getBlockByHash", listOf(existingBlock.hash.toHexWithPrefix(), false))) .flatMap(JsonRpcResponse::requireResult) .map { - BlockContainer.fromEthereumJson(it) + BlockContainer.fromEthereumJson(it, getId()) } .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) .doOnError { t -> @@ -130,6 +131,9 @@ open class EthereumPosGrpcUpstream( return remote } + override fun proxySubscribe(request: BlockchainOuterClass.NativeSubscribeRequest): Flux = + remote.nativeSubscribe(request) + // ------------------------------------------------------------------------------------------ override fun getLabels(): Collection { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt index 5b001af4..883085c3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt @@ -16,8 +16,10 @@ package io.emeraldpay.dshackle.upstream.grpc import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.api.proto.BlockchainOuterClass.NativeSubscribeRequest import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.upstream.Upstream +import reactor.core.publisher.Flux interface GrpcUpstream : Upstream { @@ -28,4 +30,6 @@ interface GrpcUpstream : Upstream { fun update(conf: BlockchainOuterClass.DescribeChain) fun getBlockchainApi(): ReactorBlockchainGrpc.ReactorBlockchainStub + + fun proxySubscribe(request: NativeSubscribeRequest): Flux } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt index 10039c13..eb5ac760 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt @@ -1,6 +1,5 @@ package io.emeraldpay.dshackle.upstream.signature -import io.emeraldpay.dshackle.upstream.Upstream import org.apache.commons.codec.binary.Hex import java.security.MessageDigest import java.security.Signature @@ -17,15 +16,13 @@ class EcdsaSigner( const val MSG_SEPARATOR = '/' } - override fun sign(nonce: Long, message: ByteArray, source: Upstream): ResponseSigner.Signature { + override fun sign(nonce: Long, message: ByteArray, source: String): ResponseSigner.Signature { val sig = Signature.getInstance(SIGN_SCHEME) sig.initSign(privateKey) val wrapped = wrapMessage(nonce, message, source) sig.update(wrapped.toByteArray()) val value = sig.sign() - return ResponseSigner.Signature( - value, source.getId(), keyId - ) + return ResponseSigner.Signature(value, source, keyId) } /** @@ -42,7 +39,7 @@ class EcdsaSigner( * - second is the nonce value encode as decimal string * - third is SHA256 hash of the original message encoded as hex string */ - fun wrapMessage(nonce: Long, message: ByteArray, source: Upstream): String { + fun wrapMessage(nonce: Long, message: ByteArray, source: String): String { val sha256 = MessageDigest.getInstance("SHA-256") // we create it with max capacity that we expect for the result, which is total lengths of its parts val formatterMsg = StringBuilder(11 + 1 + 18 + 1 + 64 + 1 + 64) @@ -52,7 +49,7 @@ class EcdsaSigner( .append(MSG_SEPARATOR) // We expect that the id is short enough (less than 64 symbols) and also it doesn't contain the `/` symbol // which is verified in UpstreamConfigReader and DefaultUpstream constructor - .append(source.getId()) + .append(source) .append(MSG_SEPARATOR) .append(Hex.encodeHexString(sha256.digest(message))) return formatterMsg.toString() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt index 2945041d..5c963b0a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt @@ -1,9 +1,7 @@ package io.emeraldpay.dshackle.upstream.signature -import io.emeraldpay.dshackle.upstream.Upstream - class NoSigner : ResponseSigner { - override fun sign(nonce: Long, message: ByteArray, source: Upstream): ResponseSigner.Signature? { + override fun sign(nonce: Long, message: ByteArray, source: String): ResponseSigner.Signature? { return null } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt index d88aee4e..39549941 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt @@ -1,10 +1,8 @@ package io.emeraldpay.dshackle.upstream.signature -import io.emeraldpay.dshackle.upstream.Upstream - interface ResponseSigner { - fun sign(nonce: Long, message: ByteArray, source: Upstream): Signature? + fun sign(nonce: Long, message: ByteArray, source: String): Signature? data class Signature( val value: ByteArray, diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy index b3c3f101..31bcf2a9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy @@ -62,7 +62,7 @@ class BlocksRedisCacheSpec extends Specification { false, "test".bytes, null, - [TxId.from(hash2), TxId.from(hash1)], 0 + [TxId.from(hash2), TxId.from(hash1)], 0, "BlocksRedisCacheSpec" ) when: diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashAddingSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashAddingSpec.groovy index cdb5c8bd..f570f152 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashAddingSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashAddingSpec.groovy @@ -27,7 +27,7 @@ class HeightByHashAddingSpec extends Specification { def block = new BlockContainer( 12079192L, BlockId.from("0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"), - BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0 + BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0, "upstream" ) def "use memory if available"() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashRedisCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashRedisCacheSpec.groovy index 5ba7d18d..632dcad2 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashRedisCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashRedisCacheSpec.groovy @@ -30,11 +30,11 @@ class HeightByHashRedisCacheSpec extends Specification { def block1 = new BlockContainer( 12079192L, BlockId.from("0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"), - BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0 + BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0, "HeightByHashRedisCacheSpec" ) def block2 = new BlockContainer( 12079193L, BlockId.from("0xd27944b460632699768fbfec3e5d454db590cae43d470b5f42fc4d091e372c25"), - BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0 + BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0, "HeightByHashRedisCacheSpec" ) StatefulRedisConnection redis diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/ReceiptMemCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/ReceiptMemCacheSpec.groovy index d950a765..25b950f4 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/ReceiptMemCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/ReceiptMemCacheSpec.groovy @@ -87,7 +87,8 @@ class ReceiptMemCacheSpec extends Specification { "{}".bytes, null, [TxId.from(receipt.transactionHash)], - 0 + 0, + "unknown" ) when: diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy index b4e94ff9..5793c17c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy @@ -19,9 +19,9 @@ import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.test.MultistreamHolderMock import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream import io.emeraldpay.dshackle.upstream.ethereum.EthereumSubscribe +import io.emeraldpay.dshackle.upstream.signature.NoSigner import io.emeraldpay.grpc.Chain import reactor.core.publisher.Flux import reactor.test.StepVerifier @@ -30,33 +30,46 @@ import spock.lang.Specification import java.time.Duration class NativeSubscribeSpec extends Specification { + def signer = new NoSigner() def "Call with empty params when not provided"() { setup: - def subscribe = Mock(EthereumSubscribe) { - 1 * it.subscribe("newHeads", null, _ as Selector.AnyLabelMatcher) >> Flux.just("{}") - } - def up = Mock(EthereumPosMultiStream) { - 1 * it.getSubscribe() >> subscribe - } - - def nativeSubscribe = new NativeSubscribe(new MultistreamHolderMock(Chain.ETHEREUM, up)) def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() .setChainValue(Chain.ETHEREUM.id) .setMethod("newHeads") .build() + + def subscribe = Mock(EthereumSubscribe) { + 1 * it.subscribe("newHeads", null, _ as Selector.AnyLabelMatcher) >> Flux.just("{}") + } + def up = Mock(EthereumPosMultiStream) { + 1 * it.tryProxy(_ as Selector.AnyLabelMatcher, call) >> null + 1 * it.getSubscribe() >> subscribe + } + + def nativeSubscribe = new NativeSubscribe(new MultistreamHolderMock(Chain.ETHEREUM, up), signer) + when: def act = nativeSubscribe.start(call) then: StepVerifier.create(act) - .expectNext("{}") + .expectNext(new NativeSubscribe.ResponseHolder("{}", null)) .expectComplete() .verify(Duration.ofSeconds(1)) } def "Call with params when provided"() { setup: + def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() + .setChainValue(Chain.ETHEREUM.id) + .setMethod("logs") + .setPayload(ByteString.copyFromUtf8( + '{"address": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", ' + + '"topics": ["0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65"]}' + )) + .build() + def subscribe = Mock(EthereumSubscribe) { 1 * it.subscribe("logs", { params -> println("params: $params") @@ -69,25 +82,43 @@ class NativeSubscribeSpec extends Specification { }, _ as Selector.AnyLabelMatcher) >> Flux.just("{}") } def up = Mock(EthereumPosMultiStream) { + 1 * it.tryProxy(_ as Selector.AnyLabelMatcher, call) >> null 1 * it.getSubscribe() >> subscribe } - def nativeSubscribe = new NativeSubscribe(new MultistreamHolderMock(Chain.ETHEREUM, up)) - def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() - .setChainValue(Chain.ETHEREUM.id) - .setMethod("logs") - .setPayload(ByteString.copyFromUtf8( - '{"address": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", ' + - '"topics": ["0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65"]}' - )) - .build() + def nativeSubscribe = new NativeSubscribe(new MultistreamHolderMock(Chain.ETHEREUM, up), signer) + when: def act = nativeSubscribe.start(call) then: StepVerifier.create(act) - .expectNext("{}") + .expectNext(new NativeSubscribe.ResponseHolder("{}", null)) + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + + def "Proxy call"() { + setup: + def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() + .setChainValue(Chain.ETHEREUM.id) + .setMethod("newHeads") + .build() + def up = Mock(EthereumPosMultiStream) { + 1 * it.tryProxy(_ as Selector.AnyLabelMatcher, call) >> Flux.just("{}") + 0 * it.getSubscribe() + } + + def nativeSubscribe = new NativeSubscribe(new MultistreamHolderMock(Chain.ETHEREUM, up), signer) + + when: + def act = nativeSubscribe.start(call) + + then: + StepVerifier.create(act) + .expectNext(new NativeSubscribe.ResponseHolder("{}", null)) .expectComplete() .verify(Duration.ofSeconds(1)) } } + diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy index 69601c45..b1d9f7c1 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy @@ -271,7 +271,7 @@ class TrackBitcoinAddressSpec extends Specification { Head head = Mock(Head) { 1 * getFlux() >> Flux.concat( Flux.just( - new BlockContainer(0L, BlockId.from(hash1), BigInteger.ZERO, Instant.now(), false, null, null, [], 0) + new BlockContainer(0L, BlockId.from(hash1), BigInteger.ZERO, Instant.now(), false, null, null, [], 0, "TrackBitcoinAddressSpec") ), blocks.asFlux() ) @@ -312,7 +312,7 @@ class TrackBitcoinAddressSpec extends Specification { StepVerifier.create(resp) .expectNext("0") .then { - blocks.tryEmitNext(new BlockContainer(1L, BlockId.from(hash1), BigInteger.ONE, Instant.now(), false, null, null, [], 0)) + blocks.tryEmitNext(new BlockContainer(1L, BlockId.from(hash1), BigInteger.ONE, Instant.now(), false, null, null, [], 0, "TrackBitcoinAddressSpec")) } .expectNext("1230000") .then { diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinTxSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinTxSpec.groovy index b1f36eba..a4f6e5df 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinTxSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinTxSpec.groovy @@ -142,7 +142,7 @@ class TrackBitcoinTxSpec extends Specification { def txid = "69cd44d7c641db82e69824523c7ac0c5c1e5628f025474529cf5ffe64527efc9" // start with the current block def next = Flux.fromIterable([10, 12, 13, 14, 15]).map { h -> - new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0) + new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0, "unknown") } Head head = Mock(Head) { 1 * getFlux() >> next @@ -173,7 +173,7 @@ class TrackBitcoinTxSpec extends Specification { def txid = "69cd44d7c641db82e69824523c7ac0c5c1e5628f025474529cf5ffe64527efc9" // start with the current block def next = Flux.fromIterable([10, 12, 13]).map { h -> - new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0) + new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0, "unknown") } Head head = Mock(Head) { 1 * getFlux() >> next @@ -268,7 +268,7 @@ class TrackBitcoinTxSpec extends Specification { ]) } def next = Flux.fromIterable([10, 11, 12]).map { h -> - new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0) + new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0, "unknown") } Head head = Mock(Head) { _ * getFlux() >> next diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackERC20AddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackERC20AddressSpec.groovy index 2a347a9a..86fc1553 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackERC20AddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackERC20AddressSpec.groovy @@ -165,7 +165,8 @@ class TrackERC20AddressSpec extends Specification { ], TransactionId.from("0x5a7898e27120575c33d3d0179af3b6353c7268bbad4255df079ed26b743a21a5"), 1, - false + false, + "unknown" ) ] def logs = Mock(ConnectLogs) { diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy index 1289714e..178b794f 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy @@ -199,7 +199,7 @@ class TrackEthereumTxSpec extends Specification { def tx = new TrackEthereumTx.TxDetails(Chain.ETHEREUM, Instant.now(), TransactionId.from(txId), 6) def block = new BlockContainer( 100, BlockId.from(txId), BigInteger.ONE, Instant.now(), false, "".bytes, null, - [TxId.from(txId)], 0 + [TxId.from(txId)], 0, "unknown" ) when: @@ -222,7 +222,7 @@ class TrackEthereumTxSpec extends Specification { def block = new BlockContainer( 100, BlockId.from(txId), BigInteger.ONE, Instant.now(), false, "".bytes, null, [TxId.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22")], - 0 + 0, "unknown" ) apiMock.answer("eth_getTransactionByHash", [txId], null) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy index 5d75a195..7c1f4954 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumPosRpcUpstreamMock.groovy @@ -52,6 +52,10 @@ class EthereumPosRpcUpstreamMock extends EthereumPosRpcUpstream { this(chain, api, allMethods()) } + EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, Map labels) { + this(id, chain, api, allMethods(), labels) + } + EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api) { this(id, chain, api, allMethods()) } @@ -61,11 +65,15 @@ class EthereumPosRpcUpstreamMock extends EthereumPosRpcUpstream { } EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods) { + this(id, chain, api, methods, Collections.emptyMap()) + } + + EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader api, CallMethods methods, Map labels) { super(id, chain, UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.UpstreamRole.PRIMARY, methods, - new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()), + new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(labels)), new ConnectorFactoryMock(api, new EthereumHeadMock())) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy index f2cac503..fbec4d77 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy @@ -16,7 +16,6 @@ */ package io.emeraldpay.dshackle.test - import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesFactory @@ -27,15 +26,12 @@ import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosRpcUpstream -import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.grpc.Chain import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.logging.LoggingMeterRegistry import org.apache.commons.lang3.StringUtils @@ -56,6 +52,10 @@ class TestingCommons { return new EthereumPosRpcUpstreamMock(id, Chain.ETHEREUM, api()) } + static EthereumPosRpcUpstreamMock upstream(String id, String provider) { + return new EthereumPosRpcUpstreamMock(id, Chain.ETHEREUM, api(), Collections.singletonMap("provider", provider)) + } + static EthereumPosRpcUpstreamMock upstream(String id, Reader api) { return new EthereumPosRpcUpstreamMock(id, Chain.ETHEREUM, api) } @@ -114,7 +114,8 @@ class TestingCommons { null, null, [], - 0 + 0, + "upstream" ) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractHeadSpec.groovy index 028c9150..10253819 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractHeadSpec.groovy @@ -34,7 +34,7 @@ class AbstractHeadSpec extends Specification { def blocks = [1L, 2, 3, 4].collect { i -> byte[] hash = new byte[32] hash[0] = i as byte - new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0) + new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0, "AbstractHeadSpec") } def "Calls beforeBlock on each block"() { @@ -96,7 +96,7 @@ class AbstractHeadSpec extends Specification { blocks[1].height, BlockId.from(blocks[1].hash.value.clone().tap { it[1] = 0xff as byte }), blocks[1].difficulty - 1, Instant.now(), - false, null, null, [], 0 + false, null, null, [], 0, "AbstractHeadSpec" ) when: head.follow(source.asFlux()) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy index 42fa96c8..8a28f4b3 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy @@ -16,25 +16,33 @@ */ package io.emeraldpay.dshackle.upstream +import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.quorum.AlwaysQuorum import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.test.EthereumPosRpcUpstreamMock -import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods -import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream +import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosUpstream +import io.emeraldpay.dshackle.upstream.grpc.EthereumPosGrpcUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.etherjar.domain.BlockHash +import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import io.emeraldpay.grpc.Chain import org.jetbrains.annotations.NotNull +import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import reactor.test.StepVerifier import spock.lang.Specification import java.time.Duration import java.time.Instant +import java.time.temporal.ChronoUnit class MultistreamSpec extends Specification { @@ -193,6 +201,147 @@ class MultistreamSpec extends Specification { 1 * postprocessor.onReceive("test_foo", [1], "\"test\"".bytes) } + def "Filter upstream matching selector single"() { + setup: + def up1 = TestingCommons.upstream("test-1", "internal") + def up2 = TestingCommons.upstream("test-2", "external") + def up3 = TestingCommons.upstream("test-3", "external") + def multistream = new EthereumPosMultiStream(Chain.ETHEREUM, [up1, up2, up3], Caches.default()) + + expect: + multistream.getHead(new Selector.LabelMatcher("provider", ["internal"])).is(up1.ethereumHeadMock) + multistream.getHead(new Selector.LabelMatcher("provider", ["unknown"])) in EmptyHead + + def head = multistream.getHead(new Selector.LabelMatcher("provider", ["external"])) + head in MergedHead + (head as MergedHead).isRunning() + (head as MergedHead).getSources().sort() == [up2.ethereumHeadMock, up3.ethereumHeadMock].sort() + + } + + def "Proxy gRPC request - select one"() { + setup: + + def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() + .setChainValue(Chain.ETHEREUM.id) + .setMethod("newHeads") + .build() + + def up1 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> true + 1 * getId() >> "internal" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + 1 * proxySubscribe(call) >> Flux.just("{}") + } + def up2 = Mock(EthereumPosGrpcUpstream) { + 1 * getId() >> "external" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "external"))] + } + def multiStream = new TestEthereumPosMultistream(Chain.ETHEREUM, [up1, up2], Caches.default()) + + when: + def act = multiStream.tryProxy(new Selector.LabelMatcher("provider", ["internal"]), call) + + then: + StepVerifier.create(act) + .expectNext("{}") + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + + def "Proxy gRPC request - not all gRPC"() { + setup: + + def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() + .setChainValue(Chain.ETHEREUM.id) + .setMethod("newHeads") + .build() + + def up1 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> true + 1 * getId() >> "1" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + } + def up2 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> false + 1 * getId() >> "2" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + } + def multiStream = new TestEthereumPosMultistream(Chain.ETHEREUM, [up1, up2], Caches.default()) + + when: + def act = multiStream.tryProxy(new Selector.LabelMatcher("provider", ["internal"]), call) + + then: + !act + } + + def "Proxy gRPC request - select many"() { + setup: + + def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() + .setChainValue(Chain.ETHEREUM.id) + .setMethod("newHeads") + .build() + + def up1 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> true + 1 * getId() >> "1" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + 1 * proxySubscribe(call) >> Flux.just("{1}") + } + def up2 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> true + 1 * getId() >> "2" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + 1 * proxySubscribe(call) >> Flux.just("{2}") + } + def multiStream = new TestEthereumPosMultistream(Chain.ETHEREUM, [up1, up2], Caches.default()) + + when: + def act = multiStream.tryProxy(new Selector.LabelMatcher("provider", ["internal"]), call) + + then: + StepVerifier.create(act) + .expectNextMatches { it as String in ["{1}", "{2}"] } + .expectNextMatches { it as String in ["{1}", "{2}"] } + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + + def "Proxy gRPC request - select many"() { + setup: + + def call = BlockchainOuterClass.NativeSubscribeRequest.newBuilder() + .setChainValue(Chain.ETHEREUM.id) + .setMethod("newHeads") + .build() + + def up1 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> true + 1 * getId() >> "1" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + 1 * proxySubscribe(call) >> Flux.just("{1}") + } + def up2 = Mock(EthereumPosGrpcUpstream) { + 1 * isGrpc() >> true + 1 * getId() >> "2" + 1 * getLabels() >> [UpstreamsConfig.Labels.fromMap(Collections.singletonMap("provider", "internal"))] + 1 * proxySubscribe(call) >> Flux.just("{2}") + } + def multiStream = new TestEthereumPosMultistream(Chain.ETHEREUM, [up1, up2], Caches.default()) + + when: + def act = multiStream.tryProxy(new Selector.LabelMatcher("provider", ["internal"]), call) + + then: + StepVerifier.create(act) + .expectNextMatches { it as String in ["{1}", "{2}"] } + .expectNextMatches { it as String in ["{1}", "{2}"] } + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + class TestMultistream extends Multistream { TestMultistream(List upstreams, @NotNull RequestPostprocessor postprocessor) { @@ -233,4 +382,57 @@ class MultistreamSpec extends Specification { return null } } + + class TestEthereumPosMultistream extends EthereumPosMultiStream { + + TestEthereumPosMultistream(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { + super(chain, upstreams, caches) + } + + @Override + Mono> getRoutedApi(@NotNull Selector.Matcher matcher) { + return null + } + + @Override + Head updateHead() { + return null + } + + @Override + void setHead(@NotNull Head head) { + + } + + @Override + Head getHead() { + return null + } + + public T cast(Class selfType) { + return this + } + + @Override + ChainFees getFeeEstimation() { + return null + } + + @Override + void init() { + + } + } + + BlockContainer createBlock(long number) { + def block = new BlockJson() + block.number = number + block.hash = BlockHash.from("0x0000000000000000000000000000000000000000000000000000000000" + number) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block.uncles = [] + block.transactions = [] + + return BlockContainer.from(block) + } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy index f0c3fc77..d94914c9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy @@ -32,7 +32,7 @@ import java.time.Instant class DefaultEthereumHeadSpec extends Specification { - DefaultEthereumHead head = new DefaultEthereumHead(new MostWorkForkChoice(), BlockValidator.@Companion.ALWAYS_VALID) + DefaultEthereumHead head = new DefaultEthereumHead("upstream", new MostWorkForkChoice(), BlockValidator.@Companion.ALWAYS_VALID) ObjectMapper objectMapper = Global.objectMapper def blocks = (10L..20L).collect { i -> diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidatorSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidatorSpec.groovy index d52201f8..571e415c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidatorSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockValidatorSpec.groovy @@ -52,6 +52,8 @@ class EthereumBlockValidatorSpec extends Specification { true, bytes, block, Collections.emptyList(), - 1) + 1, + "upstream" + ) } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReaderSpec.groovy index 0583c05d..0d81a0da 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReaderSpec.groovy @@ -302,7 +302,7 @@ class EthereumFullBlocksReaderSpec extends Specification { "extraField2": "extraValue2" } ''' - blocks.add(BlockContainer.fromEthereumJson(blockJson.bytes)) + blocks.add(BlockContainer.fromEthereumJson(blockJson.bytes, "EthereumFullBlocksReaderSpec")) def tx1 = ''' { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy index e9557335..f294225e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdatesSpec.groovy @@ -81,7 +81,7 @@ class ConnectBlockUpdatesSpec extends Specification { ] }) when: - def act = connectBlockUpdates.whenReplaced(block) + def act = connectBlockUpdates.whenReplaced(block, "ConnectBlockUpdatesSpec") .collectList().block(Duration.ofSeconds(3)) then: diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy index e7186c2a..1c49c95e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogsSpec.groovy @@ -40,7 +40,8 @@ class ConnectLogsSpec extends Specification { ], TransactionId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), 1L, - false + false, + "upstream" ) def log2 = new LogMessage( @@ -54,7 +55,8 @@ class ConnectLogsSpec extends Specification { ], TransactionId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), 1L, - false + false, + "upstream" ) def log3 = new LogMessage( @@ -68,7 +70,8 @@ class ConnectLogsSpec extends Specification { ], TransactionId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), 1L, - false + false, + "upstream" ) def log4 = new LogMessage( @@ -82,7 +85,8 @@ class ConnectLogsSpec extends Specification { ], TransactionId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), 1L, - false + false, + "upstream" ) def "Filter is empty"() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogsSpec.groovy index c0d2ba67..b5165489 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogsSpec.groovy @@ -45,7 +45,8 @@ class ProduceLogsSpec extends Specification { BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.NEW, - TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af") + TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af"), + "upstream" ) when: def act = producer.produceAdded(update) @@ -67,7 +68,8 @@ class ProduceLogsSpec extends Specification { BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.NEW, - TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af") + TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af"), + "upstream" ) when: def act = producer.produceAdded(update) @@ -112,7 +114,8 @@ class ProduceLogsSpec extends Specification { BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.NEW, - TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af") + TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af"), + "upstream" ) when: def act = producer.produceAdded(update) @@ -157,7 +160,8 @@ class ProduceLogsSpec extends Specification { BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.NEW, - TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af") + TxId.from("0x6c88df9d65ccc9351db65676c3581b29483e8dabb71c48ef7671c44b0d5568af"), + "upstream" ) when: def act = producer.produceAdded(update) @@ -267,7 +271,8 @@ class ProduceLogsSpec extends Specification { BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.NEW, - TxId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec") + TxId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), + "upstream" ) when: def act = producer.produceAdded(update) @@ -345,13 +350,15 @@ class ProduceLogsSpec extends Specification { BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.NEW, - TxId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec") + TxId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), + "upstream" ) def update2 = new ConnectBlockUpdates.Update( BlockId.from("0x668b92d6b8c7db1350fd527fec4885ce5be2159b2b7daf6b126babdcbaa349da"), 13412871, ConnectBlockUpdates.UpdateType.DROP, - TxId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec") + TxId.from("0xb5e554178a94fd993111f2ae64cb708cb0899d7b5182024e70d5c468164a8bec"), + "upstream" ) when: // first need to produce them as added, because that's when it remembers logs to "remove" diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessageSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessageSpec.groovy index 826dd710..da1aa41a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessageSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessageSpec.groovy @@ -41,7 +41,8 @@ class LogMessageSpec extends Specification { ], TransactionId.from("0xc7529e79f78f58125abafeaea01fe3abdc6f45c173d5dfb36716cbc526e5b2d1"), 0xa3, - false) + false, + "LogMessageSpec") ObjectMapper objectMapper = Global.getObjectMapper() def exp = '{' + '"address":"0x011b6e24ffb0b5f5fcc564cf4183c5bbbc96d515",' + diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessageSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessageSpec.groovy index e490083e..f0db8a88 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessageSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessageSpec.groovy @@ -23,7 +23,8 @@ class NewHeadMessageSpec extends Specification { 0x7bb33e, Bloom.from("0x012040020880820356a20b8e980a2004c19f1291800501040001180bb029d0002d8c49440e002048ca00d48581000d900a458100c90139056140880582f22a0a8050224c020c233be8c3080c0a016aa4226a2001446c800822080445a2454118139804001202068401841900840c484222420c4b2022046052c0011e81a9e450085883708545810592e40040010411442300080b0130711f602880600a30c90702cb420a0102a644820650908802840810948142541404884300acc69d000840702020224c000020200880c10858418408098a61445b0ab0480234862655a5000434311b91044849c165040411aa0400b00008222642d24313020d9022219120"), Address.from("0x829bd824b016326a401d083b33d092293333a830"), - null + null, + "NewHeadMessageSpec" ) ObjectMapper objectMapper = Global.getObjectMapper() def exp = '{' + diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/MostWorkForkChoiceSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/MostWorkForkChoiceSpec.groovy index dac1ec85..e4488f23 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/MostWorkForkChoiceSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/MostWorkForkChoiceSpec.groovy @@ -11,7 +11,7 @@ class MostWorkForkChoiceSpec extends Specification { def blocks = [1L, 2, 3, 4].collect { i -> byte[] hash = new byte[32] hash[0] = i as byte - new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0) + new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0, "MostWorkForkChoiceSpec") } def "filters blocks"() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoiceSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoiceSpec.groovy index 20f90dae..6f862f60 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoiceSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoiceSpec.groovy @@ -10,7 +10,7 @@ class NoChoiceWithPriorityForkChoiceSpec extends Specification { def blocks = [1L, 2, 3, 4].collect { i -> byte[] hash = new byte[32] hash[0] = i as byte - new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0) + new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0, "NoChoiceWithPriorityForkChoiceSpec") } def "filters blocks"() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoiceSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoiceSpec.groovy index 25848281..7ef34289 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoiceSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoiceSpec.groovy @@ -10,7 +10,7 @@ class PriorityForkChoiceSpec extends Specification { def blocks = [1L, 2, 3, 4].collect { i -> byte[] hash = new byte[32] hash[0] = i as byte - new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], i.toInteger()) + new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], i.toInteger(), "PriorityForkChoiceSpec") } def "filters blocks"() { def choice = new PriorityForkChoice() diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy index 7368290d..408488b7 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy @@ -95,7 +95,7 @@ class EcdsaSignerSpec extends Specification { def signer = new EcdsaSigner(Stub(ECPrivateKey), 100L) when: - def act = signer.wrapMessage(10, "test".bytes, up) + def act = signer.wrapMessage(10, "test".bytes, up.id) then: act == "DSHACKLESIG/10/infura/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" @@ -118,7 +118,7 @@ class EcdsaSignerSpec extends Specification { def signer = new EcdsaSigner((pair.getPrivate() as ECPrivateKey), 100L) when: - def sig = signer.sign(10, result, up) + def sig = signer.sign(10, result, up.id) then: verifier.verify(sig.value) @@ -148,7 +148,7 @@ class EcdsaSignerSpec extends Specification { def signer = factory.getObject() as EcdsaSigner when: - def sig = signer.sign(10, result, up) + def sig = signer.sign(10, result, up.id) println("Signature: ${Hex.encodeHexString(sig.value)}") then: