StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enriched merge head for ethereum subscription (#262)

Browse Source
This commit is contained in:
Vyacheslav
2023-08-01 14:11:36 +03:00
committed by GitHub
parent 9757dfd201
commit 8b6b95e46e
10 changed files with 393 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EnrichedMergedHead.kt Normal file
View File

@@ -0,0 +1,94 @@
package io.emeraldpay.dshackle.upstream.ethereum
import com.google.common.cache.CacheBuilder
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.etherjar.domain.BlockHash
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.Sinks
import reactor.core.scheduler.Scheduler
import java.time.Duration
class EnrichedMergedHead constructor(
private val sources: Iterable<Head>,
private val referenceHead: Head,
private val headScheduler: Scheduler,
private val api: Reader<BlockHash, BlockContainer>
) : Head, Lifecycle {
private val enrichedBlocks = CacheBuilder.newBuilder()
.maximumSize(10)
.build<BlockId, BlockContainer>()
private val enrichedPromises = CacheBuilder.newBuilder()
.maximumSize(10)
.build<BlockId, Sinks.One<BlockContainer>>()
private var cacheSub: Disposable? = null
private fun getEnrichBlockMono(id: BlockId): Mono<BlockContainer> {
val block = enrichedBlocks.getIfPresent(id)
return if (block != null) {
Mono.just(block)
} else {
enrichedPromises.get(id) {
Sinks.one()
}.asMono()
}
}
override fun getFlux(): Flux<BlockContainer> {
return referenceHead.getFlux().concatMap { block ->
if (block.enriched) {
Mono.just(block)
} else {
Mono.firstWithValue(
getEnrichBlockMono(block.hash),
Mono.just(block)
.delayElement(Duration.ofSeconds(1))
.flatMap {
EthereumBlockEnricher.enrich(BlockHash(block.hash.value), api, headScheduler)
}
)
}
}
}
override fun onBeforeBlock(handler: Runnable) {}
override fun getCurrentHeight(): Long? {
return referenceHead.getCurrentHeight()
}
override fun isRunning(): Boolean {
return cacheSub != null
}
override fun start() {
cacheSub?.dispose()
sources.forEach { head ->
if (head is Lifecycle && !head.isRunning()) {
head.start()
}
}
if (referenceHead is Lifecycle && !referenceHead.isRunning()) {
referenceHead.start()
}
cacheSub = Flux.merge(sources.map { it.getFlux() }).subscribe { block ->
if (block.enriched) {
enrichedBlocks.put(block.hash, block)
enrichedPromises.get(block.hash) { Sinks.one() }.tryEmitValue(block)
}
}
}
override fun stop() {
cacheSub?.dispose()
cacheSub = null
}
override fun onSyncingNode(isSyncing: Boolean) {}
}

29
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumBlockEnricher.kt Normal file
View File

@@ -0,0 +1,29 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.etherjar.domain.BlockHash
import reactor.core.publisher.Mono
import reactor.core.scheduler.Scheduler
import reactor.retry.Repeat
import java.time.Duration
class EthereumBlockEnricher {
companion object {
fun enrich(blockHash: BlockHash, api: Reader<BlockHash, BlockContainer>, scheduler: Scheduler): Mono<BlockContainer> {
return Mono.just(blockHash)
.flatMap { hash ->
api.read(hash)
.subscribeOn(scheduler)
.timeout(Defaults.timeoutInternal, Mono.empty())
}.repeatWhenEmpty { n ->
Repeat.times<Any>(5)
.exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(500))
.apply(n)
}
.timeout(Defaults.timeout, Mono.empty())
.onErrorResume { Mono.empty() }
}
}
}

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt
View File

@@ -12,6 +12,8 @@ interface EthereumLikeMultistream : Upstream, HasEgressSubscription {
fun getHead(mather: Selector.Matcher): Head
fun getEnrichedHead(mather: Selector.Matcher): Head
/**
* Tries to proxy the native subscribe request to the managed upstreams if
* - any of them matches the matcher criteria

22
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt
View File

@@ -20,7 +20,9 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.DynamicMergedHead
import io.emeraldpay.dshackle.upstream.EgressSubscription
@@ -38,6 +40,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream
import io.emeraldpay.etherjar.domain.BlockHash
import org.springframework.cloud.sleuth.Tracer
import org.springframework.util.ConcurrentReferenceHashMap
import reactor.core.publisher.Flux
@@ -209,6 +212,25 @@ open class EthereumMultistream(
}
}
override fun getEnrichedHead(mather: Selector.Matcher): Head =
filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ ->
upstreams.filter { mather.matches(it) }
.apply {
log.debug("Found $size upstreams matching [${mather.describeInternal()}]")
}.let {
val selected = it.map { source -> source.getHead() }
EnrichedMergedHead(
selected, getHead(), headScheduler,
object :
Reader<BlockHash, BlockContainer> {
override fun read(key: BlockHash): Mono<BlockContainer> {
return reader.blocksByHashAsCont().read(key).map { res -> res.data }
}
}
)
}
}
override fun getFeeEstimation(): ChainFees {
return feeEstimation
}

48
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt
View File

@@ -16,24 +16,24 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.Sinks
import reactor.core.scheduler.Scheduler
import reactor.retry.Repeat
import java.time.Duration
class EthereumWsHead(
@@ -101,7 +101,25 @@ class EthereumWsHead(
block.totalDifficulty == null
)
) {
enhanceRealBlock(block)
EthereumBlockEnricher.enrich(
block.hash,
object :
Reader<BlockHash, BlockContainer> {
override fun read(key: BlockHash): Mono<BlockContainer> {
return api.read(JsonRpcRequest("eth_getBlockByHash", listOf(block.hash.toHex(), false)))
.flatMap { resp ->
if (resp.isNull()) {
Mono.error(SilentException("Received null for block ${block.hash}"))
} else {
Mono.just(resp)
}
}
.flatMap(JsonRpcResponse::requireResult)
.map { BlockContainer.fromEthereumJson(it, upstreamId) }
}
},
headScheduler
)
} else {
Mono.just(BlockContainer.from(block))
}
@@ -113,30 +131,6 @@ class EthereumWsHead(
}
}
fun enhanceRealBlock(block: BlockJson<TransactionRefJson>): Mono<BlockContainer> {
return Mono.just(block.hash)
.flatMap { hash ->
api.read(JsonRpcRequest("eth_getBlockByHash", listOf(hash.toHex(), false)))
.flatMap { resp ->
if (resp.isNull()) {
Mono.error(SilentException("Received null for block $hash"))
} else {
Mono.just(resp)
}
}
.flatMap(JsonRpcResponse::requireResult)
.map { BlockContainer.fromEthereumJson(it, upstreamId) }
.subscribeOn(headScheduler)
.timeout(Defaults.timeoutInternal, Mono.empty())
}.repeatWhenEmpty { n ->
Repeat.times<Any>(5)
.exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(500))
.apply(n)
}
.timeout(Defaults.timeout, Mono.empty())
.onErrorResume { Mono.empty() }
}
override fun stop() {
super.stop()
cancelSub()

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt
View File

@@ -51,7 +51,7 @@ class ConnectBlockUpdates(
fun connect() = connect(Selector.empty)
override fun connect(matcher: Selector.Matcher): Flux<Update> {
return connected.computeIfAbsent(matcher.describeInternal()) { key ->
extract(upstream.getHead(matcher))
extract(upstream.getEnrichedHead(matcher))
.publishOn(scheduler)
.publish()
.refCount(1, Duration.ofSeconds(60))

22
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt
View File

@@ -20,7 +20,9 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.DynamicMergedHead
import io.emeraldpay.dshackle.upstream.EmptyHead
@@ -36,6 +38,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.subscribe.NoPendingTxes
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource
import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream
import io.emeraldpay.etherjar.domain.BlockHash
import org.springframework.cloud.sleuth.Tracer
import org.springframework.util.ConcurrentReferenceHashMap
import reactor.core.publisher.Flux
@@ -179,6 +182,25 @@ open class EthereumPosMultiStream(
}
}
override fun getEnrichedHead(mather: Selector.Matcher): Head =
filteredHeads.computeIfAbsent(mather.describeInternal().intern()) { _ ->
upstreams.filter { mather.matches(it) }
.apply {
log.debug("Found $size upstreams matching [${mather.describeInternal()}]")
}.let {
val selected = it.map { source -> source.getHead() }
EnrichedMergedHead(
selected, getHead(), headScheduler,
object :
Reader<BlockHash, BlockContainer> {
override fun read(key: BlockHash): Mono<BlockContainer> {
return reader.blocksByHashAsCont().read(key).map { res -> res.data }
}
}
)
}
}
override fun getFeeEstimation(): ChainFees {
return feeEstimation
}