Add determination of the lower block (#372)

This commit is contained in:
KirillPamPam
2023-12-26 16:19:20 +04:00
committed by GitHub
parent e79df25329
commit eb774d73e8
26 changed files with 607 additions and 9 deletions

View File

@@ -46,6 +46,8 @@ class Describe(
.addAllSupportedSubscriptions(chainUpstreams.getEgressSubscription().getAvailableTopics())
.setStatus(status)
.setCurrentHeight(chainUpstreams.getHead().getCurrentHeight() ?: 0)
.setCurrentLowerBlock(chainUpstreams.getLowerBlock().blockNumber)
.setCurrentLowerSlot(chainUpstreams.getLowerBlock().slot ?: 0)
chainUpstreams.getQuorumLabels()
.forEach { node ->
val nodeDetails = BlockchainOuterClass.NodeDetails.newBuilder()

View File

@@ -21,6 +21,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
@@ -39,20 +40,23 @@ class StreamHead(
return requestMono.map { request ->
Chain.byId(request.type.number)
}.flatMapMany { chain ->
multistreamHolder.getUpstream(chain).getHead()
val ms = multistreamHolder.getUpstream(chain)
ms.getHead()
.getFlux()
.map { asProto(chain, it!!) }
.map { asProto(ms, chain, it!!) }
.onErrorContinue { t, _ ->
log.warn("Head subscription error", t)
}
}
}
fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
fun asProto(ms: Multistream, chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
return BlockchainOuterClass.ChainHead.newBuilder()
.setChainValue(chain.id)
.setHeight(block.height)
.setSlot(block.slot)
.setCurrentLowerBlock(ms.getLowerBlock().blockNumber)
.setCurrentLowerSlot(ms.getLowerBlock().slot ?: 0)
.setTimestamp(block.timestamp.toEpochMilli())
.setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
.setBlockId(block.hash.toHex())

View File

@@ -87,6 +87,7 @@ open class GenericUpstreamCreator(
connectorFactory,
cs::validator,
cs::labelDetector,
cs::lowerBoundBlockDetector,
)
upstream.start()

View File

@@ -0,0 +1,68 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.time.Duration
import java.util.concurrent.atomic.AtomicReference
typealias LowerBoundBlockDetectorBuilder = (Chain, Upstream) -> LowerBoundBlockDetector
fun Long.toHex() = "0x${this.toString(16)}"
abstract class LowerBoundBlockDetector(
private val chain: Chain,
private val upstream: Upstream,
) {
private val currentLowerBlock = AtomicReference(LowerBlockData.default())
private val log = LoggerFactory.getLogger(this::class.java)
fun lowerBlock(): Flux<LowerBlockData> {
return Flux.interval(
Duration.ofSeconds(15),
Duration.ofSeconds(60),
)
.flatMap { lowerBlockDetect() }
.filter { it.blockNumber > currentLowerBlock.get().blockNumber }
.map {
log.info("Lower block of ${upstream.getId()} $chain: block height - {}, slot - {}", it.blockNumber, it.slot ?: "NA")
currentLowerBlock.set(it)
it
}
}
fun getCurrentLowerBlock(): LowerBlockData = currentLowerBlock.get()
protected abstract fun lowerBlockDetect(): Mono<LowerBlockData>
data class LowerBlockData(
val blockNumber: Long,
val slot: Long?,
) : Comparable<LowerBlockData> {
constructor(blockNumber: Long) : this(blockNumber, null)
companion object {
fun default() = LowerBlockData(0, 0)
}
override fun compareTo(other: LowerBlockData): Int {
return this.blockNumber.compareTo(other.blockNumber)
}
}
data class LowerBoundData(
val left: Long,
val right: Long,
val current: Long,
val found: Boolean,
) {
constructor(left: Long, right: Long) : this(left, right, 0, false)
constructor(left: Long, right: Long, current: Long) : this(left, right, current, false)
constructor(current: Long, found: Boolean) : this(0, 0, current, found)
}
}

View File

@@ -78,6 +78,9 @@ abstract class Multistream(
@Volatile
private var capabilities: Set<Capability> = emptySet()
@Volatile
private var lowerBlock: LowerBoundBlockDetector.LowerBlockData = LowerBoundBlockDetector.LowerBlockData.default()
@Volatile
private var quorumLabels: List<QuorumForLabels.QuorumItem>? = null
private val meters: MutableMap<String, List<Meter.Id>> = HashMap()
@@ -227,6 +230,10 @@ abstract class Multistream(
}
}
quorumLabels = getQuorumLabels(availableUpstreams)
availableUpstreams
.filter { it.getLowerBlock() != LowerBoundBlockDetector.LowerBlockData.default() }
.minOfOrNull { it.getLowerBlock() }
?.let { lowerBlock = it }
when {
upstreams.size == 1 -> {
lagObserver?.stop()
@@ -313,11 +320,11 @@ abstract class Multistream(
started = true
}
override fun getLowerBlock(): LowerBoundBlockDetector.LowerBlockData = lowerBlock
private fun observeUpstreamsStatuses() {
subscribeAddedUpstreams()
.distinctUntilChanged {
it.getId()
}.flatMap { upstream ->
.flatMap { upstream ->
val statusStream = upstream.observeStatus()
.map { UpstreamChangeEvent(this.chain, upstream, UpstreamChangeEvent.ChangeType.UPDATED) }
val stateStream = upstream.observeState()
@@ -407,9 +414,10 @@ abstract class Multistream(
val weak = getUpstreams()
.filter { it.getStatus() != UpstreamAvailability.OK }
.joinToString(", ") { it.getId() }
val lowerBlockData = "[height=${lowerBlock.blockNumber}, slot=${lowerBlock.slot ?: "NA"}]"
val instance = System.identityHashCode(this).toString(16)
log.info("State of ${chain.chainCode}: height=${height ?: '?'}, status=[$statuses], lag=[$lag], weak=[$weak] ($instance)")
log.info("State of ${chain.chainCode}: height=${height ?: '?'}, status=[$statuses], lag=[$lag], lower block=$lowerBlockData, weak=[$weak] ($instance)")
}
fun test(event: UpstreamChangeEvent): Boolean {

View File

@@ -0,0 +1,53 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain
import reactor.core.publisher.Mono
abstract class RecursiveLowerBoundBlockDetector(
chain: Chain,
private val upstream: Upstream,
) : LowerBoundBlockDetector(chain, upstream) {
override fun lowerBlockDetect(): Mono<LowerBlockData> {
return Mono.just(upstream.getHead())
.flatMap {
val currentHeight = it.getCurrentHeight()
if (currentHeight == null) {
Mono.empty()
} else {
Mono.just(LowerBoundData(0, currentHeight))
}
}
.expand { data ->
if (data.found) {
Mono.empty()
} else {
val middle = middleBlock(data)
if (data.left > data.right) {
val current = if (data.current == 0L) 1 else data.current
Mono.just(LowerBoundData(current, true))
} else {
hasState(middle)
.map {
if (it) {
LowerBoundData(data.left, middle - 1, middle)
} else {
LowerBoundData(middle + 1, data.right, data.current)
}
}
}
}
}
.filter { it.found }
.next()
.map {
LowerBlockData(it.current)
}
}
private fun middleBlock(lowerBoundData: LowerBoundData): Long =
lowerBoundData.left + (lowerBoundData.right - lowerBoundData.left) / 2
protected abstract fun hasState(blockNumber: Long): Mono<Boolean>
}

View File

@@ -44,6 +44,7 @@ interface Upstream : Lifecycle {
fun getId(): String
fun getCapabilities(): Set<Capability>
fun isGrpc(): Boolean
fun getLowerBlock(): LowerBoundBlockDetector.LowerBlockData
fun <T : Upstream> cast(selfType: Class<T>): T

View File

@@ -25,6 +25,7 @@ import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -71,6 +72,10 @@ open class BitcoinRpcUpstream(
return false
}
override fun getLowerBlock(): LowerBoundBlockDetector.LowerBlockData {
return LowerBoundBlockDetector.LowerBlockData.default()
}
@Suppress("UNCHECKED_CAST")
override fun <T : Upstream> cast(selfType: Class<T>): T {
if (!selfType.isAssignableFrom(this.javaClass)) {

View File

@@ -12,6 +12,7 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.LabelsDetector
import io.emeraldpay.dshackle.upstream.LogsOracle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamValidator
@@ -88,6 +89,10 @@ object EthereumChainSpecific : AbstractPollChainSpecific() {
return EthereumUpstreamValidator(chain, upstream, options, config)
}
override fun lowerBoundBlockDetector(chain: Chain, upstream: Upstream): LowerBoundBlockDetector {
return EthereumLowerBoundBlockDetector(chain, upstream)
}
override fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector {
return EthereumLabelsDetector(reader, chain)
}

View File

@@ -0,0 +1,27 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.RecursiveLowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.toHex
import reactor.core.publisher.Mono
class EthereumLowerBoundBlockDetector(
chain: Chain,
private val upstream: Upstream,
) : RecursiveLowerBoundBlockDetector(chain, upstream) {
override fun hasState(blockNumber: Long): Mono<Boolean> {
return upstream.getIngressReader().read(
JsonRpcRequest(
"eth_getBalance",
listOf("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0", blockNumber.toHex()),
),
)
.flatMap(JsonRpcResponse::requireResult)
.map { true }
.onErrorReturn(false)
}
}

View File

@@ -18,6 +18,7 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.LabelsDetector
import io.emeraldpay.dshackle.upstream.LogsOracle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamValidator
@@ -60,6 +61,8 @@ interface ChainSpecific {
fun makeIngressSubscription(ws: WsSubscriptions): IngressSubscription
fun callSelector(caches: Caches): CallSelector?
fun lowerBoundBlockDetector(chain: Chain, upstream: Upstream): LowerBoundBlockDetector
}
object ChainSpecificRegistry {

View File

@@ -13,6 +13,8 @@ import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.LabelsDetectorBuilder
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetectorBuilder
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.UpstreamValidator
@@ -40,6 +42,7 @@ open class GenericUpstream(
connectorFactory: ConnectorFactory,
validatorBuilder: UpstreamValidatorBuilder,
labelsDetectorBuilder: LabelsDetectorBuilder,
lowerBoundBlockDetectorBuilder: LowerBoundBlockDetectorBuilder,
) : DefaultUpstream(id, hash, null, UpstreamAvailability.OK, options, role, targets, node, chainConfig), Lifecycle {
private val validator: UpstreamValidator? = validatorBuilder(chain, this, getOptions(), chainConfig)
@@ -51,6 +54,8 @@ open class GenericUpstream(
private var livenessSubscription: Disposable? = null
private val labelsDetector = labelsDetectorBuilder(chain, this.getIngressReader())
private val lowerBoundBlockDetector = lowerBoundBlockDetectorBuilder(chain, this)
override fun getHead(): Head {
return connector.getHead()
}
@@ -77,6 +82,10 @@ open class GenericUpstream(
return false
}
override fun getLowerBlock(): LowerBoundBlockDetector.LowerBlockData {
return lowerBoundBlockDetector.getCurrentLowerBlock()
}
@Suppress("UNCHECKED_CAST")
override fun <T : Upstream> cast(selfType: Class<T>): T {
if (!selfType.isAssignableFrom(this.javaClass)) {
@@ -162,6 +171,8 @@ open class GenericUpstream(
log.debug("Error while checking live subscription for ${getId()}", it)
},)
detectLabels()
detectLowerBlock()
}
override fun stop() {
@@ -185,6 +196,15 @@ open class GenericUpstream(
}
}
private fun detectLowerBlock() {
lowerBoundBlockDetector.lowerBlock()
.subscribe {
stateEventStream.emitNext(
UpstreamChangeEvent(chain, this, UpstreamChangeEvent.ChangeType.UPDATED),
) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED }
}
}
fun getIngressSubscription(): IngressSubscription {
return connector.getIngressSubscription()
}

View File

@@ -29,6 +29,7 @@ import io.emeraldpay.dshackle.upstream.BuildInfo
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream
@@ -148,6 +149,10 @@ class BitcoinGrpcUpstream(
return true
}
override fun getLowerBlock(): LowerBoundBlockDetector.LowerBlockData {
return LowerBoundBlockDetector.LowerBlockData.default()
}
@Suppress("UNCHECKED_CAST")
override fun <T : Upstream> cast(selfType: Class<T>): T {
if (!selfType.isAssignableFrom(this.javaClass)) {

View File

@@ -31,6 +31,7 @@ import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
@@ -177,4 +178,8 @@ open class GenericGrpcUpstream(
override fun isGrpc(): Boolean {
return true
}
override fun getLowerBlock(): LowerBoundBlockDetector.LowerBlockData {
return LowerBoundBlockDetector.LowerBlockData.default()
}
}

View File

@@ -14,6 +14,7 @@ import io.emeraldpay.dshackle.upstream.EgressSubscription
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.LogsOracle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.SingleCallValidator
import io.emeraldpay.dshackle.upstream.Upstream
@@ -103,6 +104,10 @@ object PolkadotChainSpecific : AbstractPollChainSpecific() {
)
}
override fun lowerBoundBlockDetector(chain: Chain, upstream: Upstream): LowerBoundBlockDetector {
return PolkadotLowerBoundBlockDetector(chain, upstream)
}
fun validate(data: ByteArray, peers: Int, upstreamId: String): UpstreamAvailability {
val resp = Global.objectMapper.readValue(data, PolkadotHealth::class.java)
if (resp.isSyncing) {

View File

@@ -0,0 +1,39 @@
package io.emeraldpay.dshackle.upstream.polkadot
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.RecursiveLowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.toHex
import reactor.core.publisher.Mono
class PolkadotLowerBoundBlockDetector(
chain: Chain,
private val upstream: Upstream,
) : RecursiveLowerBoundBlockDetector(chain, upstream) {
override fun hasState(blockNumber: Long): Mono<Boolean> {
return upstream.getIngressReader().read(
JsonRpcRequest(
"chain_getBlockHash",
listOf(blockNumber.toHex()), // in polkadot state methods work only with hash
),
)
.flatMap(JsonRpcResponse::requireResult)
.map {
String(it, 1, it.size - 2)
}
.flatMap {
upstream.getIngressReader().read(
JsonRpcRequest(
"state_getMetadata",
listOf(it),
),
)
}
.flatMap(JsonRpcResponse::requireResult)
.map { true }
.onErrorReturn(false)
}
}

View File

@@ -13,6 +13,7 @@ import io.emeraldpay.dshackle.upstream.DefaultSolanaMethods
import io.emeraldpay.dshackle.upstream.EgressSubscription
import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.LabelsDetector
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.SingleCallValidator
import io.emeraldpay.dshackle.upstream.Upstream
@@ -136,6 +137,10 @@ object SolanaChainSpecific : AbstractChainSpecific() {
)
}
override fun lowerBoundBlockDetector(chain: Chain, upstream: Upstream): LowerBoundBlockDetector {
return SolanaLowerBoundBlockDetector(chain, upstream)
}
override fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector? {
return null
}

View File

@@ -0,0 +1,72 @@
package io.emeraldpay.dshackle.upstream.solana
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import reactor.core.publisher.Mono
class SolanaLowerBoundBlockDetector(
chain: Chain,
upstream: Upstream,
) : LowerBoundBlockDetector(chain, upstream) {
private val reader = upstream.getIngressReader()
override fun lowerBlockDetect(): Mono<LowerBlockData> {
return Mono.just(reader)
.flatMap {
it.read(
JsonRpcRequest("getFirstAvailableBlock", listOf()), // in case of solana we talk about the slot of the lowest confirmed block
)
}
.flatMap(JsonRpcResponse::requireResult)
.map {
String(it).toLong()
}
.flatMap { slot ->
reader.read(
JsonRpcRequest(
"getBlocks",
listOf(
slot - 10,
slot,
),
),
)
}
.flatMap(JsonRpcResponse::requireResult)
.flatMap {
val response = Global.objectMapper.readValue(it, LongArray::class.java)
if (response == null || response.isEmpty()) {
Mono.empty()
} else {
val maxSlot = response.max()
reader.read(
JsonRpcRequest(
"getBlock",
listOf(
maxSlot,
mapOf(
"showRewards" to false,
"transactionDetails" to "none",
"maxSupportedTransactionVersion" to 0,
),
),
),
)
.flatMap(JsonRpcResponse::requireResult)
.map { blockData ->
val block = Global.objectMapper.readValue(blockData, SolanaBlock::class.java)
LowerBlockData(block.height, maxSlot)
}.onErrorResume {
Mono.empty()
}
}
}
.onErrorResume {
Mono.empty()
}
}
}

View File

@@ -8,6 +8,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions.Options
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.SingleCallValidator
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
@@ -69,6 +70,10 @@ object StarknetChainSpecific : AbstractPollChainSpecific() {
)
}
override fun lowerBoundBlockDetector(chain: Chain, upstream: Upstream): LowerBoundBlockDetector {
return StarknetLowerBoundBlockDetector(chain, upstream)
}
fun validate(data: ByteArray, lagging: Int, upstreamId: String): UpstreamAvailability {
val resp = Global.objectMapper.readValue(data, StarknetSyncing::class.java)
return if (resp.highest - resp.current > lagging) {

View File

@@ -0,0 +1,17 @@
package io.emeraldpay.dshackle.upstream.starknet
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import reactor.core.publisher.Mono
class StarknetLowerBoundBlockDetector(
chain: Chain,
upstream: Upstream,
) : LowerBoundBlockDetector(chain, upstream) {
// for starknet we assume that all nodes are archive
override fun lowerBlockDetect(): Mono<LowerBlockData> {
return Mono.just(LowerBlockData(1))
}
}