Integrate rx/receipts gold bounds (#781)

This commit is contained in:
KirillPamPam
2026-01-28 12:54:50 +04:00
committed by GitHub
parent 2e4c99b202
commit 3b5940cc20
14 changed files with 374 additions and 12 deletions

View File

@@ -44,6 +44,25 @@ data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<Chains
fun rules() = conditions.joinToString { (op, limit) -> "$op $limit" }
}
enum class LowerBoundType {
UNKNOWN, STATE, SLOT, BLOCK, TX, LOGS, TRACE, PROOF, BLOB, EPOCH, RECEIPTS;
companion object {
fun byName(name: String): LowerBoundType {
return LowerBoundType.entries.firstOrNull { it.name.equals(name, ignoreCase = true) } ?: UNKNOWN
}
}
}
open class GoldLowerBound(
val block: Long,
)
class GoldLowerBoundWithHash(
block: Long,
val hash: String,
) : GoldLowerBound(block)
data class ChainConfig(
val expectedBlockTime: Duration,
val syncingLagSize: Int,
@@ -59,6 +78,7 @@ data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<Chains
val blockchain: String,
val type: String,
val gasPriceCondition: GasPriceCondition,
val goldLowerBounds: Map<LowerBoundType, GoldLowerBound>
) {
companion object {
@JvmStatic
@@ -80,6 +100,7 @@ data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<Chains
"undefined",
"unknown",
GasPriceCondition(emptyList()),
emptyMap(),
)
@JvmStatic
@@ -92,4 +113,8 @@ data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<Chains
fun resolve(chain: String): ChainConfig {
return chainMap[chain] ?: ChainConfig.default()
}
fun getChainConfigs(): Collection<ChainConfig> {
return chainMap.values
}
}

View File

@@ -137,9 +137,47 @@ class ChainsConfigReader(
blockchain = blockchain,
type = type,
gasPriceCondition = ChainsConfig.GasPriceCondition(gasPriceConditions),
goldLowerBounds = readGoldLowerBounds(node),
)
}
private fun readGoldLowerBounds(node: MappingNode): Map<ChainsConfig.LowerBoundType, ChainsConfig.GoldLowerBound> {
val goldLowerBounds = getMapping(node, "lower-bounds") ?: return emptyMap()
return goldLowerBounds.value.asSequence()
.filter {
(it.keyNode.valueAsString()?.isNotBlank() ?: false) && it.valueNode.tag == Tag.MAP
}.map {
val type = ChainsConfig.LowerBoundType.byName(it.keyNode.valueAsString()!!)
val bound = if (type == ChainsConfig.LowerBoundType.TX || type == ChainsConfig.LowerBoundType.RECEIPTS) {
readHashGoldLowerBound(it.valueNode as MappingNode)
} else {
readGoldLowerBound(it.valueNode as MappingNode)
}
if (bound != null) {
type to bound
} else {
null
}
}
.filterNotNull()
.associate { it }
}
private fun readGoldLowerBound(node: MappingNode): ChainsConfig.GoldLowerBound? {
val block = getValueAsLong(node, "block") ?: return null
return ChainsConfig.GoldLowerBound(block)
}
private fun readHashGoldLowerBound(node: MappingNode): ChainsConfig.GoldLowerBound? {
val hash = getValueAsString(node, "hash") ?: return null
val block = getValueAsLong(node, "block") ?: return null
return ChainsConfig.GoldLowerBoundWithHash(block, hash)
}
override fun read(input: MappingNode?): ChainsConfig {
val default = readChains(readNode(defaultConfig))
val current = readChains(input)

View File

@@ -1,12 +1,13 @@
package org.drpc.chainsconfig
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.ChainsConfigReader
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import java.math.BigInteger
internal class ChainsConfigReaderTest {
class ChainsConfigReaderTest {
@Test
fun `read standard config without custom`() {
@@ -41,5 +42,17 @@ internal class ChainsConfigReaderTest {
assertEquals(config.code, "FTA")
assertEquals(config.grpcId, 102)
assertEquals(config.netVersion, BigInteger.valueOf(251))
assertEquals(15L, config.goldLowerBounds[ChainsConfig.LowerBoundType.STATE]!!.block)
assertEquals(150L, config.goldLowerBounds[ChainsConfig.LowerBoundType.BLOCK]!!.block)
assertEquals(1L, config.goldLowerBounds[ChainsConfig.LowerBoundType.TX]!!.block)
assertEquals(
"0x5e77a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a",
(config.goldLowerBounds[ChainsConfig.LowerBoundType.TX] as ChainsConfig.GoldLowerBoundWithHash).hash,
)
assertEquals(1000L, config.goldLowerBounds[ChainsConfig.LowerBoundType.RECEIPTS]!!.block)
assertEquals(
"0x4e72a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a",
(config.goldLowerBounds[ChainsConfig.LowerBoundType.RECEIPTS] as ChainsConfig.GoldLowerBoundWithHash).hash,
)
}
}

View File

@@ -16,4 +16,15 @@ chain-settings:
short-names: [fantom]
code: FTA
grpcId: 102
chain-id: 0xfb
chain-id: 0xfb
lower-bounds:
tx:
block: 1
hash: "0x5e77a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a"
receipts:
block: 1000
hash: "0x4e72a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a"
state:
block: 15
block:
block: 150

View File

@@ -9,6 +9,8 @@ import io.emeraldpay.dshackle.foundation.ChainOptions.Options
import io.emeraldpay.dshackle.upstream.CallTargetsHolder
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods
import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds
import jakarta.annotation.PostConstruct
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.util.function.Function
@@ -20,6 +22,11 @@ abstract class UpstreamCreator(
) {
protected val log: Logger = LoggerFactory.getLogger(this::class.java)
@PostConstruct
fun init() {
GoldLowerBounds.init(chainsConfig.getChainConfigs())
}
companion object {
fun getHash(nodeId: Int?, obj: Any, hashes: MutableSet<Short>): Short {
val hash = nodeId?.toShort()

View File

@@ -1,15 +1,19 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds
import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
import io.emeraldpay.dshackle.upstream.lowerbound.detector.RecursiveLowerBound
import io.emeraldpay.dshackle.upstream.lowerbound.toHex
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
class EthereumLowerBoundReceiptsDetector(
private val upstream: Upstream,
@@ -29,6 +33,7 @@ class EthereumLowerBoundReceiptsDetector(
"invalid block height", // hyperliquid
"header not found",
"pruned history unavailable", // xlayer
"transaction indexing is in progress",
).plus(EthereumLowerBoundBlockDetector.NO_BLOCK_ERRORS)
}
@@ -39,6 +44,34 @@ class EthereumLowerBoundReceiptsDetector(
}
override fun internalDetectLowerBound(): Flux<LowerBoundData> {
val receiptsGoldBound = GoldLowerBounds.getBound(upstream.getChain(), LowerBoundType.RECEIPTS)
if (receiptsGoldBound == null || receiptsGoldBound !is ChainsConfig.GoldLowerBoundWithHash) {
return recursiveDetectReceiptsLowerBound()
}
return Mono.just(receiptsGoldBound)
.flatMapMany { bound ->
upstream.getIngressReader()
.read(ChainRequest("eth_getTransactionReceipt", ListParams(bound.hash)))
.timeout(Defaults.internalCallsTimeout)
.flatMap(ChainResponse::requireResult)
.flatMapMany {
if (it.contentEquals("null".toByteArray())) {
throw IllegalStateException("no gold bound")
} else {
Flux.just(LowerBoundData(1, LowerBoundType.RECEIPTS))
}
}
.onErrorResume {
recursiveDetectReceiptsLowerBound()
}
}
}
override fun types(): Set<LowerBoundType> {
return setOf(LowerBoundType.RECEIPTS)
}
private fun recursiveDetectReceiptsLowerBound(): Flux<LowerBoundData> {
return recursiveLowerBound.recursiveDetectLowerBoundWithOffset(MAX_OFFSET) { block ->
upstream.getIngressReader()
.read(
@@ -72,8 +105,4 @@ class EthereumLowerBoundReceiptsDetector(
}
}
}
override fun types(): Set<LowerBoundType> {
return setOf(LowerBoundType.RECEIPTS)
}
}

View File

@@ -1,15 +1,19 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds
import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
import io.emeraldpay.dshackle.upstream.lowerbound.detector.RecursiveLowerBound
import io.emeraldpay.dshackle.upstream.lowerbound.toHex
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
class EthereumLowerBoundTxDetector(
private val upstream: Upstream,
@@ -24,6 +28,7 @@ class EthereumLowerBoundTxDetector(
"Unexpected error", // hyperliquid
"invalid block height", // hyperliquids
"pruned history unavailable", // xlayer blocks
"transaction indexing is in progress",
).plus(EthereumLowerBoundBlockDetector.NO_BLOCK_ERRORS)
}
@@ -34,6 +39,34 @@ class EthereumLowerBoundTxDetector(
}
override fun internalDetectLowerBound(): Flux<LowerBoundData> {
val txGoldBound = GoldLowerBounds.getBound(upstream.getChain(), LowerBoundType.TX)
if (txGoldBound == null || txGoldBound !is ChainsConfig.GoldLowerBoundWithHash) {
return recursiveDetectTxLowerBound()
}
return Mono.just(txGoldBound)
.flatMapMany { bound ->
upstream.getIngressReader()
.read(ChainRequest("eth_getTransactionByHash", ListParams(bound.hash)))
.timeout(Defaults.internalCallsTimeout)
.flatMap(ChainResponse::requireResult)
.flatMapMany {
if (it.contentEquals("null".toByteArray())) {
throw IllegalStateException("no gold bound")
} else {
Flux.just(LowerBoundData(1, LowerBoundType.TX))
}
}
.onErrorResume {
recursiveDetectTxLowerBound()
}
}
}
override fun types(): Set<LowerBoundType> {
return setOf(LowerBoundType.TX)
}
private fun recursiveDetectTxLowerBound(): Flux<LowerBoundData> {
return recursiveLowerBound.recursiveDetectLowerBoundWithOffset(MAX_OFFSET) { block ->
upstream.getIngressReader()
.read(
@@ -67,8 +100,4 @@ class EthereumLowerBoundTxDetector(
}
}
}
override fun types(): Set<LowerBoundType> {
return setOf(LowerBoundType.TX)
}
}

View File

@@ -0,0 +1,28 @@
package io.emeraldpay.dshackle.upstream.lowerbound
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ChainsConfig
object GoldLowerBounds {
private lateinit var bounds: Map<Chain, Map<LowerBoundType, ChainsConfig.GoldLowerBound>>
fun init(chainConfigs: Collection<ChainsConfig.ChainConfig>) {
bounds = chainConfigs.filter {
it.shortNames.isNotEmpty() &&
Global.chainById(it.shortNames[0]) != Chain.UNSPECIFIED &&
it.goldLowerBounds.isNotEmpty()
}.associate { cfg ->
Global.chainById(cfg.shortNames[0]) to cfg.goldLowerBounds.mapKeys { toLowerBoundType(it.key) }
}
}
fun getBound(chain: Chain, boundType: LowerBoundType): ChainsConfig.GoldLowerBound? {
return bounds[chain]?.get(boundType)
}
private fun toLowerBoundType(type: ChainsConfig.LowerBoundType): LowerBoundType {
return LowerBoundType.byName(type.name)
}
}

View File

@@ -47,7 +47,7 @@ abstract class LowerBoundDetector(
},
)
.filter {
it.lowerBound >= (lowerBounds.getLastBound(it.type)?.lowerBound ?: 0)
(it.lowerBound >= (lowerBounds.getLastBound(it.type)?.lowerBound ?: 0)) || it.lowerBound == 1L
}
.map {
lowerBounds.updateBound(it)

View File

@@ -0,0 +1,56 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds
import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
import io.emeraldpay.dshackle.upstream.lowerbound.NoopManualLowerBoundService
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.junit.jupiter.api.Test
import org.mockito.kotlin.doReturn
import org.mockito.kotlin.mock
import org.mockito.kotlin.verify
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import java.time.Duration
class EthereumLowerBoundReceiptsDetectorTest {
@Test
fun `archival bound if there is a gold bound`() {
val reader = mock<ChainReader> {
on { read(ChainRequest("eth_getTransactionReceipt", ListParams("goldHash"))) } doReturn
Mono.just(ChainResponse("result".toByteArray(), null))
}
val upstream = mock<Upstream> {
on { getChain() } doReturn Chain.POLYGON__MAINNET
on { getIngressReader() } doReturn reader
}
GoldLowerBounds.init(
listOf(
ChainsConfig.ChainConfig
.default()
.copy(
shortNames = listOf("polygon"),
goldLowerBounds = mapOf(ChainsConfig.LowerBoundType.RECEIPTS to ChainsConfig.GoldLowerBoundWithHash(10L, "goldHash")),
),
),
)
val txDetector = EthereumLowerBoundReceiptsDetector(upstream)
StepVerifier.withVirtualTime { txDetector.detectLowerBound(NoopManualLowerBoundService()) }
.expectSubscription()
.expectNoEvent(Duration.ofSeconds(15))
.expectNextMatches { it.lowerBound == 1L && it.type == LowerBoundType.RECEIPTS }
.thenCancel()
.verify(Duration.ofSeconds(1))
verify(reader).read(ChainRequest("eth_getTransactionReceipt", ListParams("goldHash")))
verify(upstream).getIngressReader()
}
}

View File

@@ -0,0 +1,56 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds
import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
import io.emeraldpay.dshackle.upstream.lowerbound.NoopManualLowerBoundService
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.junit.jupiter.api.Test
import org.mockito.kotlin.doReturn
import org.mockito.kotlin.mock
import org.mockito.kotlin.verify
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import java.time.Duration
class EthereumLowerBoundTxDetectorTest {
@Test
fun `archival bound if there is a gold bound`() {
val reader = mock<ChainReader> {
on { read(ChainRequest("eth_getTransactionByHash", ListParams("goldHash"))) } doReturn
Mono.just(ChainResponse("result".toByteArray(), null))
}
val upstream = mock<Upstream> {
on { getChain() } doReturn Chain.POLYGON__MAINNET
on { getIngressReader() } doReturn reader
}
GoldLowerBounds.init(
listOf(
ChainsConfig.ChainConfig
.default()
.copy(
shortNames = listOf("polygon"),
goldLowerBounds = mapOf(ChainsConfig.LowerBoundType.TX to ChainsConfig.GoldLowerBoundWithHash(10L, "goldHash")),
),
),
)
val txDetector = EthereumLowerBoundTxDetector(upstream)
StepVerifier.withVirtualTime { txDetector.detectLowerBound(NoopManualLowerBoundService()) }
.expectSubscription()
.expectNoEvent(Duration.ofSeconds(15))
.expectNextMatches { it.lowerBound == 1L && it.type == LowerBoundType.TX }
.thenCancel()
.verify(Duration.ofSeconds(1))
verify(reader).read(ChainRequest("eth_getTransactionByHash", ListParams("goldHash")))
verify(upstream).getIngressReader()
}
}

View File

@@ -0,0 +1,63 @@
package io.emeraldpay.dshackle.upstream.lowerbound
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
class GoldLowerBoundsTest {
@Test
fun `init with all bounds`() {
val cfg = ChainsConfig.ChainConfig.default()
.copy(
shortNames = listOf("polygon"),
goldLowerBounds = ChainsConfig.LowerBoundType.entries
.associateWith {
if (it == ChainsConfig.LowerBoundType.TX || it == ChainsConfig.LowerBoundType.RECEIPTS) {
ChainsConfig.GoldLowerBoundWithHash(5L, "hash_$it")
} else {
ChainsConfig.GoldLowerBound(10L)
}
},
)
GoldLowerBounds.init(listOf(cfg))
LowerBoundType.entries
.filter { it != LowerBoundType.UNKNOWN }
.forEach {
val bound = GoldLowerBounds.getBound(Chain.POLYGON__MAINNET, it)
assertThat(bound).isNotNull
if (it == LowerBoundType.TX || it == LowerBoundType.RECEIPTS) {
assertThat(bound)
.usingRecursiveComparison()
.isEqualTo(ChainsConfig.GoldLowerBoundWithHash(5L, "hash_$it"))
} else {
assertThat(bound)
.usingRecursiveComparison()
.isEqualTo(ChainsConfig.GoldLowerBound(10L))
}
}
}
@Test
fun `no gold bound`() {
val cfg = ChainsConfig.ChainConfig.default()
.copy(
shortNames = listOf("polygon"),
)
GoldLowerBounds.init(listOf(cfg))
LowerBoundType.entries
.filter { it != LowerBoundType.UNKNOWN }
.forEach {
val bound = GoldLowerBounds.getBound(Chain.POLYGON__MAINNET, it)
assertThat(bound).isNull()
}
}
}

View File

@@ -13,6 +13,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.ZERO_ADDRESS
import io.emeraldpay.dshackle.upstream.polkadot.PolkadotLowerBoundService
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.Arguments
@@ -247,6 +248,12 @@ class RecursiveLowerBoundServiceTest {
private const val STATE_CHECKER_CALL_DATA = "0x1eaf190c"
private const val STATE_CHECKER_BYTECODE = "0x6080604052348015600e575f5ffd5b50600436106026575f3560e01c80631eaf190c14602a575b5f5ffd5b60306044565b604051603b91906078565b60405180910390f35b5f5f73ffffffffffffffffffffffffffffffffffffffff1631905090565b5f819050919050565b6072816062565b82525050565b5f60208201905060895f830184606b565b9291505056fea2646970667358221220251f5b4d2ed1abe77f66fde198a57ada08562dc3b0afbc6bac0261d1bf516b5d64736f6c634300081e0033"
@BeforeAll
@JvmStatic
fun init() {
GoldLowerBounds.init(emptyList())
}
@JvmStatic
fun detectorsFirstBlock(): List<Arguments> = listOf(
Arguments.of(