Predict lower height and use lower height matcher (#552)

2024-08-22 19:52:57 +04:00
parent 32cb726aab
commit e184bb3547
30 changed files with 582 additions and 40 deletions
--- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/SelectorTest.kt
+++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/SelectorTest.kt
@@ -195,6 +195,7 @@ class SelectorTest {
            BlockchainOuterClass.Selector.newBuilder()
                .setLowerHeightSelector(
                    BlockchainOuterClass.LowerHeightSelector.newBuilder()
+                        .setHeight(100050003)
                        .setLowerBoundType(BlockchainOuterClass.LowerBoundType.LOWER_BOUND_BLOCK)
                        .build(),
                )
@@ -204,14 +205,45 @@ class SelectorTest {
        val upstreamFilter = Selector.convertToUpstreamFilter(requestSelectors)

        val actual = ups.sortedWith(upstreamFilter.sort.comparator)
+        val actualMatcher = Selector.MultiMatcher(listOf(Selector.LowerHeightMatcher(100050003, LowerBoundType.BLOCK)))

        assertEquals(
-            listOf(up2, up3, up1),
+            upstreamFilter.matcher,
+            actualMatcher,
+        )
+        assertEquals(
+            listOf(up1, up3, up2),
            actual,
        )
    }

+    @ParameterizedTest
+    @MethodSource("lowerHeightData")
+    fun `test lower height matcher`(
+        lowerHeight: Long,
+        predicted: Long,
+        expected: MatchesResponse,
+    ) {
+        val up = mock<Upstream> {
+            on { predictLowerBound(LowerBoundType.STATE) } doReturn predicted
+        }
+        val matcher = Selector.LowerHeightMatcher(lowerHeight, LowerBoundType.STATE)
+
+        val actualResponse = matcher.matchesWithCause(up)
+
+        assertEquals(expected, actualResponse)
+    }
+
    companion object {
+        @JvmStatic
+        fun lowerHeightData(): List<Arguments> =
+            listOf(
+                of(10000, 400, MatchesResponse.Success),
+                of(10000, 50000, MatchesResponse.LowerHeightResponse(10000, 50000, LowerBoundType.STATE)),
+                of(5000, 5000, MatchesResponse.Success),
+                of(3000, 0, MatchesResponse.LowerHeightResponse(3000, 0, LowerBoundType.STATE)),
+            )
+
        @JvmStatic
        fun data(): List<Arguments> =
            listOf(
--- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundServiceTest.kt
+++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundServiceTest.kt
@@ -0,0 +1,55 @@
+package io.emeraldpay.dshackle.upstream.lowerbound
+
+import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.upstream.Upstream
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.Test
+import org.mockito.kotlin.any
+import org.mockito.kotlin.doReturn
+import org.mockito.kotlin.mock
+import org.mockito.kotlin.never
+import org.mockito.kotlin.verify
+
+class LowerBoundServiceTest {
+
+    @Test
+    fun `predict lower bound`() {
+        val detector = mock<LowerBoundDetector> {
+            on { predictLowerBound(LowerBoundType.STATE) } doReturn 4000
+            on { types() } doReturn setOf(LowerBoundType.STATE)
+        }
+        val boundService = LowerBoundServiceMock(mock<Upstream>(), listOf(detector))
+
+        val bound = boundService.predictLowerBound(LowerBoundType.STATE)
+
+        verify(detector).types()
+        verify(detector).predictLowerBound(LowerBoundType.STATE)
+
+        assertThat(bound).isEqualTo(4000)
+    }
+
+    @Test
+    fun `the predicted lower bound is 0 if there is no such bound type`() {
+        val detector = mock<LowerBoundDetector> {
+            on { types() } doReturn setOf(LowerBoundType.STATE)
+        }
+        val boundService = LowerBoundServiceMock(mock<Upstream>(), listOf(detector))
+
+        val bound = boundService.predictLowerBound(LowerBoundType.BLOCK)
+
+        verify(detector).types()
+        verify(detector, never()).predictLowerBound(any())
+
+        assertThat(bound).isEqualTo(0)
+    }
+
+    private class LowerBoundServiceMock(
+        upstream: Upstream,
+        private val detectors: List<LowerBoundDetector>,
+    ) : LowerBoundService(Chain.ETHEREUM__MAINNET, upstream) {
+
+        override fun detectors(): List<LowerBoundDetector> {
+            return detectors
+        }
+    }
+}
--- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundsPredictionTest.kt
+++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundsPredictionTest.kt
@@ -0,0 +1,227 @@
+package io.emeraldpay.dshackle.upstream.lowerbound
+
+import io.emeraldpay.dshackle.Chain
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.Test
+import java.time.Instant
+import java.time.temporal.ChronoUnit
+
+class LowerBoundsPredictionTest {
+
+    @Test
+    fun `first archival lower bound data, get it and predict the next bound`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val newLowerBound = LowerBoundData(1L, 1000, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(newLowerBound)
+
+        val lastBound = lowerBounds.getLastBound(LowerBoundType.STATE)
+        val predictedNextBound = lowerBounds.predictNextBound(LowerBoundType.STATE)
+        val allBounds = lowerBounds.getAllBounds(LowerBoundType.STATE)
+
+        assertThat(lastBound).isEqualTo(newLowerBound)
+        assertThat(predictedNextBound).isEqualTo(1)
+        assertThat(allBounds).isEqualTo(listOf(newLowerBound))
+    }
+
+    @Test
+    fun `if no bound the default values`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+
+        val lastBound = lowerBounds.getLastBound(LowerBoundType.STATE)
+        val predictedNextBound = lowerBounds.predictNextBound(LowerBoundType.STATE)
+        val allBounds = lowerBounds.getAllBounds(LowerBoundType.STATE)
+
+        assertThat(lastBound).isNull()
+        assertThat(predictedNextBound).isEqualTo(0)
+        assertThat(allBounds).isEmpty()
+    }
+
+    @Test
+    fun `sequential archival lower bound data and get only the last`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val newLowerBound = LowerBoundData(1L, 1000, LowerBoundType.STATE)
+        val nextNewLowerBound = LowerBoundData(1L, 1005, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(newLowerBound)
+
+        val lastBound = lowerBounds.getLastBound(LowerBoundType.STATE)
+        val predictedNextBound = lowerBounds.predictNextBound(LowerBoundType.STATE)
+        val allBounds = lowerBounds.getAllBounds(LowerBoundType.STATE)
+
+        assertThat(lastBound).isEqualTo(newLowerBound)
+        assertThat(predictedNextBound).isEqualTo(1)
+        assertThat(allBounds).isEqualTo(listOf(newLowerBound))
+
+        lowerBounds.updateBound(nextNewLowerBound)
+
+        val newLastBound = lowerBounds.getLastBound(LowerBoundType.STATE)
+        val newPredictedNextBound = lowerBounds.predictNextBound(LowerBoundType.STATE)
+        val newAllBounds = lowerBounds.getAllBounds(LowerBoundType.STATE)
+
+        assertThat(newLastBound).isEqualTo(nextNewLowerBound)
+        assertThat(newPredictedNextBound).isEqualTo(1)
+        assertThat(newAllBounds).isEqualTo(listOf(nextNewLowerBound))
+    }
+
+    @Test
+    fun `don't update the lower bounds if the same timestamp`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val newLowerBound = LowerBoundData(1L, 1000, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(newLowerBound)
+        lowerBounds.updateBound(LowerBoundData(100000L, 1000, LowerBoundType.STATE))
+
+        val lastBound = lowerBounds.getLastBound(LowerBoundType.STATE)
+        val predictedNextBound = lowerBounds.predictNextBound(LowerBoundType.STATE)
+        val allBounds = lowerBounds.getAllBounds(LowerBoundType.STATE)
+
+        assertThat(lastBound).isEqualTo(newLowerBound)
+        assertThat(predictedNextBound).isEqualTo(1)
+        assertThat(allBounds).isEqualTo(listOf(newLowerBound))
+    }
+
+    @Test
+    fun `always get the last bound`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val lowerBound1 = LowerBoundData(1000L, 1000, LowerBoundType.STATE)
+        val lowerBound2 = LowerBoundData(1005L, 1005, LowerBoundType.STATE)
+        val lowerBound3 = LowerBoundData(1010L, 1010, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(lowerBound1)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound1)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound1))
+
+        lowerBounds.updateBound(lowerBound2)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound2)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound1, lowerBound2))
+
+        lowerBounds.updateBound(lowerBound3)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound3)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound1, lowerBound2, lowerBound3))
+    }
+
+    @Test
+    fun `preserve the maximum number of bounds`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val lowerBound1 = LowerBoundData(1000L, 1000, LowerBoundType.STATE)
+        val lowerBound2 = LowerBoundData(1005L, 1005, LowerBoundType.STATE)
+        val lowerBound3 = LowerBoundData(1010L, 1010, LowerBoundType.STATE)
+        val lowerBound4 = LowerBoundData(1050L, 1050, LowerBoundType.STATE)
+        val lowerBound5 = LowerBoundData(1060L, 1060, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(lowerBound1)
+        lowerBounds.updateBound(lowerBound2)
+        lowerBounds.updateBound(lowerBound3)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound3)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound1, lowerBound2, lowerBound3))
+
+        lowerBounds.updateBound(lowerBound4)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound4)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound2, lowerBound3, lowerBound4))
+
+        lowerBounds.updateBound(lowerBound5)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound5)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound3, lowerBound4, lowerBound5))
+    }
+
+    @Test
+    fun `if get the archival bound then remove previous ones`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val lowerBound1 = LowerBoundData(1000L, 1000, LowerBoundType.STATE)
+        val lowerBound2 = LowerBoundData(1005L, 1005, LowerBoundType.STATE)
+        val lowerBound3 = LowerBoundData(1, 1010, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(lowerBound1)
+        lowerBounds.updateBound(lowerBound2)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound2)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound1, lowerBound2))
+
+        lowerBounds.updateBound(lowerBound3)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound3)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound3))
+    }
+
+    @Test
+    fun `predict the same bound if all bounds are equal to each other`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val lowerBound1 = LowerBoundData(15060L, 1000, LowerBoundType.STATE)
+        val lowerBound2 = LowerBoundData(15060L, 2000, LowerBoundType.STATE)
+        val lowerBound3 = LowerBoundData(15060L, 3000, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(lowerBound1)
+        lowerBounds.updateBound(lowerBound2)
+        lowerBounds.updateBound(lowerBound3)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBound3)
+        assertThat(lowerBounds.predictNextBound(LowerBoundType.STATE)).isEqualTo(15060L)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBound1, lowerBound2, lowerBound3))
+    }
+
+    @Test
+    fun `predict the next bound based on different bounds`() {
+        val now = Instant.now()
+        val lowerBounds = LowerBounds(Chain.BSC__MAINNET)
+        val lowerBound1 = LowerBoundData(37995846, now.minus(9, ChronoUnit.MINUTES).epochSecond, LowerBoundType.STATE)
+        val lowerBound2 = LowerBoundData(37995906, now.minus(6, ChronoUnit.MINUTES).epochSecond, LowerBoundType.STATE)
+        val lowerBound3 = LowerBoundData(37995966, now.minus(3, ChronoUnit.MINUTES).epochSecond, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(lowerBound1)
+        lowerBounds.updateBound(lowerBound2)
+        lowerBounds.updateBound(lowerBound3)
+
+        val predicted = lowerBounds.predictNextBound(LowerBoundType.STATE)
+
+        assertThat(predicted)
+            .isLessThan(37996030)
+            .isGreaterThan(37996020)
+    }
+
+    @Test
+    fun `predict the next bound based on average speed`() {
+        val now = Instant.now()
+        val lowerBounds = LowerBounds(Chain.BSC__MAINNET)
+        val lowerBound1 = LowerBoundData(37995966, now.minus(3, ChronoUnit.MINUTES).epochSecond, LowerBoundType.STATE)
+
+        lowerBounds.updateBound(lowerBound1)
+
+        val predicted = lowerBounds.predictNextBound(LowerBoundType.STATE)
+        println(predicted)
+
+        assertThat(predicted)
+            .isLessThan(37996030)
+            .isGreaterThan(37996020)
+    }
+
+    @Test
+    fun `update different bounds`() {
+        val lowerBounds = LowerBounds(Chain.ETHEREUM__MAINNET)
+        val lowerBoundState1 = LowerBoundData(15060L, 1010, LowerBoundType.STATE)
+        val lowerBoundState2 = LowerBoundData(16060L, 1020, LowerBoundType.STATE)
+        val lowerBoundState3 = LowerBoundData(17060L, 1030, LowerBoundType.STATE)
+        val lowerBoundBlock1 = LowerBoundData(20000, 1010, LowerBoundType.BLOCK)
+        val lowerBoundBlock2 = LowerBoundData(21000, 1020, LowerBoundType.BLOCK)
+        val lowerBoundBlock3 = LowerBoundData(22000, 1030, LowerBoundType.BLOCK)
+
+        lowerBounds.updateBound(lowerBoundState1)
+        lowerBounds.updateBound(lowerBoundState2)
+        lowerBounds.updateBound(lowerBoundState3)
+        lowerBounds.updateBound(lowerBoundBlock1)
+        lowerBounds.updateBound(lowerBoundBlock2)
+        lowerBounds.updateBound(lowerBoundBlock3)
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.STATE)).isEqualTo(lowerBoundState3)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.STATE)).isEqualTo(listOf(lowerBoundState1, lowerBoundState2, lowerBoundState3))
+
+        assertThat(lowerBounds.getLastBound(LowerBoundType.BLOCK)).isEqualTo(lowerBoundBlock3)
+        assertThat(lowerBounds.getAllBounds(LowerBoundType.BLOCK)).isEqualTo(listOf(lowerBoundBlock1, lowerBoundBlock2, lowerBoundBlock3))
+    }
+}
--- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/RecursiveLowerBoundServiceTest.kt
+++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/RecursiveLowerBoundServiceTest.kt
@@ -1,15 +1,15 @@
-package io.emeraldpay.dshackle.upstream
+package io.emeraldpay.dshackle.upstream.lowerbound

 import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.reader.ChainReader
+import io.emeraldpay.dshackle.upstream.ChainRequest
+import io.emeraldpay.dshackle.upstream.ChainResponse
+import io.emeraldpay.dshackle.upstream.Head
+import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumLowerBoundService
 import io.emeraldpay.dshackle.upstream.ethereum.EthereumLowerBoundTxDetector.Companion.MAX_OFFSET
 import io.emeraldpay.dshackle.upstream.ethereum.ZERO_ADDRESS
-import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
-import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService
-import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
-import io.emeraldpay.dshackle.upstream.lowerbound.toHex
 import io.emeraldpay.dshackle.upstream.polkadot.PolkadotLowerBoundService
 import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
 import org.assertj.core.api.Assertions.assertThat
@@ -77,6 +77,7 @@ class RecursiveLowerBoundServiceTest {
            on { getId() } doReturn "id"
            on { getHead() } doReturn head
            on { getIngressReader() } doReturn reader
+            on { getChain() } doReturn Chain.UNSPECIFIED
        }

        val detector = EthereumLowerBoundService(Chain.UNSPECIFIED, upstream)
@@ -130,6 +131,7 @@ class RecursiveLowerBoundServiceTest {
        val upstream = mock<Upstream> {
            on { getHead() } doReturn head
            on { getIngressReader() } doReturn reader
+            on { getChain() } doReturn Chain.STARKNET__MAINNET
        }

        val detector = PolkadotLowerBoundService(Chain.UNSPECIFIED, upstream)
@@ -162,6 +164,7 @@ class RecursiveLowerBoundServiceTest {
        val upstream = mock<Upstream> {
            on { getHead() } doReturn head
            on { getIngressReader() } doReturn reader
+            on { getChain() } doReturn Chain.UNSPECIFIED
        }

        val detector = detectorClass.getConstructor(Chain::class.java, Upstream::class.java).newInstance(Chain.UNSPECIFIED, upstream)
--- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundServiceTest.kt
+++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundServiceTest.kt
@@ -54,6 +54,7 @@ class SolanaLowerBoundServiceTest {
        }
        val upstream = mock<Upstream> {
            on { getIngressReader() } doReturn reader
+            on { getChain() } doReturn Chain.UNSPECIFIED
        }

        val detector = SolanaLowerBoundService(Chain.UNSPECIFIED, upstream)
--- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetLowerBoundStateDetectorTest.kt
+++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetLowerBoundStateDetectorTest.kt
@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.starknet

+import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
 import org.junit.jupiter.api.Test
@@ -10,7 +11,7 @@ class StarknetLowerBoundStateDetectorTest {

    @Test
    fun `starknet lower block is 1`() {
-        val detector = StarknetLowerBoundStateDetector()
+        val detector = StarknetLowerBoundStateDetector(Chain.STARKNET__MAINNET)

        StepVerifier.withVirtualTime { detector.detectLowerBound() }
            .expectSubscription()