From a0dfb86c54d49285284d8615a985b8a0b6a12c46 Mon Sep 17 00:00:00 2001 From: KirillPamPam Date: Wed, 15 Mar 2023 13:38:07 +0400 Subject: [PATCH] Turn off caches for main, fix cache not enrichment blocks (#160) --- .../dshackle/cache/BlocksMemCache.kt | 3 ++ .../io/emeraldpay/dshackle/cache/Caches.kt | 23 ++++++++++++--- .../dshackle/cache/CachesFactory.kt | 1 + .../dshackle/cache/HeightByHashMemCache.kt | 2 +- .../dshackle/data/BlockContainer.kt | 1 + .../upstream/calls/EthereumCallSelector.kt | 4 +-- .../dshackle/cache/BlockByHeightSpec.groovy | 18 ++++++++---- .../dshackle/cache/BlocksMemCacheSpec.groovy | 28 +++++++++++++++++-- .../calls/EthereumCallSelectorSpec.groovy | 24 ++++++++-------- 9 files changed, 77 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt index bb1d403b..5d3671a8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt @@ -39,6 +39,9 @@ open class BlocksMemCache( } open fun add(block: BlockContainer) { + if (!block.enriched) { + return + } mapping.put(block.hash, block) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt index 536ffa6c..2fff5c47 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt @@ -39,7 +39,8 @@ open class Caches( private val redisBlocksByHash: BlocksRedisCache?, private val redisTxsByHash: TxRedisCache?, private val redisReceipts: ReceiptRedisCache?, - private val redisHeightByHashCache: HeightByHashRedisCache? + private val redisHeightByHashCache: HeightByHashRedisCache?, + private val cacheEnabled: Boolean ) { companion object { @@ -89,6 +90,9 @@ open class Caches( } open fun cacheReceipt(tag: Tag, data: DefaultContainer) { + if (!cacheEnabled) { + return + } val currentHeight = head?.getCurrentHeight() if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) { memReceipts.add(data).subscribe() @@ -98,6 +102,9 @@ open class Caches( } fun cache(tag: Tag, tx: TxContainer) { + if (!cacheEnabled) { + return + } // do not cache transactions that are not in a block yet if (tx.blockId == null) { return @@ -110,6 +117,9 @@ open class Caches( } fun cache(tag: Tag, block: BlockContainer) { + if (!cacheEnabled) { + return + } val job = ArrayList>() redisHeightByHashCache?.add(block)?.let(job::add) @@ -129,7 +139,6 @@ open class Caches( blockOnlyContainer = block } memoizeBlock(blockOnlyContainer) - memBlocksByHash.add(blockOnlyContainer) redisBlocksByHash?.add(blockOnlyContainer)?.let(job::add) // now cache only transactions @@ -198,7 +207,7 @@ open class Caches( return receiptByHash } - fun getLastHeightByHash(): Reader { + open fun getLastHeightByHash(): Reader { return memHeightByHash } @@ -227,6 +236,7 @@ open class Caches( private var redisTxsByHash: TxRedisCache? = null private var redisReceiptCache: ReceiptRedisCache? = null private var redisHeightByHashCache: HeightByHashRedisCache? = null + private var cacheEnabled: Boolean = true fun setBlockByHash(cache: BlocksMemCache): Builder { blocksByHash = cache @@ -268,6 +278,11 @@ open class Caches( return this } + fun setCacheEnabled(cacheEnabled: Boolean): Builder { + this.cacheEnabled = cacheEnabled + return this + } + fun build(): Caches { if (blocksByHash == null) { blocksByHash = BlocksMemCache() @@ -283,7 +298,7 @@ open class Caches( } return Caches( blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!, - redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache + redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache, cacheEnabled ) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt index 99e1fcd6..9e663a48 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt @@ -97,6 +97,7 @@ open class CachesFactory( caches.setReceipts(ReceiptRedisCache(redis.reactive(), chain)) caches.setHeightByHash(HeightByHashRedisCache(redis.reactive(), chain)) } + caches.setCacheEnabled(cacheConfig.requestsCacheEnabled) return caches.build() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt index 731e6f8e..12b6e0a2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt @@ -22,7 +22,7 @@ import io.emeraldpay.dshackle.reader.Reader import org.slf4j.LoggerFactory import reactor.core.publisher.Mono -class HeightByHashMemCache( +open class HeightByHashMemCache( maxSize: Int = 256 ) : Reader { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt index a5aa3fef..b0454760 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt @@ -39,6 +39,7 @@ class BlockContainer( val nodeRating: Int = 0, val upstreamId: String = "" ) : SourceContainer(json, parsed) { + val enriched: Boolean = transactions.isNotEmpty() companion object { @JvmStatic diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt index ba7ff0fa..146bcdcf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt @@ -164,10 +164,10 @@ class EthereumCallSelector( private fun blockByHashFromCache(blockHash: String): Mono { return try { - caches.getBlocksByHash() + caches.getLastHeightByHash() .read(BlockId.from(blockHash)) .onErrorResume { Mono.empty() } - .map { Selector.HeightMatcher(it.height) } + .map { Selector.HeightMatcher(it) } } catch (e: DecoderException) { log.warn("Invalid blockHash: $blockHash") Mono.empty() diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy index 202a1be8..d6d782ec 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy @@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.etherjar.domain.BlockHash +import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification @@ -39,12 +41,14 @@ class BlockByHeightSpec extends Specification { def heights = new HeightCache() def block = new BlockJson() + def tx = new TransactionJson() + tx.hash = TransactionId.from(hash1) block.number = 100 block.hash = BlockHash.from(hash1) block.totalDifficulty = BigInteger.ONE block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block.uncles = [] - block.transactions = [] + block.transactions = List.of(tx) BlockContainer.from(block).with { blocks.add(it) @@ -65,12 +69,14 @@ class BlockByHeightSpec extends Specification { def heights = new HeightCache() def block1 = new BlockJson() + def tx = new TransactionJson() + tx.hash = TransactionId.from(hash1) block1.number = 100 block1.hash = BlockHash.from(hash1) block1.totalDifficulty = BigInteger.ONE block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block1.uncles = [] - block1.transactions = [] + block1.transactions = List.of(tx) def block2 = new BlockJson() block2.number = 101 @@ -78,7 +84,7 @@ class BlockByHeightSpec extends Specification { block2.totalDifficulty = BigInteger.ONE block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block2.uncles = [] - block2.transactions = [] + block2.transactions = List.of(tx) BlockContainer.from(block1).with { @@ -109,12 +115,14 @@ class BlockByHeightSpec extends Specification { def heights = new HeightCache() def block1 = new BlockJson() + def tx = new TransactionJson() + tx.hash = TransactionId.from(hash1) block1.number = 100 block1.hash = BlockHash.from(hash1) block1.totalDifficulty = BigInteger.ONE block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block1.uncles = [] - block1.transactions = [] + block1.transactions = List.of(tx) def block2 = new BlockJson() block2.number = 100 @@ -122,7 +130,7 @@ class BlockByHeightSpec extends Specification { block2.totalDifficulty = BigInteger.ONE block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block2.uncles = [] - block2.transactions = [] + block2.transactions = List.of(tx) BlockContainer.from(block1).with { blocks.add(it) diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy index c1a989cf..a4b6e82b 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy @@ -21,7 +21,9 @@ import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.etherjar.domain.BlockHash +import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification @@ -39,12 +41,14 @@ class BlocksMemCacheSpec extends Specification { setup: def cache = new BlocksMemCache() def block = new BlockJson() + def tx = new TransactionJson() + tx.hash = TransactionId.from(hash1) block.number = 100 block.hash = BlockHash.from(hash1) block.totalDifficulty = BigInteger.ONE block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block.uncles = [] - block.transactions = [] + block.transactions = List.of(tx) when: cache.add(BlockContainer.from(block)) @@ -61,12 +65,14 @@ class BlocksMemCacheSpec extends Specification { when: [hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i -> def block = new BlockJson() + def tx = new TransactionJson() + tx.hash = TransactionId.from(hash1) block.number = 100 + i block.hash = BlockHash.from(hash) block.totalDifficulty = BigInteger.ONE block.timestamp = Instant.now() block.uncles = [] - block.transactions = [] + block.transactions = List.of(tx) cache.add(BlockContainer.from(block)) } @@ -83,4 +89,22 @@ class BlocksMemCacheSpec extends Specification { act1 == null } + def "Try add not full block and read"() { + setup: + def cache = new BlocksMemCache() + def block = new BlockJson() + block.number = 100 + block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block.uncles = [] + block.transactions = [] + + when: + cache.add(BlockContainer.from(block)) + def act = cache.read(BlockId.from(hash1)).block() + then: + act == null + } + } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelectorSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelectorSpec.groovy index 07277b83..c9c4cfb6 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelectorSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelectorSpec.groovy @@ -15,9 +15,9 @@ */ package io.emeraldpay.dshackle.upstream.calls -import io.emeraldpay.dshackle.cache.BlocksMemCache + import io.emeraldpay.dshackle.cache.Caches -import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.cache.HeightByHashMemCache import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Head @@ -27,7 +27,6 @@ import reactor.test.StepVerifier import spock.lang.Specification import java.time.Duration -import java.time.Instant class EthereumCallSelectorSpec extends Specification { @@ -233,14 +232,12 @@ class EthereumCallSelectorSpec extends Specification { def "Get height matcher for getByHash and getTransactionByBlockHash methods"() { setup: def hash = "0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32" - def block = new BlockContainer( - 12079192L, BlockId.from(hash), - BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0, "upstream" - ) - def blockByHashCache = Mock(BlocksMemCache) { - 1 * read(BlockId.from(hash)) >> Mono.just(block) + def blockHeight = 12079192L + def cache = Mock(Caches) { caches -> + 1 * caches.getLastHeightByHash() >> Mock(HeightByHashMemCache) { memCache -> + 1 * memCache.read(BlockId.from(hash)) >> Mono.just(blockHeight) + } } - def cache = Caches.newBuilder().setBlockByHash(blockByHashCache).build() def callSelector = new EthereumCallSelector(Stub(Reader), cache) def head = Stub(Head) @@ -318,10 +315,11 @@ class EthereumCallSelectorSpec extends Specification { def "No height matcher for getByHash method"() { setup: def hash = "0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32" - def blockByHashCache = Mock(BlocksMemCache) { - 1 * read(BlockId.from(hash)) >> resultFromCache + def cache = Mock(Caches) { caches -> + 1 * caches.getLastHeightByHash() >> Mock(HeightByHashMemCache) { memCache -> + 1 * memCache.read(BlockId.from(hash)) >> resultFromCache + } } - def cache = Caches.newBuilder().setBlockByHash(blockByHashCache).build() def callSelector = new EthereumCallSelector(Stub(Reader), cache) def head = Stub(Head)