Turn off caches for main, fix cache not enrichment blocks (#160)

src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt

@@ -39,6 +39,9 @@ open class BlocksMemCache(
     }
 
     open fun add(block: BlockContainer) {
+        if (!block.enriched) {
+            return
+        }
         mapping.put(block.hash, block)
     }
 

src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt

@@ -39,7 +39,8 @@ open class Caches(
     private val redisBlocksByHash: BlocksRedisCache?,
     private val redisTxsByHash: TxRedisCache?,
     private val redisReceipts: ReceiptRedisCache?,
-    private val redisHeightByHashCache: HeightByHashRedisCache?
+    private val redisHeightByHashCache: HeightByHashRedisCache?,
+    private val cacheEnabled: Boolean
 ) {
 
     companion object {
@@ -89,6 +90,9 @@ open class Caches(
     }
 
     open fun cacheReceipt(tag: Tag, data: DefaultContainer<TransactionReceiptJson>) {
+        if (!cacheEnabled) {
+            return
+        }
         val currentHeight = head?.getCurrentHeight()
         if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) {
             memReceipts.add(data).subscribe()
@@ -98,6 +102,9 @@ open class Caches(
     }
 
     fun cache(tag: Tag, tx: TxContainer) {
+        if (!cacheEnabled) {
+            return
+        }
         // do not cache transactions that are not in a block yet
         if (tx.blockId == null) {
             return
@@ -110,6 +117,9 @@ open class Caches(
     }
 
     fun cache(tag: Tag, block: BlockContainer) {
+        if (!cacheEnabled) {
+            return
+        }
         val job = ArrayList<Mono<Void>>()
 
         redisHeightByHashCache?.add(block)?.let(job::add)
@@ -129,7 +139,6 @@ open class Caches(
                 blockOnlyContainer = block
             }
             memoizeBlock(blockOnlyContainer)
-            memBlocksByHash.add(blockOnlyContainer)
             redisBlocksByHash?.add(blockOnlyContainer)?.let(job::add)
 
             // now cache only transactions
@@ -198,7 +207,7 @@ open class Caches(
         return receiptByHash
     }
 
-    fun getLastHeightByHash(): Reader<BlockId, Long> {
+    open fun getLastHeightByHash(): Reader<BlockId, Long> {
         return memHeightByHash
     }
 
@@ -227,6 +236,7 @@ open class Caches(
         private var redisTxsByHash: TxRedisCache? = null
         private var redisReceiptCache: ReceiptRedisCache? = null
         private var redisHeightByHashCache: HeightByHashRedisCache? = null
+        private var cacheEnabled: Boolean = true
 
         fun setBlockByHash(cache: BlocksMemCache): Builder {
             blocksByHash = cache
@@ -268,6 +278,11 @@ open class Caches(
             return this
         }
 
+        fun setCacheEnabled(cacheEnabled: Boolean): Builder {
+            this.cacheEnabled = cacheEnabled
+            return this
+        }
+
         fun build(): Caches {
             if (blocksByHash == null) {
                 blocksByHash = BlocksMemCache()
@@ -283,7 +298,7 @@ open class Caches(
             }
             return Caches(
                 blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
-                redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache
+                redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache, cacheEnabled
             )
         }
     }

src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt

@@ -97,6 +97,7 @@ open class CachesFactory(
             caches.setReceipts(ReceiptRedisCache(redis.reactive(), chain))
             caches.setHeightByHash(HeightByHashRedisCache(redis.reactive(), chain))
         }
+        caches.setCacheEnabled(cacheConfig.requestsCacheEnabled)
         return caches.build()
     }
 

src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt

@@ -22,7 +22,7 @@ import io.emeraldpay.dshackle.reader.Reader
 import org.slf4j.LoggerFactory
 import reactor.core.publisher.Mono
 
-class HeightByHashMemCache(
+open class HeightByHashMemCache(
     maxSize: Int = 256
 ) : Reader<BlockId, Long> {
 

src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt

@@ -39,6 +39,7 @@ class BlockContainer(
     val nodeRating: Int = 0,
     val upstreamId: String = ""
 ) : SourceContainer(json, parsed) {
+    val enriched: Boolean = transactions.isNotEmpty()
 
     companion object {
         @JvmStatic

src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt

@@ -164,10 +164,10 @@ class EthereumCallSelector(
 
     private fun blockByHashFromCache(blockHash: String): Mono<Selector.Matcher> {
         return try {
-            caches.getBlocksByHash()
+            caches.getLastHeightByHash()
                 .read(BlockId.from(blockHash))
                 .onErrorResume { Mono.empty() }
-                .map { Selector.HeightMatcher(it.height) }
+                .map { Selector.HeightMatcher(it) }
         } catch (e: DecoderException) {
             log.warn("Invalid blockHash: $blockHash")
             Mono.empty()

src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy

@@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.data.BlockContainer
 import io.emeraldpay.etherjar.domain.BlockHash
+import io.emeraldpay.etherjar.domain.TransactionId
 import io.emeraldpay.etherjar.rpc.json.BlockJson
+import io.emeraldpay.etherjar.rpc.json.TransactionJson
 import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
 import spock.lang.Specification
 
@@ -39,12 +41,14 @@ class BlockByHeightSpec extends Specification {
         def heights = new HeightCache()
 
         def block = new BlockJson<TransactionRefJson>()
+        def tx = new TransactionJson()
+        tx.hash = TransactionId.from(hash1)
         block.number = 100
         block.hash = BlockHash.from(hash1)
         block.totalDifficulty = BigInteger.ONE
         block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
         block.uncles = []
-        block.transactions = []
+        block.transactions = List.of(tx)
 
         BlockContainer.from(block).with {
             blocks.add(it)
@@ -65,12 +69,14 @@ class BlockByHeightSpec extends Specification {
         def heights = new HeightCache()
 
         def block1 = new BlockJson<TransactionRefJson>()
+        def tx = new TransactionJson()
+        tx.hash = TransactionId.from(hash1)
         block1.number = 100
         block1.hash = BlockHash.from(hash1)
         block1.totalDifficulty = BigInteger.ONE
         block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
         block1.uncles = []
-        block1.transactions = []
+        block1.transactions = List.of(tx)
 
         def block2 = new BlockJson<TransactionRefJson>()
         block2.number = 101
@@ -78,7 +84,7 @@ class BlockByHeightSpec extends Specification {
         block2.totalDifficulty = BigInteger.ONE
         block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
         block2.uncles = []
-        block2.transactions = []
+        block2.transactions = List.of(tx)
 
 
         BlockContainer.from(block1).with {
@@ -109,12 +115,14 @@ class BlockByHeightSpec extends Specification {
         def heights = new HeightCache()
 
         def block1 = new BlockJson<TransactionRefJson>()
+        def tx = new TransactionJson()
+        tx.hash = TransactionId.from(hash1)
         block1.number = 100
         block1.hash = BlockHash.from(hash1)
         block1.totalDifficulty = BigInteger.ONE
         block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
         block1.uncles = []
-        block1.transactions = []
+        block1.transactions = List.of(tx)
 
         def block2 = new BlockJson<TransactionRefJson>()
         block2.number = 100
@@ -122,7 +130,7 @@ class BlockByHeightSpec extends Specification {
         block2.totalDifficulty = BigInteger.ONE
         block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
         block2.uncles = []
-        block2.transactions = []
+        block2.transactions = List.of(tx)
 
         BlockContainer.from(block1).with {
             blocks.add(it)

src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy

@@ -21,7 +21,9 @@ import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.data.BlockContainer
 import io.emeraldpay.dshackle.data.BlockId
 import io.emeraldpay.etherjar.domain.BlockHash
+import io.emeraldpay.etherjar.domain.TransactionId
 import io.emeraldpay.etherjar.rpc.json.BlockJson
+import io.emeraldpay.etherjar.rpc.json.TransactionJson
 import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
 import spock.lang.Specification
 
@@ -39,12 +41,14 @@ class BlocksMemCacheSpec extends Specification {
         setup:
         def cache = new BlocksMemCache()
         def block = new BlockJson<TransactionRefJson>()
+        def tx = new TransactionJson()
+        tx.hash = TransactionId.from(hash1)
         block.number = 100
         block.hash = BlockHash.from(hash1)
         block.totalDifficulty = BigInteger.ONE
         block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
         block.uncles = []
-        block.transactions = []
+        block.transactions = List.of(tx)
 
         when:
         cache.add(BlockContainer.from(block))
@@ -61,12 +65,14 @@ class BlocksMemCacheSpec extends Specification {
         when:
         [hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i ->
             def block = new BlockJson<TransactionRefJson>()
+            def tx = new TransactionJson()
+            tx.hash = TransactionId.from(hash1)
             block.number = 100 + i
             block.hash = BlockHash.from(hash)
             block.totalDifficulty = BigInteger.ONE
             block.timestamp = Instant.now()
             block.uncles = []
-            block.transactions = []
+            block.transactions = List.of(tx)
 
             cache.add(BlockContainer.from(block))
         }
@@ -83,4 +89,22 @@ class BlocksMemCacheSpec extends Specification {
         act1 == null
     }
 
+    def "Try add not full block and read"() {
+        setup:
+        def cache = new BlocksMemCache()
+        def block = new BlockJson<TransactionRefJson>()
+        block.number = 100
+        block.hash = BlockHash.from(hash1)
+        block.totalDifficulty = BigInteger.ONE
+        block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
+        block.uncles = []
+        block.transactions = []
+
+        when:
+        cache.add(BlockContainer.from(block))
+        def act = cache.read(BlockId.from(hash1)).block()
+        then:
+        act == null
+    }
+
 }

src/test/groovy/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelectorSpec.groovy

@@ -15,9 +15,9 @@
  */
 package io.emeraldpay.dshackle.upstream.calls
 
-import io.emeraldpay.dshackle.cache.BlocksMemCache
+
 import io.emeraldpay.dshackle.cache.Caches
-import io.emeraldpay.dshackle.data.BlockContainer
+import io.emeraldpay.dshackle.cache.HeightByHashMemCache
 import io.emeraldpay.dshackle.data.BlockId
 import io.emeraldpay.dshackle.reader.Reader
 import io.emeraldpay.dshackle.upstream.Head
@@ -27,7 +27,6 @@ import reactor.test.StepVerifier
 import spock.lang.Specification
 
 import java.time.Duration
-import java.time.Instant
 
 class EthereumCallSelectorSpec extends Specification {
 
@@ -233,14 +232,12 @@ class EthereumCallSelectorSpec extends Specification {
     def "Get height matcher for getByHash and getTransactionByBlockHash methods"() {
         setup:
         def hash = "0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"
-        def block = new BlockContainer(
+        def blockHeight = 12079192L
-                12079192L, BlockId.from(hash),
+        def cache = Mock(Caches) { caches ->
-                BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0, "upstream"
+            1 * caches.getLastHeightByHash() >> Mock(HeightByHashMemCache) { memCache ->
-        )
+                1 * memCache.read(BlockId.from(hash)) >> Mono.just(blockHeight)
-        def blockByHashCache = Mock(BlocksMemCache) {
+            }
-            1 * read(BlockId.from(hash)) >> Mono.just(block)
         }
-        def cache = Caches.newBuilder().setBlockByHash(blockByHashCache).build()
         def callSelector = new EthereumCallSelector(Stub(Reader), cache)
         def head = Stub(Head)
 
@@ -318,10 +315,11 @@ class EthereumCallSelectorSpec extends Specification {
     def "No height matcher for getByHash method"() {
         setup:
         def hash = "0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"
-        def blockByHashCache = Mock(BlocksMemCache) {
+        def cache = Mock(Caches) { caches ->
-            1 * read(BlockId.from(hash)) >> resultFromCache
+            1 * caches.getLastHeightByHash() >> Mock(HeightByHashMemCache) { memCache ->
+                1 * memCache.read(BlockId.from(hash)) >> resultFromCache
+            }
         }
-        def cache = Caches.newBuilder().setBlockByHash(blockByHashCache).build()
         def callSelector = new EthereumCallSelector(Stub(Reader), cache)
         def head = Stub(Head)