StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Turn off caches for main, fix cache not enrichment blocks (#160)

Browse Source
This commit is contained in:
KirillPamPam
2023-03-15 13:38:07 +04:00
committed by GitHub
parent 26d518d98e
commit a0dfb86c54
9 changed files with 77 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt vendored
View File

@@ -39,6 +39,9 @@ open class BlocksMemCache(
}
open fun add(block: BlockContainer) {
if (!block.enriched) {
return
}
mapping.put(block.hash, block)
}

23
src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt vendored
View File

@@ -39,7 +39,8 @@ open class Caches(
private val redisBlocksByHash: BlocksRedisCache?,
private val redisTxsByHash: TxRedisCache?,
private val redisReceipts: ReceiptRedisCache?,
private val redisHeightByHashCache: HeightByHashRedisCache?
private val redisHeightByHashCache: HeightByHashRedisCache?,
private val cacheEnabled: Boolean
) {
companion object {
@@ -89,6 +90,9 @@ open class Caches(
}
open fun cacheReceipt(tag: Tag, data: DefaultContainer<TransactionReceiptJson>) {
if (!cacheEnabled) {
return
}
val currentHeight = head?.getCurrentHeight()
if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) {
memReceipts.add(data).subscribe()
@@ -98,6 +102,9 @@ open class Caches(
}
fun cache(tag: Tag, tx: TxContainer) {
if (!cacheEnabled) {
return
}
// do not cache transactions that are not in a block yet
if (tx.blockId == null) {
return
@@ -110,6 +117,9 @@ open class Caches(
}
fun cache(tag: Tag, block: BlockContainer) {
if (!cacheEnabled) {
return
}
val job = ArrayList<Mono<Void>>()
redisHeightByHashCache?.add(block)?.let(job::add)
@@ -129,7 +139,6 @@ open class Caches(
blockOnlyContainer = block
}
memoizeBlock(blockOnlyContainer)
memBlocksByHash.add(blockOnlyContainer)
redisBlocksByHash?.add(blockOnlyContainer)?.let(job::add)
// now cache only transactions
@@ -198,7 +207,7 @@ open class Caches(
return receiptByHash
}
fun getLastHeightByHash(): Reader<BlockId, Long> {
open fun getLastHeightByHash(): Reader<BlockId, Long> {
return memHeightByHash
}
@@ -227,6 +236,7 @@ open class Caches(
private var redisTxsByHash: TxRedisCache? = null
private var redisReceiptCache: ReceiptRedisCache? = null
private var redisHeightByHashCache: HeightByHashRedisCache? = null
private var cacheEnabled: Boolean = true
fun setBlockByHash(cache: BlocksMemCache): Builder {
blocksByHash = cache
@@ -268,6 +278,11 @@ open class Caches(
return this
}
fun setCacheEnabled(cacheEnabled: Boolean): Builder {
this.cacheEnabled = cacheEnabled
return this
}
fun build(): Caches {
if (blocksByHash == null) {
blocksByHash = BlocksMemCache()
@@ -283,7 +298,7 @@ open class Caches(
}
return Caches(
blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!,
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache
redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache, cacheEnabled
)
}
}

1
src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt vendored
View File

@@ -97,6 +97,7 @@ open class CachesFactory(
caches.setReceipts(ReceiptRedisCache(redis.reactive(), chain))
caches.setHeightByHash(HeightByHashRedisCache(redis.reactive(), chain))
}
caches.setCacheEnabled(cacheConfig.requestsCacheEnabled)
return caches.build()
}

2
src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt vendored
View File

@@ -22,7 +22,7 @@ import io.emeraldpay.dshackle.reader.Reader
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
class HeightByHashMemCache(
open class HeightByHashMemCache(
maxSize: Int = 256
) : Reader<BlockId, Long> {

1
src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt
View File

@@ -39,6 +39,7 @@ class BlockContainer(
val nodeRating: Int = 0,
val upstreamId: String = ""
) : SourceContainer(json, parsed) {
val enriched: Boolean = transactions.isNotEmpty()
companion object {
@JvmStatic

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt
View File

@@ -164,10 +164,10 @@ class EthereumCallSelector(
private fun blockByHashFromCache(blockHash: String): Mono<Selector.Matcher> {
return try {
caches.getBlocksByHash()
caches.getLastHeightByHash()
.read(BlockId.from(blockHash))
.onErrorResume { Mono.empty() }
.map { Selector.HeightMatcher(it.height) }
.map { Selector.HeightMatcher(it) }
} catch (e: DecoderException) {
log.warn("Invalid blockHash: $blockHash")
Mono.empty()

18
src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy vendored
View File

@@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification
@@ -39,12 +41,14 @@ class BlockByHeightSpec extends Specification {
def heights = new HeightCache()
def block = new BlockJson<TransactionRefJson>()
def tx = new TransactionJson()
tx.hash = TransactionId.from(hash1)
block.number = 100
block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.uncles = []
block.transactions = []
block.transactions = List.of(tx)
BlockContainer.from(block).with {
blocks.add(it)
@@ -65,12 +69,14 @@ class BlockByHeightSpec extends Specification {
def heights = new HeightCache()
def block1 = new BlockJson<TransactionRefJson>()
def tx = new TransactionJson()
tx.hash = TransactionId.from(hash1)
block1.number = 100
block1.hash = BlockHash.from(hash1)
block1.totalDifficulty = BigInteger.ONE
block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block1.uncles = []
block1.transactions = []
block1.transactions = List.of(tx)
def block2 = new BlockJson<TransactionRefJson>()
block2.number = 101
@@ -78,7 +84,7 @@ class BlockByHeightSpec extends Specification {
block2.totalDifficulty = BigInteger.ONE
block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block2.uncles = []
block2.transactions = []
block2.transactions = List.of(tx)
BlockContainer.from(block1).with {
@@ -109,12 +115,14 @@ class BlockByHeightSpec extends Specification {
def heights = new HeightCache()
def block1 = new BlockJson<TransactionRefJson>()
def tx = new TransactionJson()
tx.hash = TransactionId.from(hash1)
block1.number = 100
block1.hash = BlockHash.from(hash1)
block1.totalDifficulty = BigInteger.ONE
block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block1.uncles = []
block1.transactions = []
block1.transactions = List.of(tx)
def block2 = new BlockJson<TransactionRefJson>()
block2.number = 100
@@ -122,7 +130,7 @@ class BlockByHeightSpec extends Specification {
block2.totalDifficulty = BigInteger.ONE
block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block2.uncles = []
block2.transactions = []
block2.transactions = List.of(tx)
BlockContainer.from(block1).with {
blocks.add(it)

28
src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy vendored
View File

@@ -21,7 +21,9 @@ import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification
@@ -39,12 +41,14 @@ class BlocksMemCacheSpec extends Specification {
setup:
def cache = new BlocksMemCache()
def block = new BlockJson<TransactionRefJson>()
def tx = new TransactionJson()
tx.hash = TransactionId.from(hash1)
block.number = 100
block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.uncles = []
block.transactions = []
block.transactions = List.of(tx)
when:
cache.add(BlockContainer.from(block))
@@ -61,12 +65,14 @@ class BlocksMemCacheSpec extends Specification {
when:
[hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i ->
def block = new BlockJson<TransactionRefJson>()
def tx = new TransactionJson()
tx.hash = TransactionId.from(hash1)
block.number = 100 + i
block.hash = BlockHash.from(hash)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
block.uncles = []
block.transactions = []
block.transactions = List.of(tx)
cache.add(BlockContainer.from(block))
}
@@ -83,4 +89,22 @@ class BlocksMemCacheSpec extends Specification {
act1 == null
}
def "Try add not full block and read"() {
setup:
def cache = new BlocksMemCache()
def block = new BlockJson<TransactionRefJson>()
block.number = 100
block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.uncles = []
block.transactions = []
when:
cache.add(BlockContainer.from(block))
def act = cache.read(BlockId.from(hash1)).block()
then:
act == null
}
}

24
src/test/groovy/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelectorSpec.groovy
View File

@@ -15,9 +15,9 @@
*/
package io.emeraldpay.dshackle.upstream.calls
import io.emeraldpay.dshackle.cache.BlocksMemCache
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.cache.HeightByHashMemCache
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
@@ -27,7 +27,6 @@ import reactor.test.StepVerifier
import spock.lang.Specification
import java.time.Duration
import java.time.Instant
class EthereumCallSelectorSpec extends Specification {
@@ -233,14 +232,12 @@ class EthereumCallSelectorSpec extends Specification {
def "Get height matcher for getByHash and getTransactionByBlockHash methods"() {
setup:
def hash = "0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"
def block = new BlockContainer(
12079192L, BlockId.from(hash),
BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0, "upstream"
)
def blockByHashCache = Mock(BlocksMemCache) {
1 * read(BlockId.from(hash)) >> Mono.just(block)
def blockHeight = 12079192L
def cache = Mock(Caches) { caches ->
1 * caches.getLastHeightByHash() >> Mock(HeightByHashMemCache) { memCache ->
1 * memCache.read(BlockId.from(hash)) >> Mono.just(blockHeight)
}
}
def cache = Caches.newBuilder().setBlockByHash(blockByHashCache).build()
def callSelector = new EthereumCallSelector(Stub(Reader), cache)
def head = Stub(Head)
@@ -318,10 +315,11 @@ class EthereumCallSelectorSpec extends Specification {
def "No height matcher for getByHash method"() {
setup:
def hash = "0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"
def blockByHashCache = Mock(BlocksMemCache) {
1 * read(BlockId.from(hash)) >> resultFromCache
def cache = Mock(Caches) { caches ->
1 * caches.getLastHeightByHash() >> Mock(HeightByHashMemCache) { memCache ->
1 * memCache.read(BlockId.from(hash)) >> resultFromCache
}
}
def cache = Caches.newBuilder().setBlockByHash(blockByHashCache).build()
def callSelector = new EthereumCallSelector(Stub(Reader), cache)
def head = Stub(Head)