StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solution: cache requested transactions

Browse Source
This commit is contained in:
Igor Artamonov
2020-02-07 23:51:28 -05:00
parent d13397260f
commit d551d95a3f
25 changed files with 775 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt vendored
View File

@@ -10,7 +10,7 @@ import reactor.core.publisher.Mono
/**
* Connects two caches to read through them. First is cache height->hash, second is hash->block.
*/
class BlockByHeight(
open class BlockByHeight(
private val heights: Reader<Long, BlockHash>,
private val blocks: Reader<BlockHash, BlockJson<TransactionRefJson>>
): Reader<Long, BlockJson<TransactionRefJson>> {

9
src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt vendored
View File

@@ -24,7 +24,7 @@ import reactor.core.publisher.Mono
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
class BlocksMemCache(
open class BlocksMemCache(
val maxSize: Int = 64
): Reader<BlockHash, BlockJson<TransactionRefJson>> {
@@ -35,7 +35,11 @@ class BlocksMemCache(
return Mono.justOrEmpty(mapping[key])
}
fun add(block: BlockJson<TransactionRefJson>) {
open fun get(key: BlockHash): BlockJson<TransactionRefJson>? {
return mapping[key]
}
open fun add(block: BlockJson<TransactionRefJson>) {
mapping.put(block.hash, block)
queue.add(block.hash)
@@ -44,4 +48,5 @@ class BlocksMemCache(
mapping.remove(old)
}
}
}

131
src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt vendored Normal file
View File

@@ -0,0 +1,131 @@
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory
open class Caches(
private val blocksByHash: BlocksMemCache,
private val blocksByHeight: HeightCache,
private val txsByHash: TxMemCache
) {
companion object {
private val log = LoggerFactory.getLogger(Caches::class.java)
@JvmStatic
fun newBuilder(): Builder {
return Builder()
}
@JvmStatic
fun default(): Caches {
return newBuilder().build()
}
}
/**
* Cache data that was just requested
*/
fun cacheRequested(data: Any) {
if (data is TransactionJson) {
cache(Tag.REQUESTED, data)
} else if (data is BlockJson<*>) {
cache(Tag.REQUESTED, data as BlockJson<TransactionRefJson>)
}
}
fun cache(tag: Tag, tx: TransactionJson) {
txsByHash.add(tx)
}
fun cache(tag: Tag, block: BlockJson<TransactionRefJson>) {
if (tag == Tag.LATEST) {
blocksByHash.add(block)
val replaced = blocksByHeight.add(block)
//evict cached transactions if an existing block was updated
replaced?.let { replacedBlockHash ->
var evicted = false
blocksByHash.get(replacedBlockHash)?.let { block ->
txsByHash.evict(block)
evicted = true
}
if (!evicted) {
txsByHash.evict(replacedBlockHash)
}
}
} else if (tag == Tag.REQUESTED) {
// if block with transactions was requests cache only transactions
block.transactions.forEach { tx ->
if (tx is TransactionJson) {
cache(Tag.REQUESTED, tx)
}
}
}
}
fun getBlocksByHash(): Reader<BlockHash, BlockJson<TransactionRefJson>> {
return blocksByHash
}
fun getBlockHashByHeight(): Reader<Long, BlockHash> {
return blocksByHeight
}
fun getBlocksByHeight(): Reader<Long, BlockJson<TransactionRefJson>> {
return BlockByHeight(blocksByHeight, blocksByHash)
}
fun getTxByHash(): Reader<TransactionId, TransactionJson> {
return txsByHash
}
enum class Tag {
/**
* Latest data produced by blockchain
*/
LATEST,
/**
* Data requested by client
*/
REQUESTED
}
class Builder() {
private var blocksByHash: BlocksMemCache? = null
private var blocksByHeight: HeightCache? = null
private var txsByHash: TxMemCache? = null
fun setBlockByHash(cache: BlocksMemCache): Builder {
blocksByHash = cache
return this
}
fun setBlockByHeight(cache: HeightCache): Builder {
blocksByHeight = cache
return this
}
fun setTxByHash(cache: TxMemCache): Builder {
txsByHash = cache
return this
}
fun build(): Caches {
if (blocksByHash == null) {
blocksByHash = BlocksMemCache()
}
if (blocksByHeight == null) {
blocksByHeight = HeightCache()
}
if (txsByHash == null) {
txsByHash = TxMemCache()
}
return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!)
}
}
}

10
src/main/kotlin/io/emeraldpay/dshackle/cache/CachesEnabled.kt vendored Normal file
View File

@@ -0,0 +1,10 @@
package io.emeraldpay.dshackle.cache
/**
* Service is using caches
*/
interface CachesEnabled {
fun setCaches(caches: Caches)
}

7
src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt vendored
View File

@@ -11,7 +11,7 @@ import java.util.concurrent.ConcurrentHashMap
/**
* Memory cache for blocks heights, keeps mapping height->hash.
*/
class HeightCache(
open class HeightCache(
val maxSize: Int = 256
): Reader<Long, BlockHash> {
@@ -25,7 +25,8 @@ class HeightCache(
return Mono.justOrEmpty(heights[key])
}
fun add(block: BlockJson<TransactionRefJson>) {
open fun add(block: BlockJson<TransactionRefJson>): BlockHash? {
val existing = heights[block.number]
heights[block.number] = block.hash
// evict old numbers if full
@@ -34,5 +35,7 @@ class HeightCache(
heights.remove(dropHeight)
dropHeight++
}
return existing
}
}

60
src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt vendored Normal file
View File

@@ -0,0 +1,60 @@
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
/**
* Memory cache for transactions
*/
open class TxMemCache(
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
private val maxSize: Int = 125 * 32
): Reader<TransactionId, TransactionJson> {
companion object {
private val log = LoggerFactory.getLogger(TxMemCache::class.java)
}
private val mapping = ConcurrentHashMap<TransactionId, TransactionJson>()
private val queue = ConcurrentLinkedQueue<TransactionId>()
override fun read(key: TransactionId): Mono<TransactionJson> {
return Mono.justOrEmpty(mapping[key])
}
open fun evict(block: BlockJson<TransactionRefJson>) {
block.transactions.forEach {
mapping.remove(it.hash)
}
}
open fun evict(block: BlockHash) {
val ids = mapping.filter { it.value.blockHash == block }
ids.forEach {
mapping.remove(it.key)
}
}
open fun add(tx: TransactionJson) {
//do not cache fresh transactions
if (tx.blockHash == null || tx.blockNumber == null) {
return
}
mapping.put(tx.hash, tx)
queue.add(tx.hash)
while (queue.size > maxSize) {
val old = queue.remove()
mapping.remove(old)
}
}
}

14
src/main/kotlin/io/emeraldpay/dshackle/upstream/AggregatedUpstream.kt
View File

@@ -16,9 +16,7 @@
package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.BlockByHeight
import io.emeraldpay.dshackle.cache.BlocksMemCache
import io.emeraldpay.dshackle.cache.HeightCache
import io.emeraldpay.dshackle.cache.*
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import org.springframework.context.Lifecycle
@@ -32,12 +30,11 @@ import java.util.function.Predicate
import kotlin.concurrent.withLock
abstract class AggregatedUpstream(
val objectMapper: ObjectMapper
private val objectMapper: ObjectMapper,
val caches: Caches
): Upstream, Lifecycle {
private var cacheSubscription: Disposable? = null
private val blockReaderByHash = BlocksMemCache()
private val blockReaderByHeight = HeightCache()
var cache: CachingEthereumApi = CachingEthereumApi.empty()
private val reconfigLock = ReentrantLock()
private var callMethods: CallMethods? = null
@@ -109,10 +106,9 @@ abstract class AggregatedUpstream(
reconfigLock.withLock {
cacheSubscription?.dispose()
cacheSubscription = head.getFlux().subscribe {
blockReaderByHash.add(it)
blockReaderByHeight.add(it)
caches.cache(Caches.Tag.LATEST, it)
}
cache = CachingEthereumApi(objectMapper, blockReaderByHash, BlockByHeight(blockReaderByHeight, blockReaderByHash), head)
cache = CachingEthereumApi(objectMapper, caches, head)
}
}

63
src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingEthereumApi.kt
View File

@@ -16,17 +16,14 @@
package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.reader.EmptyReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.upstream.ethereum.EmptyEthereumHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.hex.HexQuantity
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.ResponseJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
import java.math.BigInteger
@@ -34,8 +31,7 @@ import java.util.function.Function
open class CachingEthereumApi(
private val objectMapper: ObjectMapper,
private val cache: Reader<BlockHash, BlockJson<TransactionRefJson>>,
private val cacheHeight: Reader<Long, BlockJson<TransactionRefJson>>,
private val caches: Caches,
private val head: EthereumHead
): EthereumApi(objectMapper) {
@@ -44,10 +40,14 @@ open class CachingEthereumApi(
@JvmStatic
fun empty(): CachingEthereumApi {
return CachingEthereumApi(ObjectMapper(), EmptyReader(), EmptyReader(), EmptyEthereumHead())
return CachingEthereumApi(ObjectMapper(), Caches.default(), EmptyEthereumHead())
}
}
private val cacheBlocks = caches.getBlocksByHash()
private val cacheHeight = caches.getBlocksByHeight()
private val cacheTx = caches.getTxByHash()
override fun execute(id: Int, method: String, params: List<Any>): Mono<ByteArray> {
return when (method) {
"eth_blockNumber" ->
@@ -58,33 +58,54 @@ open class CachingEthereumApi(
if (params.size == 2 && (params[1] == "false" || params[1] == false))
Mono.just(params[0])
.map { BlockHash.from(it as String) }
.flatMap(cache::read)
.map(toJson(id))
.onErrorResume { t ->
log.warn("Error during read from cache", t)
Mono.empty()
}
.flatMap(cacheBlocks::read)
.transform(converter(id))
.transform(finalizer())
else Mono.empty()
"eth_getBlockByNumber" ->
if (params.size == 2 && (params[1] == "false" || params[1] == false))
Mono.just(params[0])
.map { HexQuantity.from(it as String) }
.filter {
it.value < BigInteger.valueOf(Long.MAX_VALUE)
}
.filter { it.value < BigInteger.valueOf(Long.MAX_VALUE) }
.map { it.value.toLong() }
.flatMap(cacheHeight::read)
.map(toJson(id))
.onErrorResume { t ->
log.warn("Error during read from cache", t)
Mono.empty()
}
.transform(converter(id))
.transform(finalizer())
else Mono.empty()
"eth_getTransactionByHash" ->
if (params.size == 1)
Mono.just(params[0])
.map { TransactionId.from(it as String) }
.flatMap(cacheTx::read)
.transform(converter(id))
.transform(finalizer())
else Mono.empty()
else ->
Mono.empty()
}
}
/**
* Convert to JSON RPC response
*/
fun converter(id: Int): Function<in Mono<*>, out Mono<ByteArray>> {
return Function { mono ->
mono.map(toJson(id))
}
}
/**
* Handle errors and other stuff
*/
fun finalizer(): Function<Mono<ByteArray>, Mono<ByteArray>> {
return Function { mono ->
mono.onErrorResume { t ->
log.warn("Error during read from cache", t)
Mono.empty()
}
}
}
fun toJson(id: Int): Function<Any, ByteArray> {
return Function { data ->
val resp = ResponseJson<Any, Int>()

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainUpstreams.kt
View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
@@ -31,8 +32,9 @@ import java.time.Duration
open class ChainUpstreams (
val chain: Chain,
private val upstreams: MutableList<Upstream>,
caches: Caches,
objectMapper: ObjectMapper
) : AggregatedUpstream(objectMapper), Lifecycle {
) : AggregatedUpstream(objectMapper, caches), Lifecycle {
private val log = LoggerFactory.getLogger(ChainUpstreams::class.java)
private var seq = 0

1
src/main/kotlin/io/emeraldpay/dshackle/upstream/ConfiguredUpstreams.kt
View File

@@ -144,6 +144,7 @@ open class ConfiguredUpstreams(
}
rpcApi = DirectEthereumApi(
rpcClient.build(),
null,
objectMapper,
methods
).apply {

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentUpstreams.kt
View File

@@ -16,6 +16,8 @@
package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
@@ -50,12 +52,18 @@ class CurrentUpstreams(
log.info("Upstream ${change.upstream.getId()} with chain $chain has been removed")
} else {
if (current == null) {
val created = ChainUpstreams(chain, ArrayList<Upstream>(), objectMapper)
val created = ChainUpstreams(chain, ArrayList<Upstream>(), Caches.default(), objectMapper)
if (up is CachesEnabled) {
up.setCaches(created.caches)
}
created.addUpstream(up)
created.start()
chainMapping[chain] = created
chainsBus.onNext(chain)
} else {
if (up is CachesEnabled) {
up.setCaches(current.caches)
}
current.addUpstream(up)
}
if (!callTargets.containsKey(chain)) {

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamChange.kt
View File

@@ -15,17 +15,25 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.grpc.Chain
class UpstreamChange(
val chain: Chain,
val upstream: Upstream,
val type: ChangeType
) {
): CachesEnabled {
enum class ChangeType {
ADDED,
REVALIDATED,
STALE,
REMOVED,
}
override fun setCaches(caches: Caches) {
if (upstream is CachesEnabled) {
upstream.setCaches(caches)
}
}
}

80
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DirectEthereumApi.kt
View File

@@ -17,18 +17,22 @@ package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.upstream.CallMethods
import io.grpc.Status
import io.grpc.StatusRuntimeException
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.hex.HexQuantity
import io.infinitape.etherjar.rpc.*
import io.infinitape.etherjar.rpc.json.ResponseJson
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
import reactor.core.publisher.switchIfEmpty
import java.time.Duration
import java.math.BigInteger
open class DirectEthereumApi(
val rpcClient: ReactorRpcClient,
var caches: Caches?,
private val objectMapper: ObjectMapper,
val targets: CallMethods
): EthereumApi(objectMapper) {
@@ -90,8 +94,78 @@ open class DirectEthereumApi(
}
}
/**
* Actual request to the remote endpoint
*/
private fun callUpstream(method: String, params: List<Any>): Mono<out Any> {
return rpcClient.execute(RpcCall.create(method, Any::class.java, params))
return rpcClient.execute(callMapping(method, params))
.timeout(timeout, Mono.error(RpcException(-32603, "Upstream timeout")))
.doOnNext { value ->
caches?.cacheRequested(value)
}
}
/**
* Prepare RpcCall with data types specific for that particular requests. In general it may return a call that just
* parses JSON into Map. But the purpose of further processing and caching for some of the requests we want
* to have actual data types.
*/
fun callMapping(method: String, params: List<Any>): RpcCall<out Any, out Any> {
return when {
method == "eth_getTransactionByHash" -> {
if (params.size != 1) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "Must provide 1 parameter")
}
val hash: TransactionId
try {
hash = TransactionId.from(params[0].toString())
} catch (e: IllegalArgumentException) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "[0] must be transaction id")
}
Commands.eth().getTransaction(hash)
}
method == "eth_getBlockByHash" -> {
if (params.size != 2) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "Must provide 2 parameters")
}
val hash: BlockHash
try {
hash = BlockHash.from(params[0].toString())
} catch (e: IllegalArgumentException) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "[0] must be block hash")
}
val withTx = params[1].toString().toBoolean()
if (withTx) {
Commands.eth().getBlockWithTransactions(hash)
} else {
Commands.eth().getBlock(hash)
}
}
method == "eth_getBlockByNumber" -> {
if (params.size != 2) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "Must provide 2 parameters")
}
val number: Long
try {
val quantity = HexQuantity.from(params[0].toString()) ?: throw IllegalArgumentException()
number = quantity.value.let {
if (it < BigInteger.valueOf(Long.MAX_VALUE) && it >= BigInteger.ZERO) {
it.toLong()
} else {
throw IllegalArgumentException()
}
}
} catch (e: IllegalArgumentException) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "[0] must be block number")
}
val withTx = params[1].toString().toBoolean()
if (withTx) {
Commands.eth().getBlockWithTransactions(number)
} else {
Commands.eth().getBlock(number)
}
}
else -> RpcCall.create(method, Any::class.java, params)
}
}
}

8
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt
View File

@@ -15,6 +15,8 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.grpc.Chain
@@ -32,7 +34,7 @@ open class EthereumUpstream(
private val options: UpstreamsConfig.Options,
val node: NodeDetailsList.NodeDetails,
private val targets: CallMethods
): DefaultUpstream(), Lifecycle {
): DefaultUpstream(), CachesEnabled, Lifecycle {
constructor(id: String, chain: Chain, api: DirectEthereumApi): this(id, chain, api, null,
UpstreamsConfig.Options.getDefaults(), NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels()),
@@ -48,6 +50,10 @@ open class EthereumUpstream(
api.upstream = this
}
override fun setCaches(caches: Caches) {
api.caches = caches;
}
override fun getId(): String {
return id
}

11
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt
View File

@@ -21,6 +21,8 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.ethereum.DefaultEthereumHead
@@ -53,10 +55,11 @@ open class GrpcUpstream(
private val blockchainStub: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val objectMapper: ObjectMapper,
private val rpcClient: ReactorEmeraldClient
): DefaultUpstream(), Lifecycle {
): DefaultUpstream(), CachesEnabled, Lifecycle {
private var allLabels: Collection<UpstreamsConfig.Labels> = ArrayList<UpstreamsConfig.Labels>()
private val log = LoggerFactory.getLogger(GrpcUpstream::class.java)
private var caches: Caches? = null
private val options = UpstreamsConfig.Options.getDefaults()
private val nodes = AtomicReference<NodeDetailsList>(NodeDetailsList())
@@ -71,7 +74,7 @@ open class GrpcUpstream(
val client = Selector.extractLabels(matcher)?.let { selector ->
rpcClient.copyWithSelector(selector.asProto())
} ?: rpcClient
return DirectEthereumApi(client, objectMapper, targets).let {
return DirectEthereumApi(client, caches, objectMapper, targets).let {
it.upstream = this
it
}
@@ -205,4 +208,8 @@ open class GrpcUpstream(
return options
}
override fun setCaches(caches: Caches) {
this.caches = caches
}
}

145
src/test/groovy/io/emeraldpay/dshackle/cache/CachesSpec.groovy vendored Normal file
View File

@@ -0,0 +1,145 @@
package io.emeraldpay.dshackle.cache
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification
class CachesSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33"
def "Evict txes if block updated"() {
setup:
TxMemCache txCache = Mock()
HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder()
.setTxByHash(txCache)
.setBlockByHeight(heightCache)
.setBlockByHash(blocksCache)
.build()
def block1 = new BlockJson()
block1.number = 100
block1.hash = BlockHash.from(hash1)
def block2 = new BlockJson()
block2.number = 100
block2.hash = BlockHash.from(hash2)
when:
caches.cache(Caches.Tag.LATEST, block1)
then:
1 * blocksCache.add(block1)
1 * heightCache.add(block1) >> null
when:
caches.cache(Caches.Tag.LATEST, block2)
then:
1 * blocksCache.add(block2)
1 * heightCache.add(block2) >> block1.hash
1 * blocksCache.get(block1.hash) >> block1
1 * txCache.evict(block1)
}
def "Evict txes if block updated - when block not cached"() {
setup:
TxMemCache txCache = Mock()
HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder()
.setTxByHash(txCache)
.setBlockByHeight(heightCache)
.setBlockByHash(blocksCache)
.build()
def block1 = new BlockJson()
block1.number = 100
block1.hash = BlockHash.from(hash1)
def block2 = new BlockJson()
block2.number = 100
block2.hash = BlockHash.from(hash2)
when:
caches.cache(Caches.Tag.LATEST, block1)
then:
1 * blocksCache.add(block1)
1 * heightCache.add(block1) >> null
when:
caches.cache(Caches.Tag.LATEST, block2)
then:
1 * blocksCache.add(block2)
1 * heightCache.add(block2) >> block1.hash
1 * blocksCache.get(block1.hash) >> null
1 * txCache.evict(block1.hash)
}
def "Do not cache txes of a requested block if it's just id"() {
setup:
TxMemCache txCache = Mock()
HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder()
.setTxByHash(txCache)
.setBlockByHeight(heightCache)
.setBlockByHash(blocksCache)
.build()
def block = new BlockJson()
block.number = 100
block.hash = BlockHash.from(hash1)
block.transactions = [
new TransactionRefJson(TransactionId.from(hash1)),
new TransactionRefJson(TransactionId.from(hash2)),
]
when:
caches.cache(Caches.Tag.REQUESTED, block)
then:
0 * txCache.add(_)
}
def "Cache txes of a requested block"() {
setup:
TxMemCache txCache = Mock()
HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder()
.setTxByHash(txCache)
.setBlockByHeight(heightCache)
.setBlockByHash(blocksCache)
.build()
def tx1 = new TransactionJson().with {
hash = TransactionId.from(hash1)
blockHash = BlockHash.from(hash1)
blockNumber = 100
it
}
def tx2 = new TransactionJson().with {
hash = TransactionId.from(hash2)
blockHash = BlockHash.from(hash1)
blockNumber = 100
it
}
def block = new BlockJson()
block.number = 100
block.hash = BlockHash.from(hash1)
block.transactions = [tx1, tx2]
when:
caches.cache(Caches.Tag.REQUESTED, block)
then:
1 * txCache.add(tx1)
1 * txCache.add(tx2)
}
}

131
src/test/groovy/io/emeraldpay/dshackle/cache/TxMemCacheSpec.groovy vendored Normal file
View File

@@ -0,0 +1,131 @@
package io.emeraldpay.dshackle.cache
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification
class TxMemCacheSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33"
String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
def "Add and read"() {
setup:
def cache = new TxMemCache()
def tx = new TransactionJson()
tx.hash = TransactionId.from(hash1)
tx.blockHash = BlockHash.from(hash1)
tx.blockNumber = 100
when:
cache.add(tx)
def act = cache.read(TransactionId.from(hash1)).block()
then:
act == tx
}
def "Keeps only configured amount"() {
setup:
def cache = new TxMemCache(3)
when:
[hash1, hash2, hash3, hash4].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson()
tx.blockNumber = 100 + i
tx.blockHash = BlockHash.from(hash)
tx.hash = TransactionId.from(hash)
cache.add(tx)
}
def act1 = cache.read(TransactionId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block()
then:
act2.hash.toHex() == hash2
act3.hash.toHex() == hash3
act4.hash.toHex() == hash4
act1 == null
}
def "Evict all by block hash"() {
setup:
def cache = new TxMemCache()
when:
[hash1, hash2].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson()
tx.blockNumber = 100
tx.blockHash = BlockHash.from(hash1)
tx.hash = TransactionId.from(hash)
cache.add(tx)
}
[hash3, hash4].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson()
tx.blockNumber = 101
tx.blockHash = BlockHash.from(hash2)
tx.hash = TransactionId.from(hash)
cache.add(tx)
}
cache.evict(BlockHash.from(hash1))
def act1 = cache.read(TransactionId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block()
then:
act1 == null
act2 == null
act3.hash.toHex() == hash3
act4.hash.toHex() == hash4
}
def "Evict all by block data"() {
setup:
def cache = new TxMemCache()
when:
[hash1, hash2].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson()
tx.blockNumber = 100
tx.blockHash = BlockHash.from(hash1)
tx.hash = TransactionId.from(hash)
cache.add(tx)
}
[hash3, hash4].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson()
tx.blockNumber = 100
tx.blockHash = BlockHash.from(hash2)
tx.hash = TransactionId.from(hash)
cache.add(tx)
}
def block = new BlockJson<TransactionRefJson>()
block.hash = BlockHash.from(hash1)
block.number = 100
block.transactions = [
new TransactionRefJson(TransactionId.from(hash1)),
new TransactionRefJson(TransactionId.from(hash2)),
]
cache.evict(block)
def act1 = cache.read(TransactionId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block()
then:
act1 == null
act2 == null
act3.hash.toHex() == hash3
act4.hash.toHex() == hash4
}
}

2
src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiMock.groovy
View File

@@ -39,7 +39,7 @@ class EthereumApiMock extends DirectEthereumApi {
private ObjectMapper objectMapper
EthereumApiMock(@NotNull ReactorRpcClient rpcClient, @NotNull ObjectMapper objectMapper, @NotNull Chain chain) {
super(rpcClient, objectMapper, new DirectCallMethods())
super(rpcClient, null, objectMapper, new DirectCallMethods())
this.objectMapper = objectMapper
}

2
src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiStub.groovy
View File

@@ -38,7 +38,7 @@ class EthereumApiStub extends DirectEthereumApi {
}
EthereumApiStub(String id) {
super(rpcClient, objectMapper, new DirectCallMethods())
super(rpcClient, null, objectMapper, new DirectCallMethods())
this.id = id
}

3
src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy
View File

@@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.module.SimpleModule
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.upstream.AggregatedUpstream
import io.emeraldpay.dshackle.upstream.CallMethods
import io.emeraldpay.dshackle.upstream.ChainUpstreams
@@ -73,6 +74,6 @@ class TestingCommons {
}
static AggregatedUpstream aggregatedUpstream(EthereumUpstream up) {
return new ChainUpstreams(Chain.ETHEREUM, [up], objectMapper())
return new ChainUpstreams(Chain.ETHEREUM, [up], Caches.default(), objectMapper())
}
}

4
src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy
View File

@@ -15,7 +15,7 @@
*/
package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.upstream.AggregatedUpstream
import io.emeraldpay.dshackle.upstream.ChainUpstreams
import io.emeraldpay.dshackle.upstream.QuorumBasedMethods
@@ -40,7 +40,7 @@ class UpstreamsMock implements Upstreams {
AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull Upstream up) {
if (!upstreams.containsKey(chain)) {
upstreams[chain] = new ChainUpstreams(chain, [up], TestingCommons.objectMapper())
upstreams[chain] = new ChainUpstreams(chain, [up], Caches.default(), TestingCommons.objectMapper())
} else {
upstreams[chain].addUpstream(up)
}

3
src/test/groovy/io/emeraldpay/dshackle/upstream/AggregatedUpstreamSpec.groovy
View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
@@ -28,7 +29,7 @@ class AggregatedUpstreamSpec extends Specification {
setup:
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test1", "eth_test2"]))
def up2 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test2", "eth_test3"]))
def aggr = new ChainUpstreams(Chain.ETHEREUM, [up1, up2], TestingCommons.objectMapper())
def aggr = new ChainUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(), TestingCommons.objectMapper())
when:
aggr.onUpstreamsUpdated()
def act = aggr.getMethods()

13
src/test/groovy/io/emeraldpay/dshackle/upstream/CachingEthereumApiSpec.groovy
View File

@@ -2,6 +2,7 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.cache.BlockByHeight
import io.emeraldpay.dshackle.cache.BlocksMemCache
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.HeightCache
import io.emeraldpay.dshackle.reader.EmptyReader
import io.emeraldpay.dshackle.test.TestingCommons
@@ -23,8 +24,7 @@ class CachingEthereumApiSpec extends Specification {
def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi(
TestingCommons.objectMapper(),
new EmptyReader<BlockHash, BlockJson<TransactionRefJson>>(),
new EmptyReader<>(),
Caches.default(),
head
)
1 * head.getFlux() >> Flux.just(new BlockJson<TransactionRefJson>(number: 100))
@@ -43,8 +43,7 @@ class CachingEthereumApiSpec extends Specification {
def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi(
TestingCommons.objectMapper(),
new EmptyReader<BlockHash, BlockJson<TransactionRefJson>>(),
new EmptyReader<>(),
Caches.default(),
head
)
when:
@@ -62,8 +61,7 @@ class CachingEthereumApiSpec extends Specification {
def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi(
TestingCommons.objectMapper(),
cache,
new EmptyReader<>(),
Caches.newBuilder().setBlockByHash(cache).build(),
head
)
cache.add(new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")))
@@ -85,8 +83,7 @@ class CachingEthereumApiSpec extends Specification {
def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi(
TestingCommons.objectMapper(),
blocksCache,
new BlockByHeight(heightCache, blocksCache),
Caches.newBuilder().setBlockByHash(blocksCache).setBlockByHeight(heightCache).build(),
head
)
def block = new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"))

2
src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy
View File

@@ -47,7 +47,7 @@ class FilteredApisSpec extends Specification {
new EthereumUpstream(
"test",
Chain.ETHEREUM,
new DirectEthereumApi(rpcClient, objectMapper, ethereumTargets),
new DirectEthereumApi(rpcClient, null, objectMapper, ethereumTargets),
(EthereumWs) null,
new UpstreamsConfig.Options(),
new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap(it)),

110
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DirectEthereumApiSpec.groovy
View File

@@ -20,6 +20,8 @@ import io.emeraldpay.dshackle.upstream.DirectCallMethods
import io.infinitape.etherjar.rpc.ReactorRpcClient
import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.RpcResponseError
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import spock.lang.Specification
@@ -28,7 +30,7 @@ import java.time.Duration
class DirectEthereumApiSpec extends Specification {
DirectEthereumApi api = new DirectEthereumApi(Stub(ReactorRpcClient), TestingCommons.objectMapper(), new DirectCallMethods())
DirectEthereumApi api = new DirectEthereumApi(Stub(ReactorRpcClient), null, TestingCommons.objectMapper(), new DirectCallMethods())
def "Process successful result"() {
setup:
@@ -89,4 +91,110 @@ class DirectEthereumApiSpec extends Specification {
.verify(Duration.ofSeconds(1))
}
def "Typed mapping for block request"() {
when:
def act = api.callMapping("eth_getBlockByHash", ["0xacf5611707048efc39cabed483e420672ca1ed070f248ef6202c99994dbc6061", false])
then:
act.jsonType == BlockJson
act.resultType == BlockJson
}
def "Typed mapping for block request with txes"() {
when:
def act = api.callMapping("eth_getBlockByHash", ["0xacf5611707048efc39cabed483e420672ca1ed070f248ef6202c99994dbc6061", true])
then:
act.jsonType == BlockJson
act.resultType == BlockJson
}
def "Typed mapping for block by height request"() {
when:
def act = api.callMapping("eth_getBlockByNumber", ["0x135", false])
then:
act.jsonType == BlockJson
act.resultType == BlockJson
}
def "Typed mapping for block by height request with txes"() {
when:
def act = api.callMapping("eth_getBlockByNumber", ["0xacf5", true])
then:
act.jsonType == BlockJson
act.resultType == BlockJson
}
def "Typed mapping for tx request"() {
when:
def act = api.callMapping("eth_getTransactionByHash", ["0xacf5611707048efc39cabed483e420672ca1ed070f248ef6202c99994dbc6061"])
then:
act.jsonType == TransactionJson
act.resultType == TransactionJson
}
def "Errors for mapping of invalid tx request"() {
when:
api.callMapping("eth_getTransactionByHash", ["0xacf5611707048efc39cabed483e420672ca1ed070f248ef6"])
then:
def t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getTransactionByHash", ["0xacf5611707048efc39cabed483e420672ca1ed070f248ef6202c99994dbc6061", true])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getTransactionByHash", [])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
}
def "Errors for mapping of invalid block request"() {
when:
api.callMapping("eth_getBlockByHash", ["0xacf5611707048efc39cabed483e420672ca1ed070f248ef6202c99994dbc6061"])
then:
def t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getBlockByHash", ["0xacf5611707048efc39cabed48f6202c99994dbc6061", true])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getBlockByHash", [])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
}
def "Errors for mapping of invalid block by number request"() {
when:
api.callMapping("eth_getBlockByNumber", ["0xacf5611707048efc3248ef6202c99994dbc6061"])
then:
def t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getBlockByNumber", ["0x", true])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getBlockByNumber", ["-0x23", true])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
when:
api.callMapping("eth_getBlockByNumber", [])
then:
t = thrown(RpcException)
t.code == RpcResponseError.CODE_INVALID_METHOD_PARAMS
}
}