problem: keeps data (block/tx) as business object, which requires special treating

solution: keep as bytes (json string) + meta
This commit is contained in:
Igor Artamonov
2020-04-14 00:25:25 -04:00
parent 0ab68ea782
commit 9cf1827fe1
73 changed files with 1235 additions and 565 deletions

View File

@@ -1,25 +1,24 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
/** /**
* Connects two caches to read through them. First is cache height->hash, second is hash->block. * Connects two caches to read through them. First is cache height->hash, second is hash->block.
*/ */
open class BlockByHeight<T: TransactionRefJson>( open class BlockByHeight(
private val heights: Reader<Long, BlockHash>, private val heights: Reader<Long, BlockId>,
private val blocks: Reader<BlockHash, BlockJson<T>> private val blocks: Reader<BlockId, BlockContainer>
): Reader<Long, BlockJson<T>> { ) : Reader<Long, BlockContainer> {
companion object { companion object {
private val log = LoggerFactory.getLogger(BlockByHeight::class.java) private val log = LoggerFactory.getLogger(BlockByHeight::class.java)
} }
override fun read(key: Long): Mono<BlockJson<T>> { override fun read(key: Long): Mono<BlockContainer> {
return heights.read(key) return heights.read(key)
.flatMap { blocks.read(it) } .flatMap { blocks.read(it) }
} }

View File

@@ -15,31 +15,29 @@
*/ */
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.ConcurrentLinkedQueue
open class BlocksMemCache( open class BlocksMemCache(
val maxSize: Int = 64 val maxSize: Int = 64
): Reader<BlockHash, BlockJson<TransactionRefJson>> { ) : Reader<BlockId, BlockContainer> {
private val mapping = ConcurrentHashMap<BlockHash, BlockJson<TransactionRefJson>>() private val mapping = ConcurrentHashMap<BlockId, BlockContainer>()
private val queue = ConcurrentLinkedQueue<BlockHash>() private val queue = ConcurrentLinkedQueue<BlockId>()
override fun read(key: BlockHash): Mono<BlockJson<TransactionRefJson>> { override fun read(key: BlockId): Mono<BlockContainer> {
return Mono.justOrEmpty(mapping[key]) return Mono.justOrEmpty(mapping[key])
} }
open fun get(key: BlockHash): BlockJson<TransactionRefJson>? { open fun get(key: BlockId): BlockContainer? {
return mapping[key] return mapping[key]
} }
open fun add(block: BlockJson<TransactionRefJson>) { open fun add(block: BlockContainer) {
mapping.put(block.hash, block) mapping.put(block.hash, block)
queue.add(block.hash) queue.add(block.hash)

View File

@@ -1,12 +1,13 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import io.lettuce.core.api.reactive.RedisReactiveCommands import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.apache.commons.codec.binary.Base64
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.time.Instant import java.time.Instant
@@ -20,25 +21,27 @@ class BlocksRedisCache(
private val redis: RedisReactiveCommands<String, String>, private val redis: RedisReactiveCommands<String, String>,
private val chain: Chain, private val chain: Chain,
private val objectMapper: ObjectMapper private val objectMapper: ObjectMapper
): Reader<BlockHash, BlockJson<TransactionRefJson>> { ) : Reader<BlockId, BlockContainer> {
companion object { companion object {
private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java) private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java)
private const val MAX_CACHE_TIME_MINUTES = 60L private const val MAX_CACHE_TIME_MINUTES = 60L
// doesn't make sense to cached in redis short living objects // doesn't make sense to cached in redis short living objects
private const val MIN_CACHE_TIME_SECONDS = 10 private const val MIN_CACHE_TIME_SECONDS = 10
} }
override fun read(key: BlockHash): Mono<BlockJson<TransactionRefJson>> { override fun read(key: BlockId): Mono<BlockContainer> {
return redis.get(key(key)) return redis.get(key(key))
.map { data -> .map { data ->
objectMapper.readValue(data, BlockJson::class.java) as BlockJson<TransactionRefJson> val block = objectMapper.readValue(data, BlockJson::class.java)
BlockContainer.from(block, objectMapper)
}.onErrorResume { }.onErrorResume {
Mono.empty() Mono.empty()
} }
} }
fun evict(id: BlockHash): Mono<Void> { fun evict(id: BlockId): Mono<Void> {
return Mono.just(id) return Mono.just(id)
.flatMap { .flatMap {
redis.del(key(it)) redis.del(key(it))
@@ -50,18 +53,17 @@ class BlocksRedisCache(
* Add to cache. * Add to cache.
* Note that it returns Mono<Void> which must be subscribed to actually save * Note that it returns Mono<Void> which must be subscribed to actually save
*/ */
fun add(block: BlockJson<TransactionRefJson>): Mono<Void> { fun add(block: BlockContainer): Mono<Void> {
if (block.timestamp == null || block.hash == null) { if (block.timestamp == null || block.hash == null) {
return Mono.empty() return Mono.empty()
} }
return Mono.just(block) return Mono.just(block)
.flatMap { block -> .flatMap { block ->
val data = String(block.json!!)
val data = objectMapper.writeValueAsString(block)
//default caching time is age of the block, i.e. block create hour ago //default caching time is age of the block, i.e. block create hour ago
//keep for hour, but block create 10 seconds ago cache for 10 seconds, as it //keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
//still can be replaced in the blockchain //still can be replaced in the blockchain
val age = Instant.now().epochSecond - block.timestamp.epochSecond val age = Instant.now().epochSecond - block.timestamp!!.epochSecond
val ttl = min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)) val ttl = min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES))
if (ttl > MIN_CACHE_TIME_SECONDS) { if (ttl > MIN_CACHE_TIME_SECONDS) {
redis.setex(key(block.hash), ttl, data) redis.setex(key(block.hash), ttl, data)
@@ -82,7 +84,7 @@ class BlocksRedisCache(
/** /**
* Key in Redis * Key in Redis
*/ */
fun key(hash: BlockHash): String { fun key(hash: BlockId): String {
return "block:${chain.id}:${hash.toHex()}" return "block:${chain.id}:${hash.toHex()}"
} }
} }

View File

@@ -1,23 +1,25 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.CompoundReader import io.emeraldpay.dshackle.reader.CompoundReader
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor
open class Caches( open class Caches(
private val memBlocksByHash: BlocksMemCache, private val memBlocksByHash: BlocksMemCache,
private val blocksByHeight: HeightCache, private val blocksByHeight: HeightCache,
private val memTxsByHash: TxMemCache, private val memTxsByHash: TxMemCache,
private val redisBlocksByHash: BlocksRedisCache?, private val redisBlocksByHash: BlocksRedisCache?,
private val redisTxsByHash: TxRedisCache? private val redisTxsByHash: TxRedisCache?,
private val objectMapper: ObjectMapper
) { ) {
companion object { companion object {
@@ -29,13 +31,13 @@ open class Caches(
} }
@JvmStatic @JvmStatic
fun default(): Caches { fun default(objectMapper: ObjectMapper): Caches {
return newBuilder().build() return newBuilder().setObjectMapper(objectMapper).build()
} }
} }
private val blocksByHash: Reader<BlockHash, BlockJson<TransactionRefJson>> private val blocksByHash: Reader<BlockId, BlockContainer>
private val txsByHash: Reader<TransactionId, TransactionJson> private val txsByHash: Reader<TxId, TxContainer>
init { init {
blocksByHash = if (redisBlocksByHash == null) { blocksByHash = if (redisBlocksByHash == null) {
@@ -54,25 +56,25 @@ open class Caches(
* Cache data that was just requested * Cache data that was just requested
*/ */
fun cacheRequested(data: Any) { fun cacheRequested(data: Any) {
if (data is TransactionJson) { if (data is TxContainer) {
cache(Tag.REQUESTED, data)
} else if (data is BlockContainer) {
cache(Tag.REQUESTED, data) cache(Tag.REQUESTED, data)
} else if (data is BlockJson<*>) {
cache(Tag.REQUESTED, data as BlockJson<TransactionRefJson>)
} }
} }
fun cache(tag: Tag, tx: TransactionJson) { fun cache(tag: Tag, tx: TxContainer) {
//do not cache transactions that are not in a block yet //do not cache transactions that are not in a block yet
if (tx.blockHash == null) { if (tx.blockId == null) {
return return
} }
memTxsByHash.add(tx) memTxsByHash.add(tx)
memBlocksByHash.get(tx.blockHash)?.let { block -> memBlocksByHash.get(tx.blockId)?.let { block ->
redisTxsByHash?.add(tx, block) redisTxsByHash?.add(tx, block)
} }
} }
fun cache(tag: Tag, block: BlockJson<TransactionRefJson>) { fun cache(tag: Tag, block: BlockContainer) {
val job = ArrayList<Mono<Void>>() val job = ArrayList<Mono<Void>>()
if (tag == Tag.LATEST) { if (tag == Tag.LATEST) {
//for LATEST data cache in memory, it will be short living so better to avoid Redis //for LATEST data cache in memory, it will be short living so better to avoid Redis
@@ -92,45 +94,60 @@ open class Caches(
} }
} }
} else if (tag == Tag.REQUESTED) { } else if (tag == Tag.REQUESTED) {
var blockOnlyContainer: BlockContainer? = null
var jsonValue: BlockJson<*>? = null
if (block.full) {
jsonValue = objectMapper.readValue<BlockJson<*>>(block.json, BlockJson::class.java)
//shouldn't cache block json with transactions, separate txes and blocks with refs //shouldn't cache block json with transactions, separate txes and blocks with refs
val blockOnly = block.withoutTransactionDetails() val blockOnly = jsonValue.withoutTransactionDetails()
memBlocksByHash.add(blockOnly) blockOnlyContainer = BlockContainer.from(blockOnly, objectMapper)
redisBlocksByHash?.add(blockOnly)?.let(job::add) } else {
blockOnlyContainer = block
}
memBlocksByHash.add(blockOnlyContainer)
redisBlocksByHash?.add(blockOnlyContainer)?.let(job::add)
// now cache only transactions // now cache only transactions
val transactions = block.transactions.filterIsInstance<TransactionJson>() jsonValue?.let { jsonValue ->
if (transactions.isNotEmpty()) { val plainTransactions = jsonValue.transactions.filterIsInstance<TransactionJson>()
transactions.forEach { cache(Tag.REQUESTED, it) } if (plainTransactions.isNotEmpty()) {
val transactions = plainTransactions.map { tx ->
TxContainer.from(tx, objectMapper)
}
transactions.forEach {
cache(Tag.REQUESTED, it)
}
if (redisTxsByHash != null) { if (redisTxsByHash != null) {
job.add(Flux.fromIterable(transactions).flatMap { redisTxsByHash.add(it, block) }.then()) job.add(Flux.fromIterable(transactions).flatMap { redisTxsByHash.add(it, block) }.then())
} }
} }
} }
}
Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller
} }
fun getBlocksByHash(): Reader<BlockHash, BlockJson<TransactionRefJson>> { fun getBlocksByHash(): Reader<BlockId, BlockContainer> {
return blocksByHash return blocksByHash
} }
fun getBlockHashByHeight(): Reader<Long, BlockHash> { fun getBlockHashByHeight(): Reader<Long, BlockId> {
return blocksByHeight return blocksByHeight
} }
fun getBlocksByHeight(): Reader<Long, BlockJson<TransactionRefJson>> { fun getBlocksByHeight(): Reader<Long, BlockContainer> {
return BlockByHeight(blocksByHeight, blocksByHash) return BlockByHeight(blocksByHeight, blocksByHash)
} }
fun getTxByHash(): Reader<TransactionId, TransactionJson> { fun getTxByHash(): Reader<TxId, TxContainer> {
return txsByHash return txsByHash
} }
fun getFullBlocks(): Reader<BlockHash, BlockJson<TransactionJson>> { fun getFullBlocks(): Reader<BlockId, BlockContainer> {
return BlocksWithTxCache(blocksByHash, txsByHash) return EthereumBlocksWithTxCache(objectMapper, blocksByHash, txsByHash)
} }
fun getFullBlocksByHeight(): Reader<Long, BlockJson<TransactionJson>> { fun getFullBlocksByHeight(): Reader<Long, BlockContainer> {
return BlockByHeight(blocksByHeight, BlocksWithTxCache(blocksByHash, txsByHash)) return BlockByHeight(blocksByHeight, EthereumBlocksWithTxCache(objectMapper, blocksByHash, txsByHash))
} }
enum class Tag { enum class Tag {
@@ -138,6 +155,7 @@ open class Caches(
* Latest data produced by blockchain * Latest data produced by blockchain
*/ */
LATEST, LATEST,
/** /**
* Data requested by client * Data requested by client
*/ */
@@ -150,6 +168,7 @@ open class Caches(
private var txsByHash: TxMemCache? = null private var txsByHash: TxMemCache? = null
private var redisBlocksByHash: BlocksRedisCache? = null private var redisBlocksByHash: BlocksRedisCache? = null
private var redisTxsByHash: TxRedisCache? = null private var redisTxsByHash: TxRedisCache? = null
private var objectMapper: ObjectMapper? = null
fun setBlockByHash(cache: BlocksMemCache): Builder { fun setBlockByHash(cache: BlocksMemCache): Builder {
blocksByHash = cache blocksByHash = cache
@@ -176,6 +195,11 @@ open class Caches(
return this return this
} }
fun setObjectMapper(value: ObjectMapper): Builder {
objectMapper = value
return this
}
fun build(): Caches { fun build(): Caches {
if (blocksByHash == null) { if (blocksByHash == null) {
blocksByHash = BlocksMemCache() blocksByHash = BlocksMemCache()
@@ -186,7 +210,10 @@ open class Caches(
if (txsByHash == null) { if (txsByHash == null) {
txsByHash = TxMemCache() txsByHash = TxMemCache()
} }
return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, redisBlocksByHash, redisTxsByHash) if (objectMapper == null) {
throw IllegalStateException("ObjectMapper is not set")
}
return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, redisBlocksByHash, redisTxsByHash, objectMapper!!)
} }
} }
} }

View File

@@ -61,6 +61,7 @@ class CachesFactory(
private fun initCache(chain: Chain): Caches { private fun initCache(chain: Chain): Caches {
val caches = Caches.newBuilder() val caches = Caches.newBuilder()
.setObjectMapper(objectMapper)
redis?.let { redis -> redis?.let { redis ->
caches.setBlockByHash(BlocksRedisCache(redis.reactive(), chain, objectMapper)) caches.setBlockByHash(BlocksRedisCache(redis.reactive(), chain, objectMapper))
caches.setTxByHash(TxRedisCache(redis.reactive(), chain, objectMapper)) caches.setTxByHash(TxRedisCache(redis.reactive(), chain, objectMapper))

View File

@@ -1,8 +1,11 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
@@ -18,25 +21,27 @@ import reactor.core.publisher.Mono
* If source block, with just transaction hashes is not available, it returns empty * If source block, with just transaction hashes is not available, it returns empty
* If any of the expected block transactions is not available it returns empty * If any of the expected block transactions is not available it returns empty
*/ */
class BlocksWithTxCache( class EthereumBlocksWithTxCache(
private val blocks: Reader<BlockHash, BlockJson<TransactionRefJson>>, private val objectMapper: ObjectMapper,
private val txes: Reader<TransactionId, TransactionJson> private val blocks: Reader<BlockId, BlockContainer>,
): Reader<BlockHash, BlockJson<TransactionJson>> { private val txes: Reader<TxId, TxContainer>
) : Reader<BlockId, BlockContainer> {
companion object { companion object {
private val log = LoggerFactory.getLogger(BlocksWithTxCache::class.java) private val log = LoggerFactory.getLogger(EthereumBlocksWithTxCache::class.java)
} }
override fun read(key: BlockHash): Mono<BlockJson<TransactionJson>> { override fun read(key: BlockId): Mono<BlockContainer> {
return blocks.read(key).flatMap { block -> return blocks.read(key).flatMap { block ->
if (block.transactions == null || block.transactions.isEmpty()) { val block = objectMapper.readValue(block.json, BlockJson::class.java) as BlockJson<TransactionRefJson>
// in fact it's not necessary to create a copy, made just for code clarity but may be performance loss val fullBlock = if (block.transactions == null || block.transactions.isEmpty()) {
// in fact it's not necessary to create a copy, made just for code clarity but it may be a performance loss
val fullBlock = BlockJson<TransactionJson>() val fullBlock = BlockJson<TransactionJson>()
BeanUtils.copyProperties(block, fullBlock) BeanUtils.copyProperties(block, fullBlock)
Mono.just(fullBlock) Mono.just(fullBlock)
} else { } else {
Flux.fromIterable(block.transactions) Flux.fromIterable(block.transactions)
.map { it.hash } .map { TxId.from(it.hash) }
.flatMap { txes.read(it) } .flatMap { txes.read(it) }
.collectList() .collectList()
.flatMap { list -> .flatMap { list ->
@@ -45,11 +50,20 @@ class BlocksWithTxCache(
} else { } else {
val fullBlock = BlockJson<TransactionJson>() val fullBlock = BlockJson<TransactionJson>()
BeanUtils.copyProperties(block, fullBlock) BeanUtils.copyProperties(block, fullBlock)
fullBlock.transactions = list fullBlock.transactions = list.map {
objectMapper.readValue(it.json, TransactionJson::class.java)
}
Mono.just(fullBlock) Mono.just(fullBlock)
} }
} }
} }
fullBlock
.map { block ->
BlockContainer(block.number, BlockId.from(block.hash), block.totalDifficulty, block.timestamp, true,
objectMapper.writeValueAsBytes(block),
block.transactions.map { tx -> TxId.from(tx) }
)
}
} }
} }

View File

@@ -1,9 +1,8 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
@@ -13,25 +12,25 @@ import java.util.concurrent.ConcurrentHashMap
*/ */
open class HeightCache( open class HeightCache(
val maxSize: Int = 256 val maxSize: Int = 256
): Reader<Long, BlockHash> { ) : Reader<Long, BlockId> {
companion object { companion object {
private val log = LoggerFactory.getLogger(HeightCache::class.java) private val log = LoggerFactory.getLogger(HeightCache::class.java)
} }
private val heights = ConcurrentHashMap<Long, BlockHash>() private val heights = ConcurrentHashMap<Long, BlockId>()
override fun read(key: Long): Mono<BlockHash> { override fun read(key: Long): Mono<BlockId> {
return Mono.justOrEmpty(heights[key]) return Mono.justOrEmpty(heights[key])
} }
open fun add(block: BlockJson<TransactionRefJson>): BlockHash? { open fun add(block: BlockContainer): BlockId? {
val existing = heights[block.number] val existing = heights[block.height]
heights[block.number] = block.hash heights[block.height] = block.hash
// evict old numbers if full // evict old numbers if full
var dropHeight = block.number - maxSize var dropHeight = block.height - maxSize
while (heights.size > maxSize && dropHeight < block.number) { while (heights.size > maxSize && dropHeight < block.height) {
heights.remove(dropHeight) heights.remove(dropHeight)
dropHeight++ dropHeight++
} }

View File

@@ -1,11 +1,10 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
@@ -17,35 +16,35 @@ import java.util.concurrent.ConcurrentLinkedQueue
open class TxMemCache( open class TxMemCache(
// usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default // usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default
private val maxSize: Int = 125 * 32 private val maxSize: Int = 125 * 32
): Reader<TransactionId, TransactionJson> { ) : Reader<TxId, TxContainer> {
companion object { companion object {
private val log = LoggerFactory.getLogger(TxMemCache::class.java) private val log = LoggerFactory.getLogger(TxMemCache::class.java)
} }
private val mapping = ConcurrentHashMap<TransactionId, TransactionJson>() private val mapping = ConcurrentHashMap<TxId, TxContainer>()
private val queue = ConcurrentLinkedQueue<TransactionId>() private val queue = ConcurrentLinkedQueue<TxId>()
override fun read(key: TransactionId): Mono<TransactionJson> { override fun read(key: TxId): Mono<TxContainer> {
return Mono.justOrEmpty(mapping[key]) return Mono.justOrEmpty(mapping[key])
} }
open fun evict(block: BlockJson<TransactionRefJson>) { open fun evict(block: BlockContainer) {
block.transactions.forEach { block.transactions.forEach {
mapping.remove(it.hash) mapping.remove(it)
} }
} }
open fun evict(block: BlockHash) { open fun evict(block: BlockId) {
val ids = mapping.filter { it.value.blockHash == block } val ids = mapping.filter { it.value.blockId == block }
ids.forEach { ids.forEach {
mapping.remove(it.key) mapping.remove(it.key)
} }
} }
open fun add(tx: TransactionJson) { open fun add(tx: TxContainer) {
//do not cache fresh transactions //do not cache fresh transactions
if (tx.blockHash == null || tx.blockNumber == null) { if (tx.blockId == null) {
return return
} }
mapping.put(tx.hash, tx) mapping.put(tx.hash, tx)

View File

@@ -1,13 +1,14 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import io.lettuce.core.api.reactive.RedisReactiveCommands import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.apache.commons.codec.binary.Base64
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import reactor.util.function.Tuples import reactor.util.function.Tuples
@@ -22,35 +23,38 @@ class TxRedisCache(
private val redis: RedisReactiveCommands<String, String>, private val redis: RedisReactiveCommands<String, String>,
private val chain: Chain, private val chain: Chain,
private val objectMapper: ObjectMapper private val objectMapper: ObjectMapper
): Reader<TransactionId, TransactionJson> { ) : Reader<TxId, TxContainer> {
companion object { companion object {
private val log = LoggerFactory.getLogger(TxRedisCache::class.java) private val log = LoggerFactory.getLogger(TxRedisCache::class.java)
// max caching time is 24 hours // max caching time is 24 hours
private const val MAX_CACHE_TIME_HOURS = 24L private const val MAX_CACHE_TIME_HOURS = 24L
} }
override fun read(key: TransactionId): Mono<TransactionJson> { override fun read(key: TxId): Mono<TxContainer> {
return redis.get(key(key)) return redis.get(key(key))
.map { data -> .map { data ->
objectMapper.readValue(data, TransactionJson::class.java) as TransactionJson val json = data
val tx = objectMapper.readValue(json, TransactionJson::class.java)
TxContainer.from(tx, objectMapper)
}.onErrorResume { }.onErrorResume {
Mono.empty() Mono.empty()
} }
} }
fun evict(block: BlockJson<TransactionRefJson>): Mono<Void> { fun evict(block: BlockContainer): Mono<Void> {
return Mono.just(block) return Mono.just(block)
.map { block -> .map { block ->
block.transactions.map { block.transactions.map {
key(it.hash) key(it)
}.toTypedArray() }.toTypedArray()
}.flatMap { keys -> }.flatMap { keys ->
redis.del(*keys) redis.del(*keys)
}.then() }.then()
} }
fun evict(id: TransactionId): Mono<Void> { fun evict(id: TxId): Mono<Void> {
return Mono.just(id) return Mono.just(id)
.flatMap { .flatMap {
redis.del(key(it)) redis.del(key(it))
@@ -58,17 +62,17 @@ class TxRedisCache(
.then() .then()
} }
fun add(tx: TransactionJson, block: BlockJson<TransactionRefJson>): Mono<Void> { fun add(tx: TxContainer, block: BlockContainer): Mono<Void> {
if (tx.blockHash == null || block.hash == null || tx.blockHash != block.hash || block.timestamp == null) { if (tx.blockId == null || block.hash == null || tx.blockId != block.hash || block.timestamp == null) {
return Mono.empty() return Mono.empty()
} }
return Mono.just(Tuples.of(tx, block)) return Mono.just(Tuples.of(tx, block))
.flatMap { .flatMap {
val data = objectMapper.writeValueAsString(it.t1) val data = String(it.t1.json!!)
//default caching time is age of the block, i.e. block create hour ago //default caching time is age of the block, i.e. block create hour ago
//keep for hour, but block create 10 seconds ago cache for 10 seconds, as it //keep for hour, but block create 10 seconds ago cache for 10 seconds, as it
//still can be replaced in the blockchain //still can be replaced in the blockchain
val age = Instant.now().epochSecond - it.t2.timestamp.epochSecond val age = Instant.now().epochSecond - it.t2.timestamp!!.epochSecond
val ttl = min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS)) val ttl = min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS))
redis.setex(key(it.t1.hash), ttl, data) redis.setex(key(it.t1.hash), ttl, data)
} }
@@ -85,7 +89,7 @@ class TxRedisCache(
/** /**
* Key in Redis * Key in Redis
*/ */
fun key(hash: TransactionId): String { fun key(hash: TxId): String {
return "tx:${chain.id}:${hash.toHex()}" return "tx:${chain.id}:${hash.toHex()}"
} }
} }

View File

@@ -0,0 +1,76 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
import com.fasterxml.jackson.databind.ObjectMapper
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import java.math.BigInteger
import java.time.Instant
class BlockContainer(
val height: Long,
val hash: BlockId,
val difficulty: BigInteger,
val timestamp: Instant,
val full: Boolean,
json: ByteArray?,
val transactions: List<TxId> = emptyList()
) : SourceContainer(json) {
companion object {
@JvmStatic
fun from(block: BlockJson<*>, objectMapper: ObjectMapper): BlockContainer {
val hasTransactions = block.transactions?.filterIsInstance<TransactionJson>()?.count() ?: 0 > 0
return BlockContainer(
block.number,
BlockId.from(block),
block.totalDifficulty,
block.timestamp,
hasTransactions,
objectMapper.writeValueAsBytes(block),
block.transactions?.map { TxId.from(it.hash) } ?: emptyList()
)
}
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
if (!super.equals(other)) return false
other as BlockContainer
if (height != other.height) return false
if (hash != other.hash) return false
if (difficulty != other.difficulty) return false
if (timestamp != other.timestamp) return false
if (full != other.full) return false
if (transactions != other.transactions) return false
return true
}
override fun hashCode(): Int {
var result = super.hashCode()
result = 31 * result + height.hashCode()
result = 31 * result + hash.hashCode()
return result
}
}

View File

@@ -0,0 +1,42 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
import io.infinitape.etherjar.rpc.json.BlockJson
class BlockId(
value: ByteArray
) : HashId(value) {
companion object {
@JvmStatic
fun from(hash: io.infinitape.etherjar.domain.BlockHash): BlockId {
return BlockId(hash.bytes)
}
@JvmStatic
fun from(block: BlockJson<*>): BlockId {
return from(block.hash)
}
@JvmStatic
fun from(id: String): BlockId {
return from(io.infinitape.etherjar.domain.BlockHash.from(id))
}
}
}

View File

@@ -0,0 +1,58 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
open class HashId(
val value: ByteArray
) {
companion object {
val HEX_DIGITS = "0123456789abcdef".toCharArray()
}
override fun toString(): String {
return toHex()
}
fun toHex(): String {
val hex = CharArray(value.size * 2 + 2)
hex[0] = '0'
hex[1] = 'x'
var i = 0
var j = 2
while (i < value.size) {
hex[j++] = HEX_DIGITS[0xF0 and value[i].toInt() ushr 4]
hex[j++] = HEX_DIGITS[0x0F and value[i].toInt()]
i++
}
return String(hex)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is HashId) return false
if (!value.contentEquals(other.value)) return false
return true
}
override fun hashCode(): Int {
return value.contentHashCode()
}
}

View File

@@ -0,0 +1,48 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
import org.slf4j.LoggerFactory
import java.io.ByteArrayOutputStream
class RawJsonBuilder {
companion object {
private val log = LoggerFactory.getLogger(RawJsonBuilder::class.java)
private val START = "{\"jsonrpc\":\"2.0\"".toByteArray()
private val ID_START = "\"id\":".toByteArray()
private val RESULT_START = "\"result\":".toByteArray()
private val COMMA = ",".toByteArray()
private val END = "}".toByteArray()
}
fun write(id: Int, data: ByteArray): ByteArray {
val buf = ByteArrayOutputStream(data.size + 100)
buf.write(START)
buf.write(COMMA)
buf.write(ID_START)
buf.write(id.toString().toByteArray());
buf.write(COMMA)
buf.write(RESULT_START)
buf.write(data)
buf.write(END)
return buf.toByteArray()
}
}

View File

@@ -0,0 +1,37 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
abstract class SourceContainer(
val json: ByteArray?
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is SourceContainer) return false
if (json != null) {
if (other.json == null) return false
if (!json.contentEquals(other.json)) return false
} else if (other.json != null) return false
return true
}
override fun hashCode(): Int {
return json?.contentHashCode() ?: 0
}
}

View File

@@ -0,0 +1,62 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
import com.fasterxml.jackson.databind.ObjectMapper
import io.infinitape.etherjar.rpc.json.TransactionJson
class TxContainer(
val height: Long,
val hash: TxId,
val blockId: BlockId?,
json: ByteArray?
) : SourceContainer(json) {
companion object {
@JvmStatic
fun from(tx: TransactionJson, objectMapper: ObjectMapper): TxContainer {
return TxContainer(
tx.blockNumber,
TxId.from(tx.hash),
BlockId.from(tx.blockHash),
objectMapper.writeValueAsBytes(tx)
)
}
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
if (!super.equals(other)) return false
other as TxContainer
if (height != other.height) return false
if (hash != other.hash) return false
if (blockId != other.blockId) return false
return true
}
override fun hashCode(): Int {
var result = super.hashCode()
result = 31 * result + height.hashCode()
result = 31 * result + hash.hashCode()
return result
}
}

View File

@@ -0,0 +1,41 @@
/**
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.data
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.TransactionJson
class TxId(
value: ByteArray
) : HashId(value) {
companion object {
@JvmStatic
fun from(id: TransactionId): TxId {
return TxId(id.bytes)
}
@JvmStatic
fun from(tx: TransactionJson): TxId {
return from(tx.hash)
}
@JvmStatic
fun from(id: String): TxId {
return from(TransactionId.from(id))
}
}
}

View File

@@ -17,30 +17,27 @@ package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.RpcException import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
open class AlwaysQuorum: CallQuorum { open class AlwaysQuorum: CallQuorum {
private var resolved = false private var resolved = false
private var result: ByteArray? = null private var result: ByteArray? = null
override fun init(head: Head<BlockJson<TransactionRefJson>>) { override fun init(head: Head) {
} }
override fun isResolved(): Boolean { override fun isResolved(): Boolean {
return resolved return resolved
} }
override fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean { override fun record(response: ByteArray, upstream: Upstream<*>): Boolean {
result = response result = response
resolved = true resolved = true
return true return true
} }
override fun record(error: RpcException, upstream: Upstream<*, *>) { override fun record(error: RpcException, upstream: Upstream<*>) {
} }
override fun getResult(): ByteArray? { override fun getResult(): ByteArray? {

View File

@@ -17,10 +17,7 @@ package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.JacksonRpcConverter import io.infinitape.etherjar.rpc.JacksonRpcConverter
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
open class BroadcastQuorum( open class BroadcastQuorum(
jacksonRpcConverter: JacksonRpcConverter, jacksonRpcConverter: JacksonRpcConverter,
@@ -31,7 +28,7 @@ open class BroadcastQuorum(
private var txid: String? = null private var txid: String? = null
private var calls = 0 private var calls = 0
override fun init(head: Head<BlockJson<TransactionRefJson>>) { override fun init(head: Head) {
} }
override fun isResolved(): Boolean { override fun isResolved(): Boolean {
@@ -42,7 +39,7 @@ open class BroadcastQuorum(
return result return result
} }
override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*, *>) { override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*>) {
calls++ calls++
if (txid == null && responseValue != null) { if (txid == null && responseValue != null) {
txid = responseValue txid = responseValue
@@ -50,7 +47,7 @@ open class BroadcastQuorum(
} }
} }
override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) { override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) {
// can be "message: known transaction: TXID", "Transaction with the same hash was already imported" or "message: Nonce too low" // can be "message: known transaction: TXID", "Transaction with the same hash was already imported" or "message: Nonce too low"
calls++ calls++
if (result == null) { if (result == null) {

View File

@@ -27,11 +27,11 @@ import java.util.function.Predicate
interface CallQuorum { interface CallQuorum {
fun init(head: Head<BlockJson<TransactionRefJson>>) fun init(head: Head)
fun isResolved(): Boolean fun isResolved(): Boolean
fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean fun record(response: ByteArray, upstream: Upstream<*>): Boolean
fun record(error: RpcException, upstream: Upstream<*, *>) fun record(error: RpcException, upstream: Upstream<*>)
fun getResult(): ByteArray? fun getResult(): ByteArray?
companion object { companion object {
@@ -41,8 +41,8 @@ interface CallQuorum {
} }
} }
fun asReducer(): BiFunction<CallQuorum, Tuple2<ByteArray, Upstream<*, *>>, CallQuorum> { fun asReducer(): BiFunction<CallQuorum, Tuple2<ByteArray, Upstream<*>>, CallQuorum> {
return BiFunction<CallQuorum, Tuple2<ByteArray, Upstream<*, *>>, CallQuorum> { a, b -> return BiFunction<CallQuorum, Tuple2<ByteArray, Upstream<*>>, CallQuorum> { a, b ->
a.record(b.t1, b.t2) a.record(b.t1, b.t2)
return@BiFunction a return@BiFunction a
} }

View File

@@ -17,11 +17,8 @@ package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.JacksonRpcConverter import io.infinitape.etherjar.rpc.JacksonRpcConverter
import io.infinitape.etherjar.rpc.RpcException import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
open class NonEmptyQuorum( open class NonEmptyQuorum(
jacksonRpcConverter: JacksonRpcConverter, jacksonRpcConverter: JacksonRpcConverter,
@@ -31,14 +28,14 @@ open class NonEmptyQuorum(
private var result: ByteArray? = null private var result: ByteArray? = null
private var tries: Int = 0 private var tries: Int = 0
override fun init(head: Head<BlockJson<TransactionRefJson>>) { override fun init(head: Head) {
} }
override fun isResolved(): Boolean { override fun isResolved(): Boolean {
return result != null || tries >= maxTries return result != null || tries >= maxTries
} }
override fun recordValue(response: ByteArray, responseValue: Any?, upstream: Upstream<*, *>) { override fun recordValue(response: ByteArray, responseValue: Any?, upstream: Upstream<*>) {
tries++ tries++
if (responseValue != null) { if (responseValue != null) {
result = response result = response
@@ -49,10 +46,10 @@ open class NonEmptyQuorum(
return result return result
} }
override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) { override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) {
} }
override fun record(error: RpcException, upstream: Upstream<*, *>) { override fun record(error: RpcException, upstream: Upstream<*>) {
} }
} }

View File

@@ -17,12 +17,9 @@ package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.hex.HexQuantity import io.infinitape.etherjar.hex.HexQuantity
import io.infinitape.etherjar.rpc.JacksonRpcConverter import io.infinitape.etherjar.rpc.JacksonRpcConverter
import io.infinitape.etherjar.rpc.RpcException import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock import kotlin.concurrent.withLock
@@ -37,7 +34,7 @@ open class NonceQuorum(
private var receivedTimes = 0 private var receivedTimes = 0
private var errors = 0 private var errors = 0
override fun init(head: Head<BlockJson<TransactionRefJson>>) { override fun init(head: Head) {
} }
override fun isResolved(): Boolean { override fun isResolved(): Boolean {
@@ -46,7 +43,7 @@ open class NonceQuorum(
} }
} }
override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*, *>) { override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*>) {
val value = responseValue?.let { str -> val value = responseValue?.let { str ->
HexQuantity.from(str).value.toLong() HexQuantity.from(str).value.toLong()
} }
@@ -65,11 +62,11 @@ open class NonceQuorum(
return result return result
} }
override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) { override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) {
errors++ errors++
} }
override fun record(error: RpcException, upstream: Upstream<*, *>) { override fun record(error: RpcException, upstream: Upstream<*>) {
errors++ errors++
} }

View File

@@ -17,24 +17,21 @@ package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.RpcException import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum { class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
private val result: AtomicReference<ByteArray> = AtomicReference() private val result: AtomicReference<ByteArray> = AtomicReference()
override fun init(head: Head<BlockJson<TransactionRefJson>>) { override fun init(head: Head) {
} }
override fun isResolved(): Boolean { override fun isResolved(): Boolean {
return result.get() != null return result.get() != null
} }
override fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean { override fun record(response: ByteArray, upstream: Upstream<*>): Boolean {
val lagging = upstream.getLag() > maxLag val lagging = upstream.getLag() > maxLag
if (!lagging) { if (!lagging) {
result.set(response) result.set(response)
@@ -43,7 +40,7 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
return false return false
} }
override fun record(error: RpcException, upstream: Upstream<*, *>) { override fun record(error: RpcException, upstream: Upstream<*>) {
} }

View File

@@ -31,7 +31,7 @@ abstract class ValueAwareQuorum<T>(
return jacksonRpcConverter.fromJson(response.inputStream(), clazz) return jacksonRpcConverter.fromJson(response.inputStream(), clazz)
} }
override fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean { override fun record(response: ByteArray, upstream: Upstream<*>): Boolean {
try { try {
val value = extractValue(response, clazz) val value = extractValue(response, clazz)
recordValue(response, value, upstream) recordValue(response, value, upstream)
@@ -43,12 +43,12 @@ abstract class ValueAwareQuorum<T>(
return isResolved(); return isResolved();
} }
override fun record(error: RpcException, upstream: Upstream<*, *>) { override fun record(error: RpcException, upstream: Upstream<*>) {
recordError(null, error.rpcMessage, upstream) recordError(null, error.rpcMessage, upstream)
} }
abstract fun recordValue(response: ByteArray, responseValue: T?, upstream: Upstream<*, *>) abstract fun recordValue(response: ByteArray, responseValue: T?, upstream: Upstream<*>)
abstract fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) abstract fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>)
} }

View File

@@ -95,10 +95,10 @@ open class NativeCall(
val upstream = upstreams.getUpstream(chain) val upstream = upstreams.getUpstream(chain)
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain))) ?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
return prepareCall(request, upstream as AggregatedUpstream<EthereumApi, BlockJson<TransactionRefJson>>) return prepareCall(request, upstream as AggregatedUpstream<EthereumApi>)
} }
fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: AggregatedUpstream<EthereumApi, BlockJson<TransactionRefJson>>): Flux<CallContext<RawCallDetails>> { fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: AggregatedUpstream<EthereumApi>): Flux<CallContext<RawCallDetails>> {
return request.itemsList.toFlux().map { return request.itemsList.toFlux().map {
val method = it.method val method = it.method
val params = it.payload.toStringUtf8() val params = it.payload.toStringUtf8()
@@ -205,7 +205,7 @@ open class NativeCall(
} }
open class CallContext<T>(val id: Int, open class CallContext<T>(val id: Int,
val upstream: AggregatedUpstream<EthereumApi, BlockJson<TransactionRefJson>>, val upstream: AggregatedUpstream<EthereumApi>,
val matcher: Selector.Matcher, val matcher: Selector.Matcher,
val callQuorum: CallQuorum, val callQuorum: CallQuorum,
val payload: T) { val payload: T) {

View File

@@ -19,6 +19,7 @@ import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.Upstreams import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.grpc.Chain import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
@@ -51,23 +52,19 @@ class StreamHead(
} }
} }
fun asProto(chain: Chain, block: Any): BlockchainOuterClass.ChainHead { fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
if (BlockchainType.fromBlockchain(chain) == BlockchainType.ETHEREUM) { if (BlockchainType.fromBlockchain(chain) == BlockchainType.ETHEREUM) {
if (BlockJson::class.java.isAssignableFrom(block.javaClass)) { return asEthereumProto(chain, block)
return asEthereumProto(chain, block as BlockJson<TransactionRefJson>)
} else {
throw IllegalArgumentException("Invalid block type: ${block.javaClass}")
}
} }
throw IllegalArgumentException("Unsupported blockchain ${chain}") throw IllegalArgumentException("Unsupported blockchain ${chain}")
} }
fun asEthereumProto(chain: Chain, block: BlockJson<TransactionRefJson>): BlockchainOuterClass.ChainHead { fun asEthereumProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead {
return BlockchainOuterClass.ChainHead.newBuilder() return BlockchainOuterClass.ChainHead.newBuilder()
.setChainValue(chain.id) .setChainValue(chain.id)
.setHeight(block.number) .setHeight(block.height)
.setTimestamp(block.timestamp.toEpochMilli()) .setTimestamp(block.timestamp!!.toEpochMilli())
.setWeight(ByteString.copyFrom(block.totalDifficulty.toByteArray())) .setWeight(ByteString.copyFrom(block.difficulty.toByteArray()))
.setBlockId(block.hash.toHex().substring(2)) .setBlockId(block.hash.toHex().substring(2))
.build() .build()
} }

View File

@@ -45,7 +45,7 @@ class SubscribeStatus(
} }
} }
fun chainStatus(chain: Chain, ups: List<Upstream<*, *>>): BlockchainOuterClass.ChainStatus { fun chainStatus(chain: Chain, ups: List<Upstream<*>>): BlockchainOuterClass.ChainStatus {
val available = ups.map { u -> val available = ups.map { u ->
u.getStatus() u.getStatus()
}.min() ?: UpstreamAvailability.UNAVAILABLE }.min() ?: UpstreamAvailability.UNAVAILABLE
@@ -59,6 +59,6 @@ class SubscribeStatus(
.build() .build()
} }
class ChainSubscription(val chain: Chain, val up: AggregatedUpstream<*, *>, val avail: UpstreamAvailability) class ChainSubscription(val chain: Chain, val up: AggregatedUpstream<*>, val avail: UpstreamAvailability)
} }

View File

@@ -187,7 +187,7 @@ class TrackEthereumAddress(
} }
fun getBalance(addr: SimpleAddress): Mono<Wei> { fun getBalance(addr: SimpleAddress): Mono<Wei> {
val up = upstreams.getUpstream(addr.chain) as AggregatedUpstream<EthereumApi, BlockJson<TransactionRefJson>>? val up = upstreams.getUpstream(addr.chain) as AggregatedUpstream<EthereumApi>?
?: return Mono.error(SilentException.UnsupportedBlockchain(addr.chain)) ?: return Mono.error(SilentException.UnsupportedBlockchain(addr.chain))
return up.getApi(Selector.empty) return up.getApi(Selector.empty)
.flatMap { api -> api.executeAndConvert(Commands.eth().getBalance(addr.address, BlockTag.LATEST)) } .flatMap { api -> api.executeAndConvert(Commands.eth().getBalance(addr.address, BlockTag.LATEST)) }

View File

@@ -213,7 +213,7 @@ class TrackEthereumTx(
} }
private fun loadWeight(tx: TxDetails): Mono<TxDetails> { private fun loadWeight(tx: TxDetails): Mono<TxDetails> {
val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream<EthereumApi, BlockJson<TransactionRefJson>>? val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream<EthereumApi>?
?: return Mono.error(SilentException.UnsupportedBlockchain(tx.chain)) ?: return Mono.error(SilentException.UnsupportedBlockchain(tx.chain))
return upstream.getApi(Selector.empty) return upstream.getApi(Selector.empty)
.flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(tx.status.blockHash)) } .flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(tx.status.blockHash)) }
@@ -224,7 +224,7 @@ class TrackEthereumTx(
} }
} }
fun updateFromBlock(upstream: Upstream<EthereumApi, BlockJson<TransactionRefJson>>, tx: TxDetails, it: TransactionJson): Mono<TxDetails> { fun updateFromBlock(upstream: Upstream<EthereumApi>, tx: TxDetails, it: TransactionJson): Mono<TxDetails> {
return if (it.blockNumber != null && it.blockHash != null && it.blockHash != ZERO_BLOCK) { return if (it.blockNumber != null && it.blockHash != null && it.blockHash != ZERO_BLOCK) {
val updated = tx.withStatus( val updated = tx.withStatus(
blockHash = it.blockHash, blockHash = it.blockHash,
@@ -235,11 +235,11 @@ class TrackEthereumTx(
) )
upstream.getHead().getFlux().next().map { head -> upstream.getHead().getFlux().next().map { head ->
val height = updated.status.height val height = updated.status.height
if (height == null || head.number < height) { if (height == null || head.height < height) {
updated updated
} else { } else {
updated.withStatus( updated.withStatus(
confirmations = head.number - height + 1 confirmations = head.height - height + 1
) )
} }
}.doOnError { t -> }.doOnError { t ->
@@ -255,7 +255,7 @@ class TrackEthereumTx(
private fun checkForUpdate(tx: TxDetails): Mono<TxDetails> { private fun checkForUpdate(tx: TxDetails): Mono<TxDetails> {
val initialStatus = tx.status val initialStatus = tx.status
val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream<EthereumApi, BlockJson<TransactionRefJson>>? val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream<EthereumApi>?
?: return Mono.error(SilentException.UnsupportedBlockchain(tx.chain)) ?: return Mono.error(SilentException.UnsupportedBlockchain(tx.chain))
val execution = upstream.getApi(Selector.empty) val execution = upstream.getApi(Selector.empty)
.flatMap { api -> api.executeAndConvert(Commands.eth().getTransaction(tx.txid)) } .flatMap { api -> api.executeAndConvert(Commands.eth().getTransaction(tx.txid)) }

View File

@@ -144,7 +144,8 @@ open class ConfiguredUpstreams(
val wsApi = EthereumWs( val wsApi = EthereumWs(
endpoint.url, endpoint.url,
endpoint.origin ?: URI("http://localhost"), endpoint.origin ?: URI("http://localhost"),
rpcApi!! rpcApi!!,
objectMapper
) )
endpoint.basicAuth?.let { auth -> endpoint.basicAuth?.let { auth ->
wsApi.basicAuth = auth wsApi.basicAuth = auth
@@ -159,7 +160,8 @@ open class ConfiguredUpstreams(
config.id!!, config.id!!,
chain, rpcApi!!, wsApi, options, chain, rpcApi!!, wsApi, options,
QuorumForLabels.QuorumItem(1, config.labels), QuorumForLabels.QuorumItem(1, config.labels),
methods) methods,
objectMapper)
ethereumUpstream.start() ethereumUpstream.start()
currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED)) currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED))
} }

View File

@@ -31,7 +31,7 @@ class UpstreamChange(
/** /**
* Corresponding upstream * Corresponding upstream
*/ */
val upstream: Upstream<*, *>, val upstream: Upstream<*>,
/** /**
* Type of the change * Type of the change
*/ */

View File

@@ -34,18 +34,18 @@ import kotlin.concurrent.withLock
/** /**
* Aggregation of multiple upstreams responding to a single blockchain * Aggregation of multiple upstreams responding to a single blockchain
*/ */
abstract class AggregatedUpstream<U : UpstreamApi, B>( abstract class AggregatedUpstream<U : UpstreamApi>(
private val objectMapper: ObjectMapper, private val objectMapper: ObjectMapper,
val caches: Caches val caches: Caches
) : Upstream<U, B>, Lifecycle { ) : Upstream<U>, Lifecycle {
private var cacheSubscription: Disposable? = null private var cacheSubscription: Disposable? = null
var cache: CachingEthereumApi = CachingEthereumApi.empty() var cache: CachingEthereumApi = CachingEthereumApi.empty(objectMapper)
private val reconfigLock = ReentrantLock() private val reconfigLock = ReentrantLock()
private var callMethods: CallMethods? = null private var callMethods: CallMethods? = null
abstract fun getAll(): List<Upstream<U, B>> abstract fun getAll(): List<Upstream<U>>
abstract fun addUpstream(upstream: Upstream<U, B>) abstract fun addUpstream(upstream: Upstream<U>)
abstract fun getApis(matcher: Selector.Matcher): ApiSource<U> abstract fun getApis(matcher: Selector.Matcher): ApiSource<U>
fun onUpstreamsUpdated() { fun onUpstreamsUpdated() {
@@ -101,12 +101,12 @@ abstract class AggregatedUpstream<U : UpstreamApi, B>(
// -------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------
class UpstreamStatus<H>(val upstream: Upstream<UpstreamApi, H>, val status: UpstreamAvailability, val ts: Instant = Instant.now()) class UpstreamStatus(val upstream: Upstream<UpstreamApi>, val status: UpstreamAvailability, val ts: Instant = Instant.now())
class FilterBestAvailability() : Predicate<UpstreamStatus<*>> { class FilterBestAvailability() : Predicate<UpstreamStatus> {
private val lastRef = AtomicReference<UpstreamStatus<*>>() private val lastRef = AtomicReference<UpstreamStatus>()
override fun test(t: UpstreamStatus<*>): Boolean { override fun test(t: UpstreamStatus): Boolean {
val last = lastRef.get() val last = lastRef.get()
val changed = last == null val changed = last == null
|| t.status > last.status || t.status > last.status

View File

@@ -17,13 +17,11 @@ package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.data.*
import io.emeraldpay.dshackle.upstream.ethereum.EmptyEthereumHead import io.emeraldpay.dshackle.upstream.ethereum.EmptyEthereumHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumApi import io.emeraldpay.dshackle.upstream.ethereum.EthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.hex.HexQuantity import io.infinitape.etherjar.hex.HexQuantity
import io.infinitape.etherjar.rpc.json.ResponseJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.math.BigInteger import java.math.BigInteger
@@ -42,11 +40,13 @@ open class CachingEthereumApi(
* Create caching API with empty memory-only cache * Create caching API with empty memory-only cache
*/ */
@JvmStatic @JvmStatic
fun empty(): CachingEthereumApi { fun empty(objectMapper: ObjectMapper): CachingEthereumApi {
return CachingEthereumApi(ObjectMapper(), Caches.default(), EmptyEthereumHead()) return CachingEthereumApi(objectMapper, Caches.default(objectMapper), EmptyEthereumHead())
} }
} }
private val rawJsonBuilder = RawJsonBuilder()
private val cacheBlocks = caches.getBlocksByHash() private val cacheBlocks = caches.getBlocksByHash()
private val cacheBlocksByHeight = caches.getBlocksByHeight() private val cacheBlocksByHeight = caches.getBlocksByHeight()
private val cacheTx = caches.getTxByHash() private val cacheTx = caches.getTxByHash()
@@ -62,7 +62,7 @@ open class CachingEthereumApi(
cacheBlocks cacheBlocks
} }
Mono.just(params[0]) Mono.just(params[0])
.map { BlockHash.from(it as String) } .map { BlockId.from(it as String) }
.flatMap(cache::read) .flatMap(cache::read)
.transform(converter(id)) .transform(converter(id))
.transform(finalizer()) .transform(finalizer())
@@ -93,14 +93,15 @@ open class CachingEthereumApi(
return when (method) { return when (method) {
"eth_blockNumber" -> "eth_blockNumber" ->
head.getFlux().next() head.getFlux().next()
.map { HexQuantity.from(it.number).toHex() } .map { HexQuantity.from(it.height).toHex() }
.map(toJson(id)) .map { objectMapper.writeValueAsBytes(it) }
.map(bytesToJson(id))
"eth_getBlockByHash" -> readBlockByHash(id, method, params) "eth_getBlockByHash" -> readBlockByHash(id, method, params)
"eth_getBlockByNumber" -> readBlockByNumber(id, method, params) "eth_getBlockByNumber" -> readBlockByNumber(id, method, params)
"eth_getTransactionByHash" -> "eth_getTransactionByHash" ->
if (params.size == 1) if (params.size == 1)
Mono.just(params[0]) Mono.just(params[0])
.map { TransactionId.from(it as String) } .map { TxId.from(it as String) }
.flatMap(cacheTx::read) .flatMap(cacheTx::read)
.transform(converter(id)) .transform(converter(id))
.transform(finalizer()) .transform(finalizer())
@@ -113,9 +114,9 @@ open class CachingEthereumApi(
/** /**
* Convert to JSON RPC response * Convert to JSON RPC response
*/ */
fun converter(id: Int): Function<in Mono<*>, out Mono<ByteArray>> { fun converter(id: Int): Function<in Mono<out SourceContainer>, out Mono<ByteArray>> {
return Function { mono -> return Function { mono ->
mono.map(toJson(id)) mono.map(containerToJson(id))
} }
} }
@@ -131,12 +132,15 @@ open class CachingEthereumApi(
} }
} }
fun toJson(id: Int): Function<Any, ByteArray> { fun bytesToJson(id: Int): Function<ByteArray, ByteArray> {
return Function { data -> return Function { data ->
val resp = ResponseJson<Any, Int>() rawJsonBuilder.write(id, data)
resp.id = id }
resp.result = data }
objectMapper.writer().writeValueAsBytes(resp)
fun containerToJson(id: Int): Function<SourceContainer, ByteArray> {
return Function { data ->
rawJsonBuilder.write(id, data.json!!)
} }
} }
} }

View File

@@ -26,24 +26,24 @@ import reactor.core.publisher.Mono
/** /**
* General interface to upstream(s) to a single chain * General interface to upstream(s) to a single chain
*/ */
abstract class ChainUpstreams<U : UpstreamApi, B>( abstract class ChainUpstreams<U : UpstreamApi>(
val chain: Chain, val chain: Chain,
private val upstreams: MutableList<Upstream<U, B>>, private val upstreams: MutableList<Upstream<U>>,
caches: Caches, caches: Caches,
objectMapper: ObjectMapper objectMapper: ObjectMapper
) : AggregatedUpstream<U, B>(objectMapper, caches), Lifecycle { ) : AggregatedUpstream<U>(objectMapper, caches), Lifecycle {
private val log = LoggerFactory.getLogger(ChainUpstreams::class.java) private val log = LoggerFactory.getLogger(ChainUpstreams::class.java)
private var seq = 0 private var seq = 0
protected var lagObserver: HeadLagObserver<U, B>? = null protected var lagObserver: HeadLagObserver<U>? = null
private var subscription: Disposable? = null private var subscription: Disposable? = null
open fun init() { open fun init() {
onUpstreamsUpdated() onUpstreamsUpdated()
} }
abstract fun updateHead(): Head<B> abstract fun updateHead(): Head
abstract fun setHead(head: Head<B>) abstract fun setHead(head: Head)
override fun getId(): String { override fun getId(): String {
return "!all:${chain.chainCode}" return "!all:${chain.chainCode}"
@@ -72,11 +72,11 @@ abstract class ChainUpstreams<U : UpstreamApi, B>(
lagObserver?.stop() lagObserver?.stop()
} }
override fun getAll(): List<Upstream<U, B>> { override fun getAll(): List<Upstream<U>> {
return upstreams return upstreams
} }
override fun addUpstream(upstream: Upstream<U, B>) { override fun addUpstream(upstream: Upstream<U>) {
upstreams.add(upstream) upstreams.add(upstream)
setHead(updateHead()) setHead(updateHead())
onUpstreamsUpdated() onUpstreamsUpdated()

View File

@@ -46,7 +46,7 @@ class CurrentUpstreams(
private val log = LoggerFactory.getLogger(CurrentUpstreams::class.java) private val log = LoggerFactory.getLogger(CurrentUpstreams::class.java)
private val chainMapping = ConcurrentHashMap<Chain, ChainUpstreams<*, *>>() private val chainMapping = ConcurrentHashMap<Chain, ChainUpstreams<*>>()
private val chainsBus = TopicProcessor.create<Chain>() private val chainsBus = TopicProcessor.create<Chain>()
private val callTargets = HashMap<Chain, CallMethods>() private val callTargets = HashMap<Chain, CallMethods>()
private val updateLock = ReentrantLock() private val updateLock = ReentrantLock()
@@ -55,8 +55,8 @@ class CurrentUpstreams(
updateLock.withLock { updateLock.withLock {
val chain = change.chain val chain = change.chain
val up = change.upstream val up = change.upstream
.cast(EthereumUpstream::class.java, EthereumApi::class.java, BlockJson::class.java) as Upstream<EthereumApi, BlockJson<TransactionRefJson>> .cast(EthereumUpstream::class.java, EthereumApi::class.java) as Upstream<EthereumApi>
val current = chainMapping[chain] as ChainUpstreams<EthereumApi, BlockJson<TransactionRefJson>>? val current = chainMapping[chain] as ChainUpstreams<EthereumApi>?
if (change.type == UpstreamChange.ChangeType.REMOVED) { if (change.type == UpstreamChange.ChangeType.REMOVED) {
current?.removeUpstream(up.getId()) current?.removeUpstream(up.getId())
log.info("Upstream ${change.upstream.getId()} with chain $chain has been removed") log.info("Upstream ${change.upstream.getId()} with chain $chain has been removed")
@@ -84,7 +84,7 @@ class CurrentUpstreams(
} }
} }
override fun getUpstream(chain: Chain): AggregatedUpstream<*, *>? { override fun getUpstream(chain: Chain): AggregatedUpstream<*>? {
return chainMapping[chain] return chainMapping[chain]
} }

View File

@@ -19,10 +19,10 @@ import reactor.core.publisher.Flux
import reactor.core.publisher.TopicProcessor import reactor.core.publisher.TopicProcessor
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
abstract class DefaultUpstream<U : UpstreamApi, B>( abstract class DefaultUpstream<U : UpstreamApi>(
defaultLag: Long, defaultLag: Long,
defaultAvail: UpstreamAvailability defaultAvail: UpstreamAvailability
) : Upstream<U, B> { ) : Upstream<U> {
constructor() : this(Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE) constructor() : this(Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE)

View File

@@ -26,7 +26,7 @@ import kotlin.math.roundToLong
import kotlin.random.Random import kotlin.random.Random
class FilteredApis<U : UpstreamApi>( class FilteredApis<U : UpstreamApi>(
allUpstreams: List<Upstream<U, *>>, allUpstreams: List<Upstream<U>>,
private val matcher: Selector.Matcher, private val matcher: Selector.Matcher,
pos: Int, pos: Int,
private val repeatLimit: Long, private val repeatLimit: Long,
@@ -38,15 +38,15 @@ class FilteredApis<U : UpstreamApi>(
private const val MAX_WAIT_MILLIS = 5000L private const val MAX_WAIT_MILLIS = 5000L
} }
constructor(allUpstreams: List<Upstream<U, *>>, constructor(allUpstreams: List<Upstream<U>>,
matcher: Selector.Matcher, matcher: Selector.Matcher,
pos: Int) : this(allUpstreams, matcher, pos, 10, 7) pos: Int) : this(allUpstreams, matcher, pos, 10, 7)
constructor(allUpstreams: List<Upstream<U, *>>, constructor(allUpstreams: List<Upstream<U>>,
matcher: Selector.Matcher) : this(allUpstreams, matcher, 0, 10, 10) matcher: Selector.Matcher) : this(allUpstreams, matcher, 0, 10, 10)
private val delay: Int private val delay: Int
private val upstreams: List<Upstream<UpstreamApi, *>> private val upstreams: List<Upstream<UpstreamApi>>
private val control = EmitterProcessor.create<Boolean>(32, false) private val control = EmitterProcessor.create<Boolean>(32, false)
@@ -81,7 +81,7 @@ class FilteredApis<U : UpstreamApi>(
}.let { Flux.concat(it) } }.let { Flux.concat(it) }
Flux.concat(first, retries) Flux.concat(first, retries)
.filter(Upstream<UpstreamApi, *>::isAvailable) .filter(Upstream<UpstreamApi>::isAvailable)
.filter(matcher::matches) .filter(matcher::matches)
.flatMap { it.getApi(matcher) } .flatMap { it.getApi(matcher) }
.zipWith(control).map { it.t1 } .zipWith(control).map { it.t1 }

View File

@@ -15,9 +15,9 @@
*/ */
package io.emeraldpay.dshackle.upstream package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.data.BlockContainer
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
interface Head<out T> { interface Head {
fun getFlux(): Flux<out T> fun getFlux(): Flux<BlockContainer>
} }

View File

@@ -15,6 +15,7 @@
*/ */
package io.emeraldpay.dshackle.upstream package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.data.BlockContainer
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle import org.springframework.context.Lifecycle
import reactor.core.Disposable import reactor.core.Disposable
@@ -26,9 +27,9 @@ import reactor.util.function.Tuples
* Observer group of upstreams and defined a distance in blocks (lag) between a leader (best height/difficulty) and * Observer group of upstreams and defined a distance in blocks (lag) between a leader (best height/difficulty) and
* other upstreams. * other upstreams.
*/ */
abstract class HeadLagObserver<A : UpstreamApi, B>( abstract class HeadLagObserver<A : UpstreamApi>(
private val master: Head<B>, private val master: Head,
private val followers: Collection<Upstream<A, B>> private val followers: Collection<Upstream<A>>
) : Lifecycle { ) : Lifecycle {
private val log = LoggerFactory.getLogger(HeadLagObserver::class.java) private val log = LoggerFactory.getLogger(HeadLagObserver::class.java)
@@ -56,7 +57,7 @@ abstract class HeadLagObserver<A : UpstreamApi, B>(
} }
} }
fun probeFollowers(top: B): Flux<Tuple2<Long, Upstream<A, B>>> { fun probeFollowers(top: BlockContainer): Flux<Tuple2<Long, Upstream<A>>> {
return Flux.fromIterable(followers) return Flux.fromIterable(followers)
.parallel(followers.size) .parallel(followers.size)
.flatMap { mapLagging(top, it, getCurrentBlocks(it)) } .flatMap { mapLagging(top, it, getCurrentBlocks(it)) }
@@ -64,9 +65,9 @@ abstract class HeadLagObserver<A : UpstreamApi, B>(
.onErrorContinue { t, _ -> log.warn("Failed to update lagging distance", t) } .onErrorContinue { t, _ -> log.warn("Failed to update lagging distance", t) }
} }
abstract fun getCurrentBlocks(up: Upstream<A, B>): Flux<B> abstract fun getCurrentBlocks(up: Upstream<A>): Flux<BlockContainer>
fun mapLagging(top: B, up: Upstream<A, B>, blocks: Flux<B>): Flux<Tuple2<Long, Upstream<A, B>>> { fun mapLagging(top: BlockContainer, up: Upstream<A>, blocks: Flux<BlockContainer>): Flux<Tuple2<Long, Upstream<A>>> {
return blocks return blocks
.map { extractDistance(top, it) } .map { extractDistance(top, it) }
.takeUntil { lag -> lag <= 0L } .takeUntil { lag -> lag <= 0L }
@@ -76,9 +77,9 @@ abstract class HeadLagObserver<A : UpstreamApi, B>(
} }
} }
abstract fun extractDistance(top: B, curr: B): Long abstract fun extractDistance(top: BlockContainer, curr: BlockContainer): Long
fun forkDistance(top: B, curr: B): Long { fun forkDistance(top: BlockContainer, curr: BlockContainer): Long {
//TODO look for common ancestor? though it may be a corruption //TODO look for common ancestor? though it may be a corruption
return 6 return 6
} }

View File

@@ -95,13 +95,13 @@ class Selector {
} }
interface Matcher { interface Matcher {
fun matches(up: Upstream<UpstreamApi, *>): Boolean fun matches(up: Upstream<UpstreamApi>): Boolean
} }
class MultiMatcher( class MultiMatcher(
private val matchers: Collection<Matcher> private val matchers: Collection<Matcher>
): Matcher { ): Matcher {
override fun matches(up: Upstream<UpstreamApi, *>): Boolean { override fun matches(up: Upstream<UpstreamApi>): Boolean {
return matchers.all { it.matches(up) } return matchers.all { it.matches(up) }
} }
@@ -113,13 +113,13 @@ class Selector {
class MethodMatcher( class MethodMatcher(
val method: String val method: String
): Matcher { ): Matcher {
override fun matches(up: Upstream<UpstreamApi, *>): Boolean { override fun matches(up: Upstream<UpstreamApi>): Boolean {
return up.getMethods().isAllowed(method) return up.getMethods().isAllowed(method)
} }
} }
abstract class LabelSelectorMatcher: Matcher { abstract class LabelSelectorMatcher: Matcher {
override fun matches(up: Upstream<UpstreamApi, *>): Boolean { override fun matches(up: Upstream<UpstreamApi>): Boolean {
return up.getLabels().any(this::matches) return up.getLabels().any(this::matches)
} }
@@ -128,7 +128,7 @@ class Selector {
} }
class EmptyMatcher: Matcher { class EmptyMatcher: Matcher {
override fun matches(up: Upstream<UpstreamApi, *>): Boolean { override fun matches(up: Upstream<UpstreamApi>): Boolean {
return true return true
} }
} }
@@ -143,7 +143,7 @@ class Selector {
return null return null
} }
override fun matches(up: Upstream<UpstreamApi, *>): Boolean { override fun matches(up: Upstream<UpstreamApi>): Boolean {
return true return true
} }
} }

View File

@@ -17,16 +17,14 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
interface Upstream<out T : UpstreamApi, out B> { interface Upstream<out T : UpstreamApi> {
fun isAvailable(): Boolean fun isAvailable(): Boolean
fun getStatus(): UpstreamAvailability fun getStatus(): UpstreamAvailability
fun observeStatus(): Flux<UpstreamAvailability> fun observeStatus(): Flux<UpstreamAvailability>
fun getHead(): Head<B> fun getHead(): Head
fun getApi(matcher: Selector.Matcher): Mono<out T> fun getApi(matcher: Selector.Matcher): Mono<out T>
fun getOptions(): UpstreamsConfig.Options fun getOptions(): UpstreamsConfig.Options
fun setLag(lag: Long) fun setLag(lag: Long)
@@ -35,5 +33,5 @@ interface Upstream<out T : UpstreamApi, out B> {
fun getMethods(): CallMethods fun getMethods(): CallMethods
fun getId(): String fun getId(): String
fun <T : Upstream<TA, BA>, TA : UpstreamApi, BA> cast(selfType: Class<T>, upstreamType: Class<TA>, blockType: Class<BA>): T fun <T : Upstream<TA>, TA : UpstreamApi> cast(selfType: Class<T>, upstreamType: Class<TA>): T
} }

View File

@@ -20,7 +20,7 @@ import io.emeraldpay.grpc.Chain
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
interface Upstreams { interface Upstreams {
fun getUpstream(chain: Chain): AggregatedUpstream<*, *>? fun getUpstream(chain: Chain): AggregatedUpstream<*>?
fun getAvailable(): List<Chain> fun getAvailable(): List<Chain>
fun observeChains(): Flux<Chain> fun observeChains(): Flux<Chain>
fun getDefaultMethods(chain: Chain): CallMethods fun getDefaultMethods(chain: Chain): CallMethods

View File

@@ -1,8 +1,6 @@
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.infinitape.etherjar.domain.TransactionId import io.emeraldpay.dshackle.data.BlockContainer
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.Disposable import reactor.core.Disposable
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
@@ -13,39 +11,39 @@ import java.util.concurrent.atomic.AtomicReference
open class DefaultEthereumHead: EthereumHead { open class DefaultEthereumHead: EthereumHead {
private val log = LoggerFactory.getLogger(DefaultEthereumHead::class.java) private val log = LoggerFactory.getLogger(DefaultEthereumHead::class.java)
private val head = AtomicReference<BlockJson<TransactionRefJson>>(null) private val head = AtomicReference<BlockContainer>(null)
private val stream: TopicProcessor<BlockJson<TransactionRefJson>> = TopicProcessor.create() private val stream: TopicProcessor<BlockContainer> = TopicProcessor.create()
fun follow(source: Flux<BlockJson<TransactionRefJson>>): Disposable { fun follow(source: Flux<BlockContainer>): Disposable {
return source.distinctUntilChanged { return source.distinctUntilChanged {
it.hash it.hash
}.filter { block -> }.filter { block ->
val curr = head.get() val curr = head.get()
curr == null || curr.totalDifficulty < block.totalDifficulty curr == null || curr.difficulty < block.difficulty
} }
.subscribe { block -> .subscribe { block ->
val prev = head.getAndUpdate { curr -> val prev = head.getAndUpdate { curr ->
if (curr == null || curr.totalDifficulty < block.totalDifficulty) { if (curr == null || curr.difficulty < block.difficulty) {
block block
} else { } else {
curr curr
} }
} }
if (prev == null || prev.hash != block.hash) { if (prev == null || prev.hash != block.hash) {
log.debug("New block ${block.number} ${block.hash}") log.debug("New block ${block.height} ${block.hash}")
stream.onNext(block) stream.onNext(block)
} }
} }
} }
override fun getFlux(): Flux<BlockJson<TransactionRefJson>> { override fun getFlux(): Flux<BlockContainer> {
return Flux.merge( return Flux.merge(
Mono.justOrEmpty(head.get()), Mono.justOrEmpty(head.get()),
Flux.from(stream) Flux.from(stream)
).onBackpressureLatest() ).onBackpressureLatest()
} }
fun getCurrent(): BlockJson<TransactionRefJson>? { fun getCurrent(): BlockContainer? {
return head.get() return head.get()
} }
} }

View File

@@ -15,14 +15,12 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.infinitape.etherjar.domain.TransactionId import io.emeraldpay.dshackle.data.BlockContainer
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
class EmptyEthereumHead : EthereumHead { class EmptyEthereumHead : EthereumHead {
override fun getFlux(): Flux<BlockJson<TransactionRefJson>> { override fun getFlux(): Flux<BlockContainer> {
return Flux.empty() return Flux.empty()
} }
} }

View File

@@ -34,7 +34,7 @@ abstract class EthereumApi(
} }
private val jacksonRpcConverter = JacksonRpcConverter(objectMapper) private val jacksonRpcConverter = JacksonRpcConverter(objectMapper)
var upstream: Upstream<EthereumApi, BlockJson<TransactionRefJson>>? = null var upstream: Upstream<EthereumApi>? = null
fun <JS, RS> execute(rpcCall: RpcCall<JS, RS>): Mono<ByteArray> { fun <JS, RS> execute(rpcCall: RpcCall<JS, RS>): Mono<ByteArray> {
return execute(0, rpcCall.method, rpcCall.params as List<Any>) return execute(0, rpcCall.method, rpcCall.params as List<Any>)

View File

@@ -33,7 +33,7 @@ class EthereumChainUpstreams(
val upstreams: MutableList<EthereumUpstream>, val upstreams: MutableList<EthereumUpstream>,
caches: Caches, caches: Caches,
objectMapper: ObjectMapper objectMapper: ObjectMapper
) : ChainUpstreams<EthereumApi, BlockJson<TransactionRefJson>>(chain, upstreams as MutableList<Upstream<EthereumApi, BlockJson<TransactionRefJson>>>, caches, objectMapper) { ) : ChainUpstreams<EthereumApi>(chain, upstreams as MutableList<Upstream<EthereumApi>>, caches, objectMapper) {
companion object { companion object {
private val log = LoggerFactory.getLogger(EthereumChainUpstreams::class.java) private val log = LoggerFactory.getLogger(EthereumChainUpstreams::class.java)
@@ -56,7 +56,7 @@ class EthereumChainUpstreams(
return head!! return head!!
} }
override fun setHead(head: Head<BlockJson<TransactionRefJson>>) { override fun setHead(head: Head) {
this.head = head as EthereumHead this.head = head as EthereumHead
} }
@@ -76,7 +76,7 @@ class EthereumChainUpstreams(
val newHead = EthereumHeadMerge(upstreams.map { it.getHead() }).apply { val newHead = EthereumHeadMerge(upstreams.map { it.getHead() }).apply {
this.start() this.start()
} }
val lagObserver = EthereumHeadLagObserver(newHead, upstreams).apply { val lagObserver = EthereumHeadLagObserver(newHead, upstreams as Collection<Upstream<EthereumApi>>).apply {
this.start() this.start()
} }
this.lagObserver = lagObserver this.lagObserver = lagObserver
@@ -93,7 +93,7 @@ class EthereumChainUpstreams(
override fun printStatus() { override fun printStatus() {
var height: Long? = null var height: Long? = null
try { try {
height = getHead().getFlux().next().block(Duration.ofSeconds(1))?.number height = getHead().getFlux().next().block(Duration.ofSeconds(1))?.height
} catch (e: IllegalStateException) { } catch (e: IllegalStateException) {
//timout //timout
} catch (e: Exception) { } catch (e: Exception) {
@@ -110,18 +110,14 @@ class EthereumChainUpstreams(
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
override fun <T : Upstream<TA, BA>, TA : UpstreamApi, BA> cast(selfType: Class<T>, upstreamType: Class<TA>, blockType: Class<BA>): T { override fun <T : Upstream<TA>, TA : UpstreamApi> cast(selfType: Class<T>, upstreamType: Class<TA>): T {
if (!selfType.isAssignableFrom(this.javaClass)) { if (!selfType.isAssignableFrom(this.javaClass)) {
throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") throw ClassCastException("Cannot cast ${this.javaClass} to $selfType")
} }
if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) { if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) {
throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType") throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType")
} }
if (!blockType.isAssignableFrom(BlockJson::class.java)) {
throw ClassCastException("Cannot cast ${BlockJson::class.java} to $blockType")
}
return this as T return this as T
} }
} }

View File

@@ -20,5 +20,5 @@ import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
interface EthereumHead: Head<BlockJson<TransactionRefJson>> { interface EthereumHead : Head {
} }

View File

@@ -17,31 +17,30 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.HeadLagObserver import io.emeraldpay.dshackle.upstream.HeadLagObserver
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.rpc.json.BlockJson import io.emeraldpay.dshackle.data.BlockContainer
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import java.time.Duration import java.time.Duration
class EthereumHeadLagObserver( class EthereumHeadLagObserver(
master: EthereumHead, master: EthereumHead,
followers: Collection<Upstream<EthereumApi, BlockJson<TransactionRefJson>>> followers: Collection<Upstream<EthereumApi>>
) : HeadLagObserver<EthereumApi, BlockJson<TransactionRefJson>>(master, followers) { ) : HeadLagObserver<EthereumApi>(master, followers) {
companion object { companion object {
private val log = LoggerFactory.getLogger(EthereumHeadLagObserver::class.java) private val log = LoggerFactory.getLogger(EthereumHeadLagObserver::class.java)
} }
override fun getCurrentBlocks(up: Upstream<EthereumApi, BlockJson<TransactionRefJson>>): Flux<BlockJson<TransactionRefJson>> { override fun getCurrentBlocks(up: Upstream<EthereumApi>): Flux<BlockContainer> {
val head = up.getHead() val head = up.getHead()
return Flux.from(head.getFlux()).take(Duration.ofSeconds(1)) return head.getFlux().take(Duration.ofSeconds(1))
} }
override fun extractDistance(top: BlockJson<TransactionRefJson>, curr: BlockJson<TransactionRefJson>): Long { override fun extractDistance(top: BlockContainer, curr: BlockContainer): Long {
return when { return when {
curr.number > top.number -> if (curr.totalDifficulty >= top.totalDifficulty) 0 else forkDistance(top, curr) curr.height > top.height -> if (curr.difficulty >= top.difficulty) 0 else forkDistance(top, curr)
curr.number == top.number -> if (curr.totalDifficulty == top.totalDifficulty) 0 else forkDistance(top, curr) curr.height == top.height -> if (curr.difficulty == top.difficulty) 0 else forkDistance(top, curr)
else -> top.number - curr.number else -> top.height - curr.height
} }
} }
} }

View File

@@ -15,18 +15,10 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.reader.EmptyReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.CachingEthereumApi
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.Batch
import io.infinitape.etherjar.rpc.Commands import io.infinitape.etherjar.rpc.Commands
import io.infinitape.etherjar.rpc.ReactorBatch
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle import org.springframework.context.Lifecycle
import org.springframework.scheduling.concurrent.CustomizableThreadFactory import org.springframework.scheduling.concurrent.CustomizableThreadFactory
@@ -39,6 +31,7 @@ import java.util.concurrent.Executors
class EthereumRpcHead( class EthereumRpcHead(
private val api: DirectEthereumApi, private val api: DirectEthereumApi,
private val objectMapper: ObjectMapper,
private val interval: Duration = Duration.ofSeconds(10) private val interval: Duration = Duration.ofSeconds(10)
): DefaultEthereumHead(), Lifecycle { ): DefaultEthereumHead(), Lifecycle {
@@ -67,6 +60,9 @@ class EthereumRpcHead(
.subscribeOn(scheduler) .subscribeOn(scheduler)
.timeout(Defaults.timeout, Mono.error(Exception("Block data not received"))) .timeout(Defaults.timeout, Mono.error(Exception("Block data not received")))
} }
.map {
BlockContainer.from(it, objectMapper)
}
.onErrorContinue { err, _ -> .onErrorContinue { err, _ ->
log.debug("RPC error ${err.message}") log.debug("RPC error ${err.message}")
} }

View File

@@ -15,6 +15,7 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
@@ -38,12 +39,13 @@ open class EthereumUpstream(
private val ethereumWs: EthereumWs? = null, private val ethereumWs: EthereumWs? = null,
private val options: UpstreamsConfig.Options, private val options: UpstreamsConfig.Options,
val node: QuorumForLabels.QuorumItem, val node: QuorumForLabels.QuorumItem,
private val targets: CallMethods private val targets: CallMethods,
) : DefaultUpstream<EthereumApi, BlockJson<TransactionRefJson>>(), Upstream<EthereumApi, BlockJson<TransactionRefJson>>, CachesEnabled, Lifecycle { private val objectMapper: ObjectMapper
) : DefaultUpstream<EthereumApi>(), Upstream<EthereumApi>, CachesEnabled, Lifecycle {
constructor(id: String, chain: Chain, api: DirectEthereumApi) : this(id, chain, api, null, constructor(id: String, chain: Chain, api: DirectEthereumApi, objectMapper: ObjectMapper) : this(id, chain, api, null,
UpstreamsConfig.Options.getDefaults(), QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()), UpstreamsConfig.Options.getDefaults(), QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()),
DirectCallMethods()) DirectCallMethods(), objectMapper)
private val log = LoggerFactory.getLogger(EthereumUpstream::class.java) private val log = LoggerFactory.getLogger(EthereumUpstream::class.java)
@@ -97,7 +99,7 @@ open class EthereumUpstream(
this.start() this.start()
} }
// receive bew blocks through Websockets, but periodically verify with RPC // receive bew blocks through Websockets, but periodically verify with RPC
val rpc = EthereumRpcHead(api, Duration.ofSeconds(30)).apply { val rpc = EthereumRpcHead(api, objectMapper, Duration.ofSeconds(30)).apply {
this.start() this.start()
} }
EthereumHeadMerge(listOf(rpc, ws)).apply { EthereumHeadMerge(listOf(rpc, ws)).apply {
@@ -105,7 +107,7 @@ open class EthereumUpstream(
} }
} else { } else {
log.warn("Setting up upstream $id with RPC-only access, less effective than WS+RPC") log.warn("Setting up upstream $id with RPC-only access, less effective than WS+RPC")
EthereumRpcHead(api).apply { EthereumRpcHead(api, objectMapper).apply {
this.start() this.start()
} }
} }
@@ -136,16 +138,13 @@ open class EthereumUpstream(
} }
@Suppress("unchecked") @Suppress("unchecked")
override fun <T : Upstream<TA, BA>, TA : UpstreamApi, BA> cast(selfType: Class<T>, upstreamType: Class<TA>, blockType: Class<BA>): T { override fun <T : Upstream<TA>, TA : UpstreamApi> cast(selfType: Class<T>, upstreamType: Class<TA>): T {
if (!selfType.isAssignableFrom(this.javaClass)) { if (!selfType.isAssignableFrom(this.javaClass)) {
throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") throw ClassCastException("Cannot cast ${this.javaClass} to $selfType")
} }
if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) { if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) {
throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType") throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType")
} }
if (!blockType.isAssignableFrom(BlockJson::class.java)) {
throw ClassCastException("Cannot cast ${BlockJson::class.java} to $blockType")
}
return this as T return this as T
} }

View File

@@ -15,14 +15,15 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.AuthConfig import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.reader.EmptyReader
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.Commands import io.infinitape.etherjar.rpc.Commands
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
@@ -38,17 +39,18 @@ import java.time.Duration
class EthereumWs( class EthereumWs(
private val uri: URI, private val uri: URI,
private val origin: URI, private val origin: URI,
private val api: EthereumApi private val api: EthereumApi,
private val objectMapper: ObjectMapper
): CachesEnabled { ): CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumWs::class.java) private val log = LoggerFactory.getLogger(EthereumWs::class.java)
private val topic = TopicProcessor private val topic = TopicProcessor
.builder<BlockJson<TransactionRefJson>>() .builder<BlockContainer>()
.name("new-blocks") .name("new-blocks")
.build() .build()
var basicAuth: AuthConfig.ClientBasicAuth? = null var basicAuth: AuthConfig.ClientBasicAuth? = null
private var blockCache: Reader<BlockHash, BlockJson<TransactionRefJson>> = EmptyReader() private var blockCache: Reader<BlockId, BlockContainer> = EmptyReader()
fun connect() { fun connect() {
log.info("Connecting to WebSocket: $uri") log.info("Connecting to WebSocket: $uri")
@@ -68,11 +70,13 @@ class EthereumWs(
} }
fun onNewBlock(block: BlockJson<TransactionRefJson>) { fun onNewBlock(block: BlockJson<TransactionRefJson>) {
if (block.totalDifficulty == null || block.transactions == null) { // WS returns incomplete blocks
if (block.difficulty == null || block.transactions == null) {
Mono.just(block.hash).flatMap { hash -> Mono.just(block.hash).flatMap { hash ->
val hash = BlockId.from(hash)
// first check in cache, if empty then check api // first check in cache, if empty then check api
blockCache.read(hash) blockCache.read(hash)
.switchIfEmpty(api.executeAndConvert(Commands.eth().getBlock(hash))) .switchIfEmpty(request(hash))
}.repeatWhenEmpty { n -> }.repeatWhenEmpty { n ->
Repeat.times<Any>(10) Repeat.times<Any>(10)
.exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(250)) .exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(250))
@@ -80,12 +84,19 @@ class EthereumWs(
} }
.timeout(Defaults.timeout, Mono.empty()) .timeout(Defaults.timeout, Mono.empty())
.subscribe(topic::onNext) .subscribe(topic::onNext)
} else { } else {
topic.onNext(block) topic.onNext(BlockContainer.from(block, objectMapper))
} }
} }
fun getFlux(): Flux<BlockJson<TransactionRefJson>> { fun request(hash: BlockId): Mono<BlockContainer> {
return api
.executeAndConvert(Commands.eth().getBlock(io.infinitape.etherjar.domain.BlockHash(hash.value)))
.map { BlockContainer.from(it, objectMapper) }
}
fun getFlux(): Flux<BlockContainer> {
return Flux.from(this.topic) return Flux.from(this.topic)
.onBackpressureLatest() .onBackpressureLatest()
} }

View File

@@ -24,6 +24,8 @@ import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.* import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -36,8 +38,6 @@ import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.* import io.infinitape.etherjar.rpc.*
import io.infinitape.etherjar.rpc.emerald.ReactorEmeraldClient import io.infinitape.etherjar.rpc.emerald.ReactorEmeraldClient
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle import org.springframework.context.Lifecycle
import reactor.core.Disposable import reactor.core.Disposable
@@ -46,6 +46,7 @@ import reactor.core.publisher.Mono
import reactor.core.publisher.toMono import reactor.core.publisher.toMono
import java.math.BigInteger import java.math.BigInteger
import java.time.Duration import java.time.Duration
import java.time.Instant
import java.util.* import java.util.*
import java.util.concurrent.TimeoutException import java.util.concurrent.TimeoutException
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
@@ -58,7 +59,7 @@ open class EthereumGrpcUpstream(
private val blockchainStub: ReactorBlockchainGrpc.ReactorBlockchainStub, private val blockchainStub: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val objectMapper: ObjectMapper, private val objectMapper: ObjectMapper,
private val rpcClient: ReactorEmeraldClient private val rpcClient: ReactorEmeraldClient
) : DefaultUpstream<EthereumApi, BlockJson<TransactionRefJson>>(), CachesEnabled, Lifecycle { ) : DefaultUpstream<EthereumApi>(), CachesEnabled, Lifecycle {
private var allLabels: Collection<UpstreamsConfig.Labels> = ArrayList<UpstreamsConfig.Labels>() private var allLabels: Collection<UpstreamsConfig.Labels> = ArrayList<UpstreamsConfig.Labels>()
private val log = LoggerFactory.getLogger(EthereumGrpcUpstream::class.java) private val log = LoggerFactory.getLogger(EthereumGrpcUpstream::class.java)
@@ -117,19 +118,24 @@ open class EthereumGrpcUpstream(
internal fun observeHead(flux: Flux<BlockchainOuterClass.ChainHead>) { internal fun observeHead(flux: Flux<BlockchainOuterClass.ChainHead>) {
val base = flux.map { value -> val base = flux.map { value ->
val block = BlockJson<TransactionRefJson>() val block = BlockContainer(
block.number = value.height value.height,
block.totalDifficulty = BigInteger(1, value.weight.toByteArray()) BlockId.from(BlockHash.from("0x" + value.blockId)),
block.hash = BlockHash.from("0x"+value.blockId) BigInteger(1, value.weight.toByteArray()),
Instant.ofEpochMilli(value.timestamp),
false,
null
)
block block
}.distinctUntilChanged { }.distinctUntilChanged {
it.hash it.hash
}.filter { block -> }.filter { block ->
val curr = head.getCurrent() val curr = head.getCurrent()
curr == null || curr.totalDifficulty < block.totalDifficulty curr == null || curr.difficulty < block.difficulty
}.flatMap { }.flatMap {
getApi(Selector.EmptyMatcher()) getApi(Selector.EmptyMatcher())
.flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(it.hash)) } .flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(BlockHash(it.hash.value))) }
.map { BlockContainer.from(it, objectMapper) }
.timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream")))
.doOnError { t -> .doOnError { t ->
setStatus(UpstreamAvailability.UNAVAILABLE) setStatus(UpstreamAvailability.UNAVAILABLE)
@@ -216,16 +222,13 @@ open class EthereumGrpcUpstream(
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
override fun <T : Upstream<TA, BA>, TA : UpstreamApi, BA> cast(selfType: Class<T>, upstreamType: Class<TA>, blockType: Class<BA>): T { override fun <T : Upstream<TA>, TA : UpstreamApi> cast(selfType: Class<T>, upstreamType: Class<TA>): T {
if (!selfType.isAssignableFrom(this.javaClass)) { if (!selfType.isAssignableFrom(this.javaClass)) {
throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") throw ClassCastException("Cannot cast ${this.javaClass} to $selfType")
} }
if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) { if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) {
throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType") throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType")
} }
if (!blockType.isAssignableFrom(BlockJson::class.java)) {
throw ClassCastException("Cannot cast ${BlockJson::class.java} to $blockType")
}
return this as T return this as T
} }

View File

@@ -1,15 +1,23 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification import spock.lang.Specification
import java.time.Instant
import java.time.temporal.ChronoUnit
class BlockByHeightSpec extends Specification { class BlockByHeightSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33" String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Fetch with all data available"() { def "Fetch with all data available"() {
setup: setup:
def blocks = new BlocksMemCache() def blocks = new BlocksMemCache()
@@ -18,16 +26,22 @@ class BlockByHeightSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.uncles = []
block.transactions = []
blocks.add(block) BlockContainer.from(block, objectMapper).with {
heights.add(block) blocks.add(it)
heights.add(it)
}
def blocksByHeight = new BlockByHeight(heights, blocks) def blocksByHeight = new BlockByHeight(heights, blocks)
when: when:
def act = blocksByHeight.read(100).block() def act = blocksByHeight.read(100).block()
then: then:
act == block objectMapper.readValue(act.json, BlockJson) == block
} }
def "Fetch correct blocks if multiple"() { def "Fetch correct blocks if multiple"() {
@@ -38,26 +52,40 @@ class BlockByHeightSpec extends Specification {
def block1 = new BlockJson<TransactionRefJson>() def block1 = new BlockJson<TransactionRefJson>()
block1.number = 100 block1.number = 100
block1.hash = BlockHash.from(hash1) block1.hash = BlockHash.from(hash1)
block1.totalDifficulty = BigInteger.ONE
block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block1.uncles = []
block1.transactions = []
def block2 = new BlockJson<TransactionRefJson>() def block2 = new BlockJson<TransactionRefJson>()
block2.number = 101 block2.number = 101
block2.hash = BlockHash.from(hash2) block2.hash = BlockHash.from(hash2)
block2.totalDifficulty = BigInteger.ONE
block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block2.uncles = []
block2.transactions = []
blocks.add(block1)
heights.add(block1) BlockContainer.from(block1, objectMapper).with {
blocks.add(block2) blocks.add(it)
heights.add(block2) heights.add(it)
}
BlockContainer.from(block2, objectMapper).with {
blocks.add(it)
heights.add(it)
}
def blocksByHeight = new BlockByHeight(heights, blocks) def blocksByHeight = new BlockByHeight(heights, blocks)
when: when:
def act = blocksByHeight.read(100).block() def act = blocksByHeight.read(100).block()
then: then:
act == block1 objectMapper.readValue(act.json, BlockJson) == block1
when: when:
act = blocksByHeight.read(101).block() act = blocksByHeight.read(101).block()
then: then:
act == block2 objectMapper.readValue(act.json, BlockJson) == block2
} }
def "Fetch last block if updated"() { def "Fetch last block if updated"() {
@@ -68,21 +96,34 @@ class BlockByHeightSpec extends Specification {
def block1 = new BlockJson<TransactionRefJson>() def block1 = new BlockJson<TransactionRefJson>()
block1.number = 100 block1.number = 100
block1.hash = BlockHash.from(hash1) block1.hash = BlockHash.from(hash1)
block1.totalDifficulty = BigInteger.ONE
block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block1.uncles = []
block1.transactions = []
def block2 = new BlockJson<TransactionRefJson>() def block2 = new BlockJson<TransactionRefJson>()
block2.number = 100 block2.number = 100
block2.hash = BlockHash.from(hash2) block2.hash = BlockHash.from(hash2)
block2.totalDifficulty = BigInteger.ONE
block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block2.uncles = []
block2.transactions = []
blocks.add(block1) BlockContainer.from(block1, objectMapper).with {
heights.add(block1) blocks.add(it)
blocks.add(block2) heights.add(it)
heights.add(block2) }
BlockContainer.from(block2, objectMapper).with {
blocks.add(it)
heights.add(it)
}
def blocksByHeight = new BlockByHeight(heights, blocks) def blocksByHeight = new BlockByHeight(heights, blocks)
when: when:
def act = blocksByHeight.read(100).block() def act = blocksByHeight.read(100).block()
then: then:
act == block2 objectMapper.readValue(act.json, BlockJson) == block2
} }
def "Fetch nothing if block expired"() { def "Fetch nothing if block expired"() {
@@ -93,9 +134,13 @@ class BlockByHeightSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
// add only to heights // add only to heights
heights.add(block) BlockContainer.from(block, objectMapper).with {
heights.add(it)
}
def blocksByHeight = new BlockByHeight(heights, blocks) def blocksByHeight = new BlockByHeight(heights, blocks)
@@ -113,9 +158,13 @@ class BlockByHeightSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
// add only to blocks // add only to blocks
blocks.add(block) BlockContainer.from(block, objectMapper).with {
blocks.add(it)
}
def blocksByHeight = new BlockByHeight(heights, blocks) def blocksByHeight = new BlockByHeight(heights, blocks)

View File

@@ -15,12 +15,18 @@
*/ */
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification import spock.lang.Specification
import java.time.Instant
import java.time.temporal.ChronoUnit
class BlocksMemCacheSpec extends Specification { class BlocksMemCacheSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
@@ -28,18 +34,24 @@ class BlocksMemCacheSpec extends Specification {
String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Add and read"() { def "Add and read"() {
setup: setup:
def cache = new BlocksMemCache() def cache = new BlocksMemCache()
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.uncles = []
block.transactions = []
when: when:
cache.add(block) cache.add(BlockContainer.from(block, objectMapper))
def act = cache.read(BlockHash.from(hash1)).block() def act = cache.read(BlockId.from(hash1)).block()
then: then:
act == block objectMapper.readValue(act.json, BlockJson) == block
} }
def "Keeps only configured amount"() { def "Keeps only configured amount"() {
@@ -52,13 +64,18 @@ class BlocksMemCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 + i block.number = 100 + i
block.hash = BlockHash.from(hash) block.hash = BlockHash.from(hash)
cache.add(block) block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
block.uncles = []
block.transactions = []
cache.add(BlockContainer.from(block, objectMapper))
} }
def act1 = cache.read(BlockHash.from(hash1)).block() def act1 = cache.read(BlockId.from(hash1)).block()
def act2 = cache.read(BlockHash.from(hash2)).block() def act2 = cache.read(BlockId.from(hash2)).block()
def act3 = cache.read(BlockHash.from(hash3)).block() def act3 = cache.read(BlockId.from(hash3)).block()
def act4 = cache.read(BlockHash.from(hash4)).block() def act4 = cache.read(BlockId.from(hash4)).block()
then: then:
act2.hash.toHex() == hash2 act2.hash.toHex() == hash2
act3.hash.toHex() == hash3 act3.hash.toHex() == hash3

View File

@@ -1,5 +1,8 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.test.IntegrationTestingCommons import io.emeraldpay.dshackle.test.IntegrationTestingCommons
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.grpc.Chain import io.emeraldpay.grpc.Chain
@@ -24,6 +27,7 @@ class BlocksRedisCacheSpec extends Specification {
String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def setup() { def setup() {
RedisClient client = IntegrationTestingCommons.redis() RedisClient client = IntegrationTestingCommons.redis()
@@ -40,15 +44,17 @@ class BlocksRedisCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block.totalDifficulty = BigInteger.ONE
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.transactions = [] block.transactions = []
block.uncles = [] block.uncles = []
when: when:
cache.add(block).subscribe() cache.add(BlockContainer.from(block, objectMapper)).subscribe()
def act = cache.read(BlockHash.from(hash1)).block() def act = cache.read(BlockId.from(hash1)).block()
then: then:
act == block act != null
objectMapper.readValue(act.json, BlockJson) == block
} }
def "Evict existing block"() { def "Evict existing block"() {
@@ -59,19 +65,20 @@ class BlocksRedisCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block.totalDifficulty = BigInteger.ONE
block.hash = BlockHash.from(hash2) block.hash = BlockHash.from(hash2)
block.transactions = [] block.transactions = []
block.uncles = [] block.uncles = []
when: when:
cache.add(block).subscribe() cache.add(BlockContainer.from(block, objectMapper)).subscribe()
def act = cache.read(BlockHash.from(hash2)).block() def act = cache.read(BlockId.from(hash2)).block()
then: then:
act == block objectMapper.readValue(act.json, BlockJson) == block
when: when:
cache.evict(block.hash).subscribe() cache.evict(BlockId.from(block.hash)).subscribe()
act = cache.read(BlockHash.from(hash2)).block() act = cache.read(BlockId.from(hash2)).block()
then: then:
act == null act == null
@@ -85,22 +92,25 @@ class BlocksRedisCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block.totalDifficulty = BigInteger.ONE
block.hash = BlockHash.from(hash2) block.hash = BlockHash.from(hash2)
block.transactions = [] block.transactions = []
block.uncles = [] block.uncles = []
when: when:
cache.add(block).subscribe() cache.add(BlockContainer.from(block, objectMapper)).subscribe()
def act = cache.read(BlockHash.from(hash2)).block() def act = cache.read(BlockId.from(hash2)).block()
then: then:
act == block act != null
objectMapper.readValue(act.json, BlockJson) == block
when: when:
cache.evict(BlockHash.from(hash3)).subscribe() cache.evict(BlockId.from(hash3)).subscribe()
act = cache.read(BlockHash.from(hash2)).block() act = cache.read(BlockId.from(hash2)).block()
then: then:
act == block act != null
objectMapper.readValue(act.json, BlockJson) == block
} }
} }

View File

@@ -1,5 +1,9 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
@@ -7,11 +11,14 @@ import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification import spock.lang.Specification
import java.time.Instant
class CachesSpec extends Specification { class CachesSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33" String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Evict txes if block updated"() { def "Evict txes if block updated"() {
setup: setup:
@@ -19,6 +26,7 @@ class CachesSpec extends Specification {
HeightCache heightCache = Mock() HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock() BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder() def caches = Caches.newBuilder()
.setObjectMapper(objectMapper)
.setTxByHash(txCache) .setTxByHash(txCache)
.setBlockByHeight(heightCache) .setBlockByHeight(heightCache)
.setBlockByHash(blocksCache) .setBlockByHash(blocksCache)
@@ -27,10 +35,18 @@ class CachesSpec extends Specification {
def block1 = new BlockJson() def block1 = new BlockJson()
block1.number = 100 block1.number = 100
block1.hash = BlockHash.from(hash1) block1.hash = BlockHash.from(hash1)
block1.totalDifficulty = BigInteger.ONE
block1.timestamp = Instant.now()
block1.transactions = []
block1 = BlockContainer.from(block1, objectMapper)
def block2 = new BlockJson() def block2 = new BlockJson()
block2.number = 100 block2.number = 100
block2.hash = BlockHash.from(hash2) block2.hash = BlockHash.from(hash2)
block2.totalDifficulty = BigInteger.ONE
block2.timestamp = Instant.now()
block2.transactions = []
block2 = BlockContainer.from(block2, objectMapper)
when: when:
caches.cache(Caches.Tag.LATEST, block1) caches.cache(Caches.Tag.LATEST, block1)
@@ -53,6 +69,7 @@ class CachesSpec extends Specification {
HeightCache heightCache = Mock() HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock() BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder() def caches = Caches.newBuilder()
.setObjectMapper(objectMapper)
.setTxByHash(txCache) .setTxByHash(txCache)
.setBlockByHeight(heightCache) .setBlockByHeight(heightCache)
.setBlockByHash(blocksCache) .setBlockByHash(blocksCache)
@@ -61,10 +78,16 @@ class CachesSpec extends Specification {
def block1 = new BlockJson() def block1 = new BlockJson()
block1.number = 100 block1.number = 100
block1.hash = BlockHash.from(hash1) block1.hash = BlockHash.from(hash1)
block1.totalDifficulty = BigInteger.ONE
block1.timestamp = Instant.now()
block1 = BlockContainer.from(block1, objectMapper)
def block2 = new BlockJson() def block2 = new BlockJson()
block2.number = 100 block2.number = 100
block2.hash = BlockHash.from(hash2) block2.hash = BlockHash.from(hash2)
block2.totalDifficulty = BigInteger.ONE
block2.timestamp = Instant.now()
block2 = BlockContainer.from(block2, objectMapper)
when: when:
caches.cache(Caches.Tag.LATEST, block1) caches.cache(Caches.Tag.LATEST, block1)
@@ -87,6 +110,7 @@ class CachesSpec extends Specification {
HeightCache heightCache = Mock() HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock() BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder() def caches = Caches.newBuilder()
.setObjectMapper(TestingCommons.objectMapper())
.setTxByHash(txCache) .setTxByHash(txCache)
.setBlockByHeight(heightCache) .setBlockByHeight(heightCache)
.setBlockByHash(blocksCache) .setBlockByHash(blocksCache)
@@ -95,13 +119,15 @@ class CachesSpec extends Specification {
def block = new BlockJson() def block = new BlockJson()
block.number = 100 block.number = 100
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
block.transactions = [ block.transactions = [
new TransactionRefJson(TransactionId.from(hash1)), new TransactionRefJson(TransactionId.from(hash1)),
new TransactionRefJson(TransactionId.from(hash2)), new TransactionRefJson(TransactionId.from(hash2)),
] ]
when: when:
caches.cache(Caches.Tag.REQUESTED, block) caches.cache(Caches.Tag.REQUESTED, BlockContainer.from(block, objectMapper))
then: then:
0 * txCache.add(_) 0 * txCache.add(_)
} }
@@ -112,6 +138,7 @@ class CachesSpec extends Specification {
HeightCache heightCache = Mock() HeightCache heightCache = Mock()
BlocksMemCache blocksCache = Mock() BlocksMemCache blocksCache = Mock()
def caches = Caches.newBuilder() def caches = Caches.newBuilder()
.setObjectMapper(TestingCommons.objectMapper())
.setTxByHash(txCache) .setTxByHash(txCache)
.setBlockByHeight(heightCache) .setBlockByHeight(heightCache)
.setBlockByHash(blocksCache) .setBlockByHash(blocksCache)
@@ -134,12 +161,15 @@ class CachesSpec extends Specification {
def block = new BlockJson() def block = new BlockJson()
block.number = 100 block.number = 100
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.totalDifficulty = BigInteger.ONE
block.transactions = [tx1, tx2] block.transactions = [tx1, tx2]
block.timestamp = Instant.now()
block = BlockContainer.from(block, objectMapper)
when: when:
caches.cache(Caches.Tag.REQUESTED, block) caches.cache(Caches.Tag.REQUESTED, block)
then: then:
1 * txCache.add(tx1) 1 * txCache.add(TxContainer.from(tx1, objectMapper))
1 * txCache.add(tx2) 1 * txCache.add(TxContainer.from(tx2, objectMapper))
} }
} }

View File

@@ -1,5 +1,10 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
@@ -7,7 +12,9 @@ import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification import spock.lang.Specification
class BlocksWithTxCacheSpec extends Specification { import java.time.Instant
class EthereumBlocksWithTxCacheSpec extends Specification {
// sorted // sorted
String hash1 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash1 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
@@ -15,6 +22,8 @@ class BlocksWithTxCacheSpec extends Specification {
String hash3 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash3 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
String hash4 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash4 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def tx1 = new TransactionJson().with { def tx1 = new TransactionJson().with {
it.blockNumber = 100 it.blockNumber = 100
it.blockHash = BlockHash.from(hash1) it.blockHash = BlockHash.from(hash1)
@@ -50,6 +59,8 @@ class BlocksWithTxCacheSpec extends Specification {
def block1 = new BlockJson().with { def block1 = new BlockJson().with {
it.number = 100 it.number = 100
it.hash = BlockHash.from(hash1) it.hash = BlockHash.from(hash1)
it.totalDifficulty = BigInteger.ONE
it.timestamp = Instant.now()
it.transactions = [ it.transactions = [
new TransactionRefJson(tx1.hash), new TransactionRefJson(tx1.hash),
new TransactionRefJson(tx2.hash) new TransactionRefJson(tx2.hash)
@@ -61,6 +72,8 @@ class BlocksWithTxCacheSpec extends Specification {
def block2 = new BlockJson().with { def block2 = new BlockJson().with {
it.number = 101 it.number = 101
it.hash = BlockHash.from(hash3) it.hash = BlockHash.from(hash3)
it.totalDifficulty = BigInteger.ONE
it.timestamp = Instant.now()
it.transactions = [ it.transactions = [
new TransactionRefJson(tx3.hash) new TransactionRefJson(tx3.hash)
] ]
@@ -71,6 +84,8 @@ class BlocksWithTxCacheSpec extends Specification {
def block3 = new BlockJson().with { def block3 = new BlockJson().with {
it.number = 102 it.number = 102
it.hash = BlockHash.from(hash4) it.hash = BlockHash.from(hash4)
it.totalDifficulty = BigInteger.ONE
it.timestamp = Instant.now()
it.transactions = [] it.transactions = []
it it
} }
@@ -81,21 +96,26 @@ class BlocksWithTxCacheSpec extends Specification {
def txes = new TxMemCache() def txes = new TxMemCache()
def blocks = new BlocksMemCache() def blocks = new BlocksMemCache()
txes.add(tx1) txes.add(TxContainer.from(tx1, objectMapper))
txes.add(tx2) txes.add(TxContainer.from(tx2, objectMapper))
txes.add(tx3) txes.add(TxContainer.from(tx3, objectMapper))
txes.add(tx4) txes.add(TxContainer.from(tx4, objectMapper))
blocks.add(block1) blocks.add(BlockContainer.from(block1, objectMapper))
blocks.add(block2) blocks.add(BlockContainer.from(block2, objectMapper))
blocks.add(block3) blocks.add(BlockContainer.from(block3, objectMapper))
def full = new BlocksWithTxCache(blocks, txes) def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes)
when: when:
def act = full.read(block1.hash).block() def act = full.read(BlockId.from(block1.hash)).block()
then: then:
act != null act != null
when:
act = objectMapper.readValue(act.json, BlockJson)
then:
act.hash == BlockHash.from(hash1) act.hash == BlockHash.from(hash1)
act.number == 100 act.number == 100
act.transactions.size() == 2 act.transactions.size() == 2
@@ -119,9 +139,15 @@ class BlocksWithTxCacheSpec extends Specification {
// request second block // request second block
when: when:
act = full.read(block2.hash).block() act = full.read(BlockId.from(block2.hash)).block()
then: then:
act != null act != null
when:
act = objectMapper.readValue(act.json, BlockJson)
then:
act.hash == BlockHash.from(hash3) act.hash == BlockHash.from(hash3)
act.number == 101 act.number == 101
act.transactions.size() == 1 act.transactions.size() == 1
@@ -136,23 +162,23 @@ class BlocksWithTxCacheSpec extends Specification {
def txes = new TxMemCache() def txes = new TxMemCache()
def blocks = new BlocksMemCache() def blocks = new BlocksMemCache()
txes.add(tx1) txes.add(TxContainer.from(tx1, objectMapper))
txes.add(tx2) txes.add(TxContainer.from(tx2, objectMapper))
txes.add(tx3) txes.add(TxContainer.from(tx3, objectMapper))
txes.add(tx4) txes.add(TxContainer.from(tx4, objectMapper))
blocks.add(block1) blocks.add(BlockContainer.from(block1, objectMapper))
blocks.add(block2) blocks.add(BlockContainer.from(block2, objectMapper))
blocks.add(block3) blocks.add(BlockContainer.from(block3, objectMapper))
def full = new BlocksWithTxCache(blocks, txes) def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes)
when: when:
def act = full.read(block3.hash).block() def act = full.read(BlockId.from(block3.hash)).block()
then: then:
act != null act != null
act.hash == BlockHash.from(hash4) act.hash == BlockId.from(hash4)
act.number == 102 act.height == 102
act.transactions.size() == 0 act.transactions.size() == 0
} }
@@ -161,13 +187,13 @@ class BlocksWithTxCacheSpec extends Specification {
def txes = new TxMemCache() def txes = new TxMemCache()
def blocks = new BlocksMemCache() def blocks = new BlocksMemCache()
txes.add(tx1) txes.add(TxContainer.from(tx1, objectMapper))
blocks.add(block1) //missing tx2 in cache blocks.add(BlockContainer.from(block1, objectMapper)) //missing tx2 in cache
def full = new BlocksWithTxCache(blocks, txes) def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes)
when: when:
def act = full.read(block1.hash).block() def act = full.read(BlockId.from(block1.hash)).block()
then: then:
act == null act == null
@@ -178,14 +204,14 @@ class BlocksWithTxCacheSpec extends Specification {
def txes = new TxMemCache() def txes = new TxMemCache()
def blocks = new BlocksMemCache() def blocks = new BlocksMemCache()
txes.add(tx1) txes.add(TxContainer.from(tx1, objectMapper))
txes.add(tx2) txes.add(TxContainer.from(tx2, objectMapper))
txes.add(tx3) txes.add(TxContainer.from(tx3, objectMapper))
def full = new BlocksWithTxCache(blocks, txes) def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes)
when: when:
def act = full.read(block1.hash).block() def act = full.read(BlockId.from(block1.hash)).block()
then: then:
act == null act == null

View File

@@ -1,10 +1,15 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification import spock.lang.Specification
import java.time.Instant
class HeightCacheSpec extends Specification { class HeightCacheSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
@@ -12,6 +17,8 @@ class HeightCacheSpec extends Specification {
String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Add and read"() { def "Add and read"() {
setup: setup:
def cache = new HeightCache() def cache = new HeightCache()
@@ -21,7 +28,9 @@ class HeightCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 + i block.number = 100 + i
block.hash = BlockHash.from(hash) block.hash = BlockHash.from(hash)
cache.add(block) block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
cache.add(BlockContainer.from(block, objectMapper))
} }
def act1 = cache.read(100).block() def act1 = cache.read(100).block()
@@ -45,7 +54,9 @@ class HeightCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 + i block.number = 100 + i
block.hash = BlockHash.from(hash) block.hash = BlockHash.from(hash)
cache.add(block) block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
cache.add(BlockContainer.from(block, objectMapper))
} }
def act1 = cache.read(100).block() def act1 = cache.read(100).block()

View File

@@ -1,5 +1,11 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
@@ -7,6 +13,8 @@ import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.infinitape.etherjar.rpc.json.TransactionRefJson
import spock.lang.Specification import spock.lang.Specification
import java.time.Instant
class TxMemCacheSpec extends Specification { class TxMemCacheSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
@@ -14,6 +22,8 @@ class TxMemCacheSpec extends Specification {
String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Add and read"() { def "Add and read"() {
setup: setup:
def cache = new TxMemCache() def cache = new TxMemCache()
@@ -23,10 +33,10 @@ class TxMemCacheSpec extends Specification {
tx.blockNumber = 100 tx.blockNumber = 100
when: when:
cache.add(tx) cache.add(TxContainer.from(tx, objectMapper))
def act = cache.read(TransactionId.from(hash1)).block() def act = cache.read(TxId.from(hash1)).block()
then: then:
act == tx objectMapper.readValue(act.json, TransactionJson.class) == tx
} }
def "Keeps only configured amount"() { def "Keeps only configured amount"() {
@@ -39,13 +49,13 @@ class TxMemCacheSpec extends Specification {
tx.blockNumber = 100 + i tx.blockNumber = 100 + i
tx.blockHash = BlockHash.from(hash) tx.blockHash = BlockHash.from(hash)
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
cache.add(tx) cache.add(TxContainer.from(tx, objectMapper))
} }
def act1 = cache.read(TransactionId.from(hash1)).block() def act1 = cache.read(TxId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block() def act2 = cache.read(TxId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block() def act3 = cache.read(TxId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block() def act4 = cache.read(TxId.from(hash4)).block()
then: then:
act2.hash.toHex() == hash2 act2.hash.toHex() == hash2
act3.hash.toHex() == hash3 act3.hash.toHex() == hash3
@@ -63,22 +73,22 @@ class TxMemCacheSpec extends Specification {
tx.blockNumber = 100 tx.blockNumber = 100
tx.blockHash = BlockHash.from(hash1) tx.blockHash = BlockHash.from(hash1)
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
cache.add(tx) cache.add(TxContainer.from(tx, objectMapper))
} }
[hash3, hash4].eachWithIndex { String hash, int i -> [hash3, hash4].eachWithIndex { String hash, int i ->
def tx = new TransactionJson() def tx = new TransactionJson()
tx.blockNumber = 101 tx.blockNumber = 101
tx.blockHash = BlockHash.from(hash2) tx.blockHash = BlockHash.from(hash2)
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
cache.add(tx) cache.add(TxContainer.from(tx, objectMapper))
} }
cache.evict(BlockHash.from(hash1)) cache.evict(BlockId.from(hash1))
def act1 = cache.read(TransactionId.from(hash1)).block() def act1 = cache.read(TxId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block() def act2 = cache.read(TxId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block() def act3 = cache.read(TxId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block() def act4 = cache.read(TxId.from(hash4)).block()
then: then:
act1 == null act1 == null
@@ -97,30 +107,32 @@ class TxMemCacheSpec extends Specification {
tx.blockNumber = 100 tx.blockNumber = 100
tx.blockHash = BlockHash.from(hash1) tx.blockHash = BlockHash.from(hash1)
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
cache.add(tx) cache.add(TxContainer.from(tx, objectMapper))
} }
[hash3, hash4].eachWithIndex{ String hash, int i -> [hash3, hash4].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson() def tx = new TransactionJson()
tx.blockNumber = 100 tx.blockNumber = 100
tx.blockHash = BlockHash.from(hash2) tx.blockHash = BlockHash.from(hash2)
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
cache.add(tx) cache.add(TxContainer.from(tx, objectMapper))
} }
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.number = 100 block.number = 100
block.totalDifficulty = BigInteger.ONE
block.timestamp = Instant.now()
block.transactions = [ block.transactions = [
new TransactionRefJson(TransactionId.from(hash1)), new TransactionRefJson(TransactionId.from(hash1)),
new TransactionRefJson(TransactionId.from(hash2)), new TransactionRefJson(TransactionId.from(hash2)),
] ]
cache.evict(block) cache.evict(BlockContainer.from(block, objectMapper))
def act1 = cache.read(TransactionId.from(hash1)).block() def act1 = cache.read(TxId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block() def act2 = cache.read(TxId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block() def act3 = cache.read(TxId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block() def act4 = cache.read(TxId.from(hash4)).block()
then: then:
act1 == null act1 == null

View File

@@ -1,5 +1,9 @@
package io.emeraldpay.dshackle.cache package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.test.IntegrationTestingCommons import io.emeraldpay.dshackle.test.IntegrationTestingCommons
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.grpc.Chain import io.emeraldpay.grpc.Chain
@@ -26,6 +30,8 @@ class TxRedisCacheSpec extends Specification {
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
TxRedisCache cache TxRedisCache cache
def objectMapper = TestingCommons.objectMapper()
def setup() { def setup() {
RedisClient client = IntegrationTestingCommons.redis() RedisClient client = IntegrationTestingCommons.redis()
StatefulRedisConnection<String, String> connection = client.connect(); StatefulRedisConnection<String, String> connection = client.connect();
@@ -39,6 +45,7 @@ class TxRedisCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block.totalDifficulty = BigInteger.ONE
block.hash = BlockHash.from(hash1) block.hash = BlockHash.from(hash1)
block.transactions = [] block.transactions = []
block.uncles = [] block.uncles = []
@@ -51,10 +58,11 @@ class TxRedisCacheSpec extends Specification {
tx.nonce = 0 tx.nonce = 0
when: when:
cache.add(tx, block).subscribe() cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block, objectMapper)).subscribe()
def act = cache.read(TransactionId.from(hash1)).block() def act = cache.read(TxId.from(hash1)).block()
then: then:
act == tx act != null
objectMapper.readValue(act.json, TransactionJson) == tx
} }
def "Evict single tx"() { def "Evict single tx"() {
@@ -62,6 +70,7 @@ class TxRedisCacheSpec extends Specification {
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100 block.number = 100
block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block.totalDifficulty = BigInteger.ONE
block.hash = BlockHash.from(hash2) block.hash = BlockHash.from(hash2)
block.transactions = [] block.transactions = []
block.uncles = [] block.uncles = []
@@ -74,14 +83,15 @@ class TxRedisCacheSpec extends Specification {
tx.nonce = 0 tx.nonce = 0
when: when:
cache.add(tx, block).subscribe() cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block, objectMapper)).subscribe()
def act = cache.read(tx.hash).block() def act = cache.read(TxId.from(tx.hash)).block()
then: then:
act == tx act != null
objectMapper.readValue(act.json, TransactionJson) == tx
when: when:
cache.evict(tx.hash).subscribe() cache.evict(TxId.from(tx.hash)).subscribe()
act = cache.read(tx.hash).block() act = cache.read(TxId.from(tx.hash)).block()
then: then:
act == null act == null
} }
@@ -92,6 +102,7 @@ class TxRedisCacheSpec extends Specification {
block1.hash = BlockHash.from(hash1) block1.hash = BlockHash.from(hash1)
block1.number = 100 block1.number = 100
block1.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block1.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block1.totalDifficulty = BigInteger.ONE
block1.transactions = [ block1.transactions = [
new TransactionRefJson(TransactionId.from(hash1)), new TransactionRefJson(TransactionId.from(hash1)),
new TransactionRefJson(TransactionId.from(hash2)), new TransactionRefJson(TransactionId.from(hash2)),
@@ -100,6 +111,7 @@ class TxRedisCacheSpec extends Specification {
block2.hash = BlockHash.from(hash2) block2.hash = BlockHash.from(hash2)
block2.number = 101 block2.number = 101
block2.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) block2.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS)
block2.totalDifficulty = BigInteger.ONE
block2.transactions = [ block2.transactions = [
new TransactionRefJson(TransactionId.from(hash3)), new TransactionRefJson(TransactionId.from(hash3)),
new TransactionRefJson(TransactionId.from(hash4)), new TransactionRefJson(TransactionId.from(hash4)),
@@ -112,7 +124,7 @@ class TxRedisCacheSpec extends Specification {
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
tx.value = Wei.ofEthers(i) tx.value = Wei.ofEthers(i)
tx.nonce = 0 tx.nonce = 0
cache.add(tx, block1).subscribe() cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block1, objectMapper)).subscribe()
} }
[hash3, hash4].eachWithIndex{ String hash, int i -> [hash3, hash4].eachWithIndex{ String hash, int i ->
def tx = new TransactionJson() def tx = new TransactionJson()
@@ -121,16 +133,16 @@ class TxRedisCacheSpec extends Specification {
tx.hash = TransactionId.from(hash) tx.hash = TransactionId.from(hash)
tx.value = Wei.ofEthers(i) tx.value = Wei.ofEthers(i)
tx.nonce = 0 tx.nonce = 0
cache.add(tx, block2).subscribe() cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block2, objectMapper)).subscribe()
} }
cache.evict(block1).subscribe() cache.evict(BlockContainer.from(block1, objectMapper)).subscribe()
def act1 = cache.read(TransactionId.from(hash1)).block() def act1 = cache.read(TxId.from(hash1)).block()
def act2 = cache.read(TransactionId.from(hash2)).block() def act2 = cache.read(TxId.from(hash2)).block()
def act3 = cache.read(TransactionId.from(hash3)).block() def act3 = cache.read(TxId.from(hash3)).block()
def act4 = cache.read(TransactionId.from(hash4)).block() def act4 = cache.read(TxId.from(hash4)).block()
then: then:
act1 == null act1 == null

View File

@@ -15,10 +15,13 @@
*/ */
package io.emeraldpay.dshackle.rpc package io.emeraldpay.dshackle.rpc
import com.fasterxml.jackson.databind.ObjectMapper
import com.google.protobuf.ByteString import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.EthereumUpstreamMock import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.test.UpstreamsMock import io.emeraldpay.dshackle.test.UpstreamsMock
import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
@@ -37,6 +40,8 @@ import java.time.Instant
class StreamHeadSpec extends Specification { class StreamHeadSpec extends Specification {
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Errors on unavailable chain"() { def "Errors on unavailable chain"() {
setup: setup:
def upstreams = new UpstreamsMock(Chain.ETHEREUM, Stub(EthereumUpstream)) def upstreams = new UpstreamsMock(Chain.ETHEREUM, Stub(EthereumUpstream))
@@ -83,9 +88,9 @@ class StreamHeadSpec extends Specification {
) )
then: then:
StepVerifier.create(flux.take(2)) StepVerifier.create(flux.take(2))
.then { upstream.nextBlock(blocks[0]) } .then { upstream.nextBlock(BlockContainer.from(blocks[0], objectMapper)) }
.expectNext(heads[0]) .expectNext(heads[0])
.then { upstream.nextBlock(blocks[1]) } .then { upstream.nextBlock(BlockContainer.from(blocks[1], objectMapper)) }
.expectNext(heads[1]) .expectNext(heads[1])
.expectComplete() .expectComplete()
.verify(Duration.ofSeconds(1)) .verify(Duration.ofSeconds(1))

View File

@@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.test.UpstreamsMock import io.emeraldpay.dshackle.test.UpstreamsMock
import io.emeraldpay.dshackle.upstream.Upstreams import io.emeraldpay.dshackle.upstream.Upstreams
@@ -32,6 +33,8 @@ import reactor.test.StepVerifier
import spock.lang.Specification import spock.lang.Specification
import java.time.Duration import java.time.Duration
import java.time.Instant
import java.time.temporal.ChronoUnit
class TrackEthereumAddressSpec extends Specification { class TrackEthereumAddressSpec extends Specification {
@@ -94,6 +97,7 @@ class TrackEthereumAddressSpec extends Specification {
it.number = 1 it.number = 1
it.totalDifficulty = 100 it.totalDifficulty = 100
it.hash = BlockHash.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22") it.hash = BlockHash.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22")
it.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
return it return it
} }
@@ -115,7 +119,7 @@ class TrackEthereumAddressSpec extends Specification {
assert trackAddress.isTracked(Chain.ETHEREUM, Address.from(address1)) assert trackAddress.isTracked(Chain.ETHEREUM, Address.from(address1))
} }
.then { .then {
upstreamMock.nextBlock(block2) upstreamMock.nextBlock(BlockContainer.from(block2, TestingCommons.objectMapper()))
} }
.expectNext(exp2) .expectNext(exp2)
.thenCancel() .thenCancel()

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.test.UpstreamsMock import io.emeraldpay.dshackle.test.UpstreamsMock
import io.emeraldpay.dshackle.upstream.Upstreams import io.emeraldpay.dshackle.upstream.Upstreams
@@ -63,6 +64,7 @@ class TrackEthereumTxSpec extends Specification {
it.timestamp = Instant.ofEpochMilli(156400200000) it.timestamp = Instant.ofEpochMilli(156400200000)
it.number = 108 it.number = 108
it.totalDifficulty = BigInteger.valueOf(800) it.totalDifficulty = BigInteger.valueOf(800)
it.transactions = []
it it
} }
@@ -75,6 +77,8 @@ class TrackEthereumTxSpec extends Specification {
it it
} }
blockJson.transactions = [new TransactionRefJson(txJson.hash)]
def exp1 = BlockchainOuterClass.TxStatus.newBuilder() def exp1 = BlockchainOuterClass.TxStatus.newBuilder()
.setTxId(txId) .setTxId(txId)
.setBroadcasted(true) .setBroadcasted(true)
@@ -96,7 +100,7 @@ class TrackEthereumTxSpec extends Specification {
apiMock.answer("eth_getTransactionByHash", [txId], txJson) apiMock.answer("eth_getTransactionByHash", [txId], txJson)
apiMock.answer("eth_getBlockByHash", [blockJson.hash.toHex(), false], blockJson) apiMock.answer("eth_getBlockByHash", [blockJson.hash.toHex(), false], blockJson)
upstreamMock.nextBlock(blockHeadJson) upstreamMock.nextBlock(BlockContainer.from(blockHeadJson, TestingCommons.objectMapper()))
when: when:
def flux = trackTx.add(Mono.just(req)) def flux = trackTx.add(Mono.just(req))
@@ -292,7 +296,7 @@ class TrackEthereumTxSpec extends Specification {
def nextBlock = { int i -> def nextBlock = { int i ->
return { return {
println("block $i"); println("block $i");
upstreamMock.nextBlock(blocks[i]) upstreamMock.nextBlock(BlockContainer.from(blocks[i], TestingCommons.objectMapper()))
} as Runnable } as Runnable
} }

View File

@@ -15,26 +15,25 @@
*/ */
package io.emeraldpay.dshackle.test package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor import reactor.core.publisher.TopicProcessor
class EthereumHeadMock implements EthereumHead { class EthereumHeadMock implements EthereumHead {
private TopicProcessor<BlockJson<TransactionId>> bus = TopicProcessor.create() private TopicProcessor<BlockContainer> bus = TopicProcessor.create()
private BlockJson<TransactionId> latest private BlockContainer latest
void nextBlock(BlockJson<TransactionId> block) { void nextBlock(BlockContainer block) {
assert block != null assert block != null
latest = block latest = block
bus.onNext(block) bus.onNext(block)
} }
@Override @Override
Flux<BlockJson<TransactionId>> getFlux() { Flux<BlockContainer> getFlux() {
return Flux.concat(Mono.justOrEmpty(latest), bus).distinctUntilChanged() return Flux.concat(Mono.justOrEmpty(latest), bus).distinctUntilChanged()
} }
} }

View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.test package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
@@ -47,12 +48,12 @@ class EthereumUpstreamMock extends EthereumUpstream {
EthereumUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull DirectEthereumApi api, CallMethods methods) { EthereumUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull DirectEthereumApi api, CallMethods methods) {
super(id, chain, api, null, super(id, chain, api, null,
UpstreamsConfig.Options.getDefaults(), new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()), UpstreamsConfig.Options.getDefaults(), new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()),
methods) methods, TestingCommons.objectMapper())
setLag(0) setLag(0)
setStatus(UpstreamAvailability.OK) setStatus(UpstreamAvailability.OK)
} }
void nextBlock(BlockJson<TransactionId> block) { void nextBlock(BlockContainer block) {
ethereumHeadMock.nextBlock(block) ethereumHeadMock.nextBlock(block)
} }

View File

@@ -76,7 +76,7 @@ class TestingCommons {
} }
static AggregatedUpstream aggregatedUpstream(EthereumUpstream up) { static AggregatedUpstream aggregatedUpstream(EthereumUpstream up) {
return new EthereumChainUpstreams(Chain.ETHEREUM, [up], Caches.default(), objectMapper()) return new EthereumChainUpstreams(Chain.ETHEREUM, [up], Caches.default(objectMapper()), objectMapper())
} }
static CachesFactory emptyCaches() { static CachesFactory emptyCaches() {

View File

@@ -41,7 +41,7 @@ class UpstreamsMock implements Upstreams {
AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull Upstream up) { AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull Upstream up) {
if (!upstreams.containsKey(chain)) { if (!upstreams.containsKey(chain)) {
upstreams[chain] = new EthereumChainUpstreams(chain, [up], Caches.default(), TestingCommons.objectMapper()) upstreams[chain] = new EthereumChainUpstreams(chain, [up], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
} else { } else {
upstreams[chain].addUpstream(up) upstreams[chain].addUpstream(up)
} }

View File

@@ -31,7 +31,7 @@ class AggregatedUpstreamSpec extends Specification {
setup: setup:
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test1", "eth_test2"])) def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test1", "eth_test2"]))
def up2 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test2", "eth_test3"])) def up2 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test2", "eth_test3"]))
def aggr = new EthereumChainUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(), TestingCommons.objectMapper()) def aggr = new EthereumChainUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
when: when:
aggr.onUpstreamsUpdated() aggr.onUpstreamsUpdated()
def act = aggr.getMethods() def act = aggr.getMethods()

View File

@@ -1,11 +1,14 @@
package io.emeraldpay.dshackle.upstream package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.BlockByHeight import io.emeraldpay.dshackle.cache.BlockByHeight
import io.emeraldpay.dshackle.cache.BlocksMemCache import io.emeraldpay.dshackle.cache.BlocksMemCache
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.HeightCache import io.emeraldpay.dshackle.cache.HeightCache
import io.emeraldpay.dshackle.cache.TxMemCache import io.emeraldpay.dshackle.cache.TxMemCache
import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
@@ -18,18 +21,31 @@ import reactor.test.StepVerifier
import spock.lang.Specification import spock.lang.Specification
import java.time.Duration import java.time.Duration
import java.time.Instant
import java.time.temporal.ChronoUnit
class CachingEthereumApiSpec extends Specification { class CachingEthereumApiSpec extends Specification {
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Get blockNumber from head"() { def "Get blockNumber from head"() {
setup: setup:
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.default(), Caches.default(objectMapper),
head head
) )
1 * head.getFlux() >> Flux.just(new BlockJson<TransactionRefJson>(number: 100)) 1 * head.getFlux() >> Flux.just(BlockContainer.from(
new BlockJson<TransactionRefJson>(
number: 100,
hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
difficulty: 1,
totalDifficulty: BigInteger.ONE,
timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS)
),
objectMapper
))
when: when:
def act = api.execute(1, "eth_blockNumber", []).map { new String(it) } def act = api.execute(1, "eth_blockNumber", []).map { new String(it) }
@@ -44,8 +60,8 @@ class CachingEthereumApiSpec extends Specification {
setup: setup:
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.default(), Caches.default(objectMapper),
head head
) )
when: when:
@@ -62,18 +78,26 @@ class CachingEthereumApiSpec extends Specification {
def cache = new BlocksMemCache(); def cache = new BlocksMemCache();
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.newBuilder().setBlockByHash(cache).build(), Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(cache).build(),
head head
) )
cache.add(new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"))) cache.add(BlockContainer.from(
new BlockJson<TransactionRefJson>(
number: 100,
hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
totalDifficulty: BigInteger.ONE,
timestamp: Instant.ofEpochSecond(0x5e95313a)
),
objectMapper
))
when: when:
def act = api.execute(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).map { new String(it) } def act = api.execute(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).map { new String(it) }
then: then:
StepVerifier.create(act) StepVerifier.create(act)
.expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","transactions":[],"uncles":[]}}') .expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","timestamp":"0x5e95313a","transactions":[],"totalDifficulty":"0x1","uncles":[]}}')
.expectComplete() .expectComplete()
.verify(Duration.ofSeconds(3)) .verify(Duration.ofSeconds(3))
} }
@@ -84,20 +108,25 @@ class CachingEthereumApiSpec extends Specification {
def heightCache = new HeightCache() def heightCache = new HeightCache()
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.newBuilder().setBlockByHash(blocksCache).setBlockByHeight(heightCache).build(), Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setBlockByHeight(heightCache).build(),
head head
) )
def block = new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) def block = new BlockJson<TransactionRefJson>(
heightCache.add(block) number: 100,
blocksCache.add(block) hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
totalDifficulty: BigInteger.ONE,
timestamp: Instant.ofEpochSecond(0x5e95313a)
)
heightCache.add(BlockContainer.from(block, objectMapper))
blocksCache.add(BlockContainer.from(block, objectMapper))
when: when:
def act = api.execute(1, "eth_getBlockByNumber", ["0x64", false]).map { new String(it) } def act = api.execute(1, "eth_getBlockByNumber", ["0x64", false]).map { new String(it) }
then: then:
StepVerifier.create(act) StepVerifier.create(act)
.expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","transactions":[],"uncles":[]}}') .expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","timestamp":"0x5e95313a","transactions":[],"totalDifficulty":"0x1","uncles":[]}}')
.expectComplete() .expectComplete()
.verify(Duration.ofSeconds(3)) .verify(Duration.ofSeconds(3))
} }
@@ -108,18 +137,23 @@ class CachingEthereumApiSpec extends Specification {
def txCache = Mock(TxMemCache) def txCache = Mock(TxMemCache)
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).build(), Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).build(),
head head
) )
def block = new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) def block = new BlockJson<TransactionRefJson>(
number: 100,
hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
totalDifficulty: BigInteger.ONE,
timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS)
)
when: when:
def act = api.readBlockByHash(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).block() def act = api.readBlockByHash(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).block()
then: then:
act != null act != null
1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper))
0 * txCache.read(_) 0 * txCache.read(_)
} }
@@ -129,11 +163,16 @@ class CachingEthereumApiSpec extends Specification {
def txCache = Mock(TxMemCache) def txCache = Mock(TxMemCache)
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).build(), Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).build(),
head head
) )
def block = new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) def block = new BlockJson<TransactionRefJson>(
number: 100,
hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
totalDifficulty: BigInteger.ONE,
timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS)
)
block.transactions = [ block.transactions = [
new TransactionRefJson(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) new TransactionRefJson(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073"))
] ]
@@ -143,8 +182,8 @@ class CachingEthereumApiSpec extends Specification {
then: then:
act == null act == null
1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper))
1 * txCache.read(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty() 1 * txCache.read(TxId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty()
} }
def "Uses base cache when requested, by height"() { def "Uses base cache when requested, by height"() {
@@ -154,19 +193,24 @@ class CachingEthereumApiSpec extends Specification {
def heightCache = Mock(HeightCache) def heightCache = Mock(HeightCache)
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(), Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(),
head head
) )
def block = new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) def block = new BlockJson<TransactionRefJson>(
number: 100,
hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
totalDifficulty: BigInteger.ONE,
timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS)
)
when: when:
def act = api.readBlockByNumber(1, "eth_getBlockByNumber", ["0x64", false]).block() def act = api.readBlockByNumber(1, "eth_getBlockByNumber", ["0x64", false]).block()
then: then:
act != null act != null
1 * heightCache.read(100) >> Mono.just(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) 1 * heightCache.read(100) >> Mono.just(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"))
1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper))
0 * txCache.read(_) 0 * txCache.read(_)
} }
@@ -177,11 +221,16 @@ class CachingEthereumApiSpec extends Specification {
def heightCache = Mock(HeightCache) def heightCache = Mock(HeightCache)
def head = Mock(EthereumHead.class) def head = Mock(EthereumHead.class)
def api = new CachingEthereumApi( def api = new CachingEthereumApi(
TestingCommons.objectMapper(), objectMapper,
Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(), Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(),
head head
) )
def block = new BlockJson<TransactionRefJson>(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) def block = new BlockJson<TransactionRefJson>(
number: 100,
hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"),
totalDifficulty: BigInteger.ONE,
timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS)
)
block.transactions = [ block.transactions = [
new TransactionRefJson(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) new TransactionRefJson(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073"))
] ]
@@ -191,8 +240,8 @@ class CachingEthereumApiSpec extends Specification {
then: then:
act == null act == null
1 * heightCache.read(100) >> Mono.just(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) 1 * heightCache.read(100) >> Mono.just(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"))
1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper))
1 * txCache.read(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty() 1 * txCache.read(TxId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty()
} }
} }

View File

@@ -53,7 +53,7 @@ class FilteredApisSpec extends Specification {
(EthereumWs) null, (EthereumWs) null,
new UpstreamsConfig.Options(), new UpstreamsConfig.Options(),
new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)), new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)),
ethereumTargets ethereumTargets, TestingCommons.objectMapper()
) )
} }
def matcher = new Selector.LabelMatcher("test", ["foo"]) def matcher = new Selector.LabelMatcher("test", ["foo"])

View File

@@ -15,23 +15,31 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.test.StepVerifier import reactor.test.StepVerifier
import spock.lang.Specification import spock.lang.Specification
import java.time.Instant
class DefaultEthereumHeadSpec extends Specification { class DefaultEthereumHeadSpec extends Specification {
DefaultEthereumHead head = new DefaultEthereumHead() DefaultEthereumHead head = new DefaultEthereumHead()
ObjectMapper objectMapper = TestingCommons.objectMapper()
def blocks = (10L..20L).collect { i -> def blocks = (10L..20L).collect { i ->
BlockContainer.from(
new BlockJson().with { new BlockJson().with {
it.number = 10000L + i it.number = 10000L + i
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec89152" + i) it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec89152" + i)
it.totalDifficulty = 11 * i it.totalDifficulty = 11 * i
it.timestamp = Instant.now()
return it return it
} }, objectMapper)
} }
def "Starts to follow"() { def "Starts to follow"() {
@@ -80,12 +88,14 @@ class DefaultEthereumHeadSpec extends Specification {
def "Ignores less difficult"() { def "Ignores less difficult"() {
when: when:
def block3less = new BlockJson().with { def block3less = BlockContainer.from(
it.number = blocks[3].number new BlockJson().with {
it.hash = blocks[3].hash it.number = blocks[3].height
it.totalDifficulty = blocks[3].totalDifficulty - 1 it.hash = BlockHash.from(blocks[3].hash.value)
it.totalDifficulty = blocks[3].difficulty - 1
it.timestamp = Instant.now()
return it return it
} }, objectMapper)
head.follow(Flux.just(blocks[0], blocks[3], block3less)) head.follow(Flux.just(blocks[0], blocks[3], block3less))
def act = head.flux def act = head.flux
then: then:
@@ -97,12 +107,14 @@ class DefaultEthereumHeadSpec extends Specification {
def "Replaces with more difficult"() { def "Replaces with more difficult"() {
when: when:
def block3less = new BlockJson().with { def block3less = BlockContainer.from(
it.number = blocks[3].number new BlockJson().with {
it.hash = blocks[3].hash it.number = blocks[3].height
it.totalDifficulty = blocks[3].totalDifficulty + 1 it.hash = BlockHash.from(blocks[3].hash.value)
it.totalDifficulty = blocks[3].difficulty + 1
it.timestamp = Instant.now()
return it return it
} }, objectMapper)
head.follow(Flux.just(blocks[0], blocks[3], block3less)) head.follow(Flux.just(blocks[0], blocks[3], block3less))
def act = head.flux def act = head.flux
then: then:

View File

@@ -15,8 +15,12 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.HeadLagObserver import io.emeraldpay.dshackle.upstream.HeadLagObserver
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.BlockJson
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.TopicProcessor import reactor.core.publisher.TopicProcessor
@@ -25,9 +29,12 @@ import reactor.util.function.Tuples
import spock.lang.Specification import spock.lang.Specification
import java.time.Duration import java.time.Duration
import java.time.Instant
class EthereumHeadLagObserverSpec extends Specification { class EthereumHeadLagObserverSpec extends Specification {
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Updates lag distance"() { def "Updates lag distance"() {
setup: setup:
EthereumHead master = Mock() EthereumHead master = Mock()
@@ -43,11 +50,15 @@ class EthereumHeadLagObserverSpec extends Specification {
} }
def blocks = [100, 101, 102].collect { i -> def blocks = [100, 101, 102].collect { i ->
return new BlockJson().with { return BlockContainer.from(
new BlockJson().with {
it.number = i it.number = i
it.totalDifficulty = 2000 + i it.totalDifficulty = 2000 + i
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915" + i)
it.timestamp = Instant.now()
return it return it
} },
objectMapper)
} }
def masterBus = TopicProcessor.create() def masterBus = TopicProcessor.create()
@@ -83,11 +94,15 @@ class EthereumHeadLagObserverSpec extends Specification {
Upstream up = Mock() Upstream up = Mock()
def blocks = [100, 101, 102].collect { i -> def blocks = [100, 101, 102].collect { i ->
return new BlockJson().with { return BlockContainer.from(
new BlockJson().with {
it.number = i it.number = i
it.totalDifficulty = 2000 + i it.totalDifficulty = 2000 + i
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915" + i)
it.timestamp = Instant.now()
return it return it
} },
objectMapper)
} }
def upblocks = Flux.fromIterable(blocks) def upblocks = Flux.fromIterable(blocks)
@@ -109,14 +124,18 @@ class EthereumHeadLagObserverSpec extends Specification {
def top = new BlockJson().with { def top = new BlockJson().with {
it.number = topHeight it.number = topHeight
it.totalDifficulty = topDiff it.totalDifficulty = topDiff
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123")
it.timestamp = Instant.now()
return it return it
} }
def curr = new BlockJson().with { def curr = new BlockJson().with {
it.number = currHeight it.number = currHeight
it.totalDifficulty = currDiff it.totalDifficulty = currDiff
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123")
it.timestamp = Instant.now()
return it return it
} }
delta as Long == observer.extractDistance(top, curr) delta as Long == observer.extractDistance(BlockContainer.from(top, objectMapper), BlockContainer.from(curr, objectMapper))
where: where:
topHeight | topDiff | currHeight | currDiff | delta topHeight | topDiff | currHeight | currDiff | delta
100 | 1000 | 100 | 1000 | 0 100 | 1000 | 100 | 1000 | 0

View File

@@ -1,8 +1,11 @@
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.BlocksMemCache import io.emeraldpay.dshackle.cache.BlocksMemCache
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.HeightCache import io.emeraldpay.dshackle.cache.HeightCache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.ReactorRpcClient import io.infinitape.etherjar.rpc.ReactorRpcClient
@@ -19,26 +22,30 @@ import java.time.temporal.ChronoUnit
class EthereumWsSpec extends Specification { class EthereumWsSpec extends Specification {
ObjectMapper objectMapper = TestingCommons.objectMapper()
def "Uses cache to fetch block"() { def "Uses cache to fetch block"() {
setup: setup:
ReactorRpcClient rpcClient = Stub(ReactorRpcClient) ReactorRpcClient rpcClient = Stub(ReactorRpcClient)
def apiMock = TestingCommons.api(rpcClient) def apiMock = TestingCommons.api(rpcClient)
def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock) def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock, objectMapper)
def blocksCache = Mock(BlocksMemCache) def blocksCache = Mock(BlocksMemCache)
def caches = Caches.newBuilder().setBlockByHash(blocksCache).build() def caches = Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).build()
ws.setCaches(caches) ws.setCaches(caches)
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100
block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200") block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.totalDifficulty = BigInteger.ONE
when: when:
ws.onNewBlock(block) ws.onNewBlock(block)
then: then:
1 * blocksCache.read(BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")) >> Mono.just(block) 1 * blocksCache.read(BlockId.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")) >> Mono.just(BlockContainer.from(block, objectMapper))
StepVerifier.create(ws.flux.take(1)) StepVerifier.create(ws.flux.take(1))
.expectNext(block) .expectNext(BlockContainer.from(block, objectMapper))
.expectComplete() .expectComplete()
.verify(Duration.ofSeconds(1)) .verify(Duration.ofSeconds(1))
} }
@@ -47,16 +54,18 @@ class EthereumWsSpec extends Specification {
setup: setup:
ReactorRpcClient rpcClient = Stub(ReactorRpcClient) ReactorRpcClient rpcClient = Stub(ReactorRpcClient)
def apiMock = TestingCommons.api(rpcClient) def apiMock = TestingCommons.api(rpcClient)
def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock) def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock, objectMapper)
def blocksCache = Mock(BlocksMemCache) def blocksCache = Mock(BlocksMemCache)
def caches = Caches.newBuilder().setBlockByHash(blocksCache).build() def caches = Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).build()
ws.setCaches(caches) ws.setCaches(caches)
def block = new BlockJson<TransactionRefJson>() def block = new BlockJson<TransactionRefJson>()
block.number = 100
block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200") block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")
block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS)
block.transactions = [] block.transactions = []
block.uncles = [] block.uncles = []
block.totalDifficulty = BigInteger.ONE
apiMock.answerOnce("eth_getBlockByHash", ["0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200", false], block) apiMock.answerOnce("eth_getBlockByHash", ["0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200", false], block)
@@ -66,7 +75,7 @@ class EthereumWsSpec extends Specification {
then: then:
1 * blocksCache.read(_) >> Mono.empty() 1 * blocksCache.read(_) >> Mono.empty()
StepVerifier.create(ws.flux.take(1)) StepVerifier.create(ws.flux.take(1))
.expectNext(block) .expectNext(BlockContainer.from(block, objectMapper))
.expectComplete() .expectComplete()
.verify(Duration.ofSeconds(1)) .verify(Duration.ofSeconds(1))
} }

View File

@@ -20,6 +20,7 @@ import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainGrpc import io.emeraldpay.api.proto.BlockchainGrpc
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.test.MockServer import io.emeraldpay.dshackle.test.MockServer
import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.UpstreamAvailability
@@ -32,6 +33,7 @@ import io.infinitape.etherjar.rpc.json.BlockJson
import spock.lang.Specification import spock.lang.Specification
import java.time.Duration import java.time.Duration
import java.time.Instant
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
class EthereumGrpcUpstreamSpec extends Specification { class EthereumGrpcUpstreamSpec extends Specification {
@@ -48,6 +50,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
it.number = 650246 it.number = 650246
it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) it.totalDifficulty = new BigInteger("35bbde5595de6456", 16)
it.timestamp = Instant.now()
return it return it
} }
api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1)
@@ -81,7 +84,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
then: then:
callData.chain == Chain.ETHEREUM.id callData.chain == Chain.ETHEREUM.id
upstream.status == UpstreamAvailability.OK upstream.status == UpstreamAvailability.OK
h.hash == BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") h.hash == BlockId.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
} }
def "Follows difficulty, ignores less difficult"() { def "Follows difficulty, ignores less difficult"() {
@@ -91,12 +94,14 @@ class EthereumGrpcUpstreamSpec extends Specification {
it.number = 650246 it.number = 650246
it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) it.totalDifficulty = new BigInteger("35bbde5595de6456", 16)
it.timestamp = Instant.now()
return it return it
} }
def block2 = new BlockJson().with { def block2 = new BlockJson().with {
it.number = 650247 it.number = 650247
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a")
it.totalDifficulty = new BigInteger("35bbde5595de6455", 16) it.totalDifficulty = new BigInteger("35bbde5595de6455", 16)
it.timestamp = Instant.now()
return it return it
} }
api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1)
@@ -136,8 +141,8 @@ class EthereumGrpcUpstreamSpec extends Specification {
def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block() def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block()
then: then:
upstream.status == UpstreamAvailability.OK upstream.status == UpstreamAvailability.OK
h.hash == BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") h.hash == BlockId.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
h.number == 650246 h.height == 650246
} }
def "Follows difficulty"() { def "Follows difficulty"() {
@@ -150,12 +155,14 @@ class EthereumGrpcUpstreamSpec extends Specification {
it.number = 650246 it.number = 650246
it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) it.totalDifficulty = new BigInteger("35bbde5595de6456", 16)
it.timestamp = Instant.now()
return it return it
} }
def block2 = new BlockJson().with { def block2 = new BlockJson().with {
it.number = 650247 it.number = 650247
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a")
it.totalDifficulty = new BigInteger("35bbde5595de6457", 16) it.totalDifficulty = new BigInteger("35bbde5595de6457", 16)
it.timestamp = Instant.now()
return it return it
} }
api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1)
@@ -197,7 +204,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block() def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block()
then: then:
upstream.status == UpstreamAvailability.OK upstream.status == UpstreamAvailability.OK
h.hash == BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") h.hash == BlockId.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a")
h.number == 650247 h.height == 650247
} }
} }