diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt index c92e55c5..843f9062 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt @@ -1,25 +1,24 @@ package io.emeraldpay.dshackle.cache +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import reactor.core.publisher.Mono /** * Connects two caches to read through them. First is cache height->hash, second is hash->block. */ -open class BlockByHeight( - private val heights: Reader, - private val blocks: Reader> -): Reader> { +open class BlockByHeight( + private val heights: Reader, + private val blocks: Reader +) : Reader { companion object { private val log = LoggerFactory.getLogger(BlockByHeight::class.java) } - override fun read(key: Long): Mono> { + override fun read(key: Long): Mono { return heights.read(key) .flatMap { blocks.read(it) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt index 05740ed2..679944f0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt @@ -15,31 +15,29 @@ */ package io.emeraldpay.dshackle.cache +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.domain.TransactionId -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import reactor.core.publisher.Mono import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentLinkedQueue open class BlocksMemCache( val maxSize: Int = 64 -): Reader> { +) : Reader { - private val mapping = ConcurrentHashMap>() - private val queue = ConcurrentLinkedQueue() + private val mapping = ConcurrentHashMap() + private val queue = ConcurrentLinkedQueue() - override fun read(key: BlockHash): Mono> { + override fun read(key: BlockId): Mono { return Mono.justOrEmpty(mapping[key]) } - open fun get(key: BlockHash): BlockJson? { + open fun get(key: BlockId): BlockContainer? { return mapping[key] } - open fun add(block: BlockJson) { + open fun add(block: BlockContainer) { mapping.put(block.hash, block) queue.add(block.hash) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt index 359c7d00..dc542440 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt @@ -1,12 +1,13 @@ package io.emeraldpay.dshackle.cache import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.grpc.Chain -import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.lettuce.core.api.reactive.RedisReactiveCommands +import org.apache.commons.codec.binary.Base64 import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import java.time.Instant @@ -20,25 +21,27 @@ class BlocksRedisCache( private val redis: RedisReactiveCommands, private val chain: Chain, private val objectMapper: ObjectMapper -): Reader> { +) : Reader { companion object { private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java) private const val MAX_CACHE_TIME_MINUTES = 60L + // doesn't make sense to cached in redis short living objects private const val MIN_CACHE_TIME_SECONDS = 10 } - override fun read(key: BlockHash): Mono> { + override fun read(key: BlockId): Mono { return redis.get(key(key)) .map { data -> - objectMapper.readValue(data, BlockJson::class.java) as BlockJson + val block = objectMapper.readValue(data, BlockJson::class.java) + BlockContainer.from(block, objectMapper) }.onErrorResume { Mono.empty() } } - fun evict(id: BlockHash): Mono { + fun evict(id: BlockId): Mono { return Mono.just(id) .flatMap { redis.del(key(it)) @@ -50,18 +53,17 @@ class BlocksRedisCache( * Add to cache. * Note that it returns Mono which must be subscribed to actually save */ - fun add(block: BlockJson): Mono { + fun add(block: BlockContainer): Mono { if (block.timestamp == null || block.hash == null) { return Mono.empty() } return Mono.just(block) .flatMap { block -> - - val data = objectMapper.writeValueAsString(block) + val data = String(block.json!!) //default caching time is age of the block, i.e. block create hour ago //keep for hour, but block create 10 seconds ago cache for 10 seconds, as it //still can be replaced in the blockchain - val age = Instant.now().epochSecond - block.timestamp.epochSecond + val age = Instant.now().epochSecond - block.timestamp!!.epochSecond val ttl = min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)) if (ttl > MIN_CACHE_TIME_SECONDS) { redis.setex(key(block.hash), ttl, data) @@ -82,7 +84,7 @@ class BlocksRedisCache( /** * Key in Redis */ - fun key(hash: BlockHash): String { + fun key(hash: BlockId): String { return "block:${chain.id}:${hash.toHex()}" } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt index e63027a5..2115c2ec 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt @@ -1,23 +1,25 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.reader.CompoundReader import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import reactor.core.publisher.TopicProcessor open class Caches( private val memBlocksByHash: BlocksMemCache, private val blocksByHeight: HeightCache, private val memTxsByHash: TxMemCache, private val redisBlocksByHash: BlocksRedisCache?, - private val redisTxsByHash: TxRedisCache? + private val redisTxsByHash: TxRedisCache?, + private val objectMapper: ObjectMapper ) { companion object { @@ -29,13 +31,13 @@ open class Caches( } @JvmStatic - fun default(): Caches { - return newBuilder().build() + fun default(objectMapper: ObjectMapper): Caches { + return newBuilder().setObjectMapper(objectMapper).build() } } - private val blocksByHash: Reader> - private val txsByHash: Reader + private val blocksByHash: Reader + private val txsByHash: Reader init { blocksByHash = if (redisBlocksByHash == null) { @@ -54,25 +56,25 @@ open class Caches( * Cache data that was just requested */ fun cacheRequested(data: Any) { - if (data is TransactionJson) { + if (data is TxContainer) { + cache(Tag.REQUESTED, data) + } else if (data is BlockContainer) { cache(Tag.REQUESTED, data) - } else if (data is BlockJson<*>) { - cache(Tag.REQUESTED, data as BlockJson) } } - fun cache(tag: Tag, tx: TransactionJson) { + fun cache(tag: Tag, tx: TxContainer) { //do not cache transactions that are not in a block yet - if (tx.blockHash == null) { + if (tx.blockId == null) { return } memTxsByHash.add(tx) - memBlocksByHash.get(tx.blockHash)?.let { block -> + memBlocksByHash.get(tx.blockId)?.let { block -> redisTxsByHash?.add(tx, block) } } - fun cache(tag: Tag, block: BlockJson) { + fun cache(tag: Tag, block: BlockContainer) { val job = ArrayList>() if (tag == Tag.LATEST) { //for LATEST data cache in memory, it will be short living so better to avoid Redis @@ -92,45 +94,60 @@ open class Caches( } } } else if (tag == Tag.REQUESTED) { - //shouldn't cache block json with transactions, separate txes and blocks with refs - val blockOnly = block.withoutTransactionDetails() - memBlocksByHash.add(blockOnly) - redisBlocksByHash?.add(blockOnly)?.let(job::add) + var blockOnlyContainer: BlockContainer? = null + var jsonValue: BlockJson<*>? = null + if (block.full) { + jsonValue = objectMapper.readValue>(block.json, BlockJson::class.java) + //shouldn't cache block json with transactions, separate txes and blocks with refs + val blockOnly = jsonValue.withoutTransactionDetails() + blockOnlyContainer = BlockContainer.from(blockOnly, objectMapper) + } else { + blockOnlyContainer = block + } + memBlocksByHash.add(blockOnlyContainer) + redisBlocksByHash?.add(blockOnlyContainer)?.let(job::add) // now cache only transactions - val transactions = block.transactions.filterIsInstance() - if (transactions.isNotEmpty()) { - transactions.forEach { cache(Tag.REQUESTED, it) } - if (redisTxsByHash != null) { - job.add(Flux.fromIterable(transactions).flatMap { redisTxsByHash.add(it, block) }.then()) + jsonValue?.let { jsonValue -> + val plainTransactions = jsonValue.transactions.filterIsInstance() + if (plainTransactions.isNotEmpty()) { + val transactions = plainTransactions.map { tx -> + TxContainer.from(tx, objectMapper) + } + transactions.forEach { + cache(Tag.REQUESTED, it) + } + if (redisTxsByHash != null) { + job.add(Flux.fromIterable(transactions).flatMap { redisTxsByHash.add(it, block) }.then()) + } } } } Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller } - fun getBlocksByHash(): Reader> { + fun getBlocksByHash(): Reader { return blocksByHash } - fun getBlockHashByHeight(): Reader { + fun getBlockHashByHeight(): Reader { return blocksByHeight } - fun getBlocksByHeight(): Reader> { + fun getBlocksByHeight(): Reader { return BlockByHeight(blocksByHeight, blocksByHash) } - fun getTxByHash(): Reader { + fun getTxByHash(): Reader { return txsByHash } - fun getFullBlocks(): Reader> { - return BlocksWithTxCache(blocksByHash, txsByHash) + fun getFullBlocks(): Reader { + return EthereumBlocksWithTxCache(objectMapper, blocksByHash, txsByHash) } - fun getFullBlocksByHeight(): Reader> { - return BlockByHeight(blocksByHeight, BlocksWithTxCache(blocksByHash, txsByHash)) + fun getFullBlocksByHeight(): Reader { + return BlockByHeight(blocksByHeight, EthereumBlocksWithTxCache(objectMapper, blocksByHash, txsByHash)) } enum class Tag { @@ -138,6 +155,7 @@ open class Caches( * Latest data produced by blockchain */ LATEST, + /** * Data requested by client */ @@ -150,6 +168,7 @@ open class Caches( private var txsByHash: TxMemCache? = null private var redisBlocksByHash: BlocksRedisCache? = null private var redisTxsByHash: TxRedisCache? = null + private var objectMapper: ObjectMapper? = null fun setBlockByHash(cache: BlocksMemCache): Builder { blocksByHash = cache @@ -176,6 +195,11 @@ open class Caches( return this } + fun setObjectMapper(value: ObjectMapper): Builder { + objectMapper = value + return this + } + fun build(): Caches { if (blocksByHash == null) { blocksByHash = BlocksMemCache() @@ -186,7 +210,10 @@ open class Caches( if (txsByHash == null) { txsByHash = TxMemCache() } - return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, redisBlocksByHash, redisTxsByHash) + if (objectMapper == null) { + throw IllegalStateException("ObjectMapper is not set") + } + return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, redisBlocksByHash, redisTxsByHash, objectMapper!!) } } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt index ed7915fa..ee2eb8b6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt @@ -61,6 +61,7 @@ class CachesFactory( private fun initCache(chain: Chain): Caches { val caches = Caches.newBuilder() + .setObjectMapper(objectMapper) redis?.let { redis -> caches.setBlockByHash(BlocksRedisCache(redis.reactive(), chain, objectMapper)) caches.setTxByHash(TxRedisCache(redis.reactive(), chain, objectMapper)) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksWithTxCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/EthereumBlocksWithTxCache.kt similarity index 53% rename from src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksWithTxCache.kt rename to src/main/kotlin/io/emeraldpay/dshackle/cache/EthereumBlocksWithTxCache.kt index 6cf7a9ae..f71bd20b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksWithTxCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/EthereumBlocksWithTxCache.kt @@ -1,8 +1,11 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionRefJson @@ -18,25 +21,27 @@ import reactor.core.publisher.Mono * If source block, with just transaction hashes is not available, it returns empty * If any of the expected block transactions is not available it returns empty */ -class BlocksWithTxCache( - private val blocks: Reader>, - private val txes: Reader -): Reader> { +class EthereumBlocksWithTxCache( + private val objectMapper: ObjectMapper, + private val blocks: Reader, + private val txes: Reader +) : Reader { companion object { - private val log = LoggerFactory.getLogger(BlocksWithTxCache::class.java) + private val log = LoggerFactory.getLogger(EthereumBlocksWithTxCache::class.java) } - override fun read(key: BlockHash): Mono> { + override fun read(key: BlockId): Mono { return blocks.read(key).flatMap { block -> - if (block.transactions == null || block.transactions.isEmpty()) { - // in fact it's not necessary to create a copy, made just for code clarity but may be performance loss + val block = objectMapper.readValue(block.json, BlockJson::class.java) as BlockJson + val fullBlock = if (block.transactions == null || block.transactions.isEmpty()) { + // in fact it's not necessary to create a copy, made just for code clarity but it may be a performance loss val fullBlock = BlockJson() BeanUtils.copyProperties(block, fullBlock) Mono.just(fullBlock) } else { Flux.fromIterable(block.transactions) - .map { it.hash } + .map { TxId.from(it.hash) } .flatMap { txes.read(it) } .collectList() .flatMap { list -> @@ -45,11 +50,20 @@ class BlocksWithTxCache( } else { val fullBlock = BlockJson() BeanUtils.copyProperties(block, fullBlock) - fullBlock.transactions = list + fullBlock.transactions = list.map { + objectMapper.readValue(it.json, TransactionJson::class.java) + } Mono.just(fullBlock) } } } + fullBlock + .map { block -> + BlockContainer(block.number, BlockId.from(block.hash), block.totalDifficulty, block.timestamp, true, + objectMapper.writeValueAsBytes(block), + block.transactions.map { tx -> TxId.from(tx) } + ) + } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt index f453a242..1436ec38 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt @@ -1,9 +1,8 @@ package io.emeraldpay.dshackle.cache +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import java.util.concurrent.ConcurrentHashMap @@ -13,25 +12,25 @@ import java.util.concurrent.ConcurrentHashMap */ open class HeightCache( val maxSize: Int = 256 -): Reader { +) : Reader { companion object { private val log = LoggerFactory.getLogger(HeightCache::class.java) } - private val heights = ConcurrentHashMap() + private val heights = ConcurrentHashMap() - override fun read(key: Long): Mono { + override fun read(key: Long): Mono { return Mono.justOrEmpty(heights[key]) } - open fun add(block: BlockJson): BlockHash? { - val existing = heights[block.number] - heights[block.number] = block.hash + open fun add(block: BlockContainer): BlockId? { + val existing = heights[block.height] + heights[block.height] = block.hash // evict old numbers if full - var dropHeight = block.number - maxSize - while (heights.size > maxSize && dropHeight < block.number) { + var dropHeight = block.height - maxSize + while (heights.size > maxSize && dropHeight < block.height) { heights.remove(dropHeight) dropHeight++ } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt index 77fa9488..f6b2b058 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt @@ -1,11 +1,10 @@ package io.emeraldpay.dshackle.cache +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.domain.TransactionId -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import java.util.concurrent.ConcurrentHashMap @@ -17,35 +16,35 @@ import java.util.concurrent.ConcurrentLinkedQueue open class TxMemCache( // usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default private val maxSize: Int = 125 * 32 -): Reader { +) : Reader { companion object { private val log = LoggerFactory.getLogger(TxMemCache::class.java) } - private val mapping = ConcurrentHashMap() - private val queue = ConcurrentLinkedQueue() + private val mapping = ConcurrentHashMap() + private val queue = ConcurrentLinkedQueue() - override fun read(key: TransactionId): Mono { + override fun read(key: TxId): Mono { return Mono.justOrEmpty(mapping[key]) } - open fun evict(block: BlockJson) { + open fun evict(block: BlockContainer) { block.transactions.forEach { - mapping.remove(it.hash) + mapping.remove(it) } } - open fun evict(block: BlockHash) { - val ids = mapping.filter { it.value.blockHash == block } + open fun evict(block: BlockId) { + val ids = mapping.filter { it.value.blockId == block } ids.forEach { mapping.remove(it.key) } } - open fun add(tx: TransactionJson) { + open fun add(tx: TxContainer) { //do not cache fresh transactions - if (tx.blockHash == null || tx.blockNumber == null) { + if (tx.blockId == null) { return } mapping.put(tx.hash, tx) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt index 478db4df..aa5e711d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt @@ -1,13 +1,14 @@ package io.emeraldpay.dshackle.cache import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.grpc.Chain -import io.infinitape.etherjar.domain.TransactionId -import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import io.lettuce.core.api.reactive.RedisReactiveCommands +import org.apache.commons.codec.binary.Base64 import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import reactor.util.function.Tuples @@ -22,35 +23,38 @@ class TxRedisCache( private val redis: RedisReactiveCommands, private val chain: Chain, private val objectMapper: ObjectMapper -): Reader { +) : Reader { companion object { private val log = LoggerFactory.getLogger(TxRedisCache::class.java) + // max caching time is 24 hours private const val MAX_CACHE_TIME_HOURS = 24L } - override fun read(key: TransactionId): Mono { + override fun read(key: TxId): Mono { return redis.get(key(key)) .map { data -> - objectMapper.readValue(data, TransactionJson::class.java) as TransactionJson + val json = data + val tx = objectMapper.readValue(json, TransactionJson::class.java) + TxContainer.from(tx, objectMapper) }.onErrorResume { Mono.empty() } } - fun evict(block: BlockJson): Mono { + fun evict(block: BlockContainer): Mono { return Mono.just(block) .map { block -> block.transactions.map { - key(it.hash) + key(it) }.toTypedArray() }.flatMap { keys -> redis.del(*keys) }.then() } - fun evict(id: TransactionId): Mono { + fun evict(id: TxId): Mono { return Mono.just(id) .flatMap { redis.del(key(it)) @@ -58,17 +62,17 @@ class TxRedisCache( .then() } - fun add(tx: TransactionJson, block: BlockJson): Mono { - if (tx.blockHash == null || block.hash == null || tx.blockHash != block.hash || block.timestamp == null) { + fun add(tx: TxContainer, block: BlockContainer): Mono { + if (tx.blockId == null || block.hash == null || tx.blockId != block.hash || block.timestamp == null) { return Mono.empty() } return Mono.just(Tuples.of(tx, block)) .flatMap { - val data = objectMapper.writeValueAsString(it.t1) + val data = String(it.t1.json!!) //default caching time is age of the block, i.e. block create hour ago //keep for hour, but block create 10 seconds ago cache for 10 seconds, as it //still can be replaced in the blockchain - val age = Instant.now().epochSecond - it.t2.timestamp.epochSecond + val age = Instant.now().epochSecond - it.t2.timestamp!!.epochSecond val ttl = min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS)) redis.setex(key(it.t1.hash), ttl, data) } @@ -85,7 +89,7 @@ class TxRedisCache( /** * Key in Redis */ - fun key(hash: TransactionId): String { + fun key(hash: TxId): String { return "tx:${chain.id}:${hash.toHex()}" } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt new file mode 100644 index 00000000..0e03bba8 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +import com.fasterxml.jackson.databind.ObjectMapper +import io.infinitape.etherjar.rpc.json.BlockJson +import io.infinitape.etherjar.rpc.json.TransactionJson +import io.infinitape.etherjar.rpc.json.TransactionRefJson +import java.math.BigInteger +import java.time.Instant + +class BlockContainer( + val height: Long, + val hash: BlockId, + val difficulty: BigInteger, + val timestamp: Instant, + val full: Boolean, + json: ByteArray?, + val transactions: List = emptyList() +) : SourceContainer(json) { + + companion object { + @JvmStatic + fun from(block: BlockJson<*>, objectMapper: ObjectMapper): BlockContainer { + val hasTransactions = block.transactions?.filterIsInstance()?.count() ?: 0 > 0 + return BlockContainer( + block.number, + BlockId.from(block), + block.totalDifficulty, + block.timestamp, + hasTransactions, + objectMapper.writeValueAsBytes(block), + block.transactions?.map { TxId.from(it.hash) } ?: emptyList() + ) + } + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + if (!super.equals(other)) return false + + other as BlockContainer + + if (height != other.height) return false + if (hash != other.hash) return false + if (difficulty != other.difficulty) return false + if (timestamp != other.timestamp) return false + if (full != other.full) return false + if (transactions != other.transactions) return false + + return true + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + height.hashCode() + result = 31 * result + hash.hashCode() + return result + } + + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt new file mode 100644 index 00000000..671730b7 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +import io.infinitape.etherjar.rpc.json.BlockJson + +class BlockId( + value: ByteArray +) : HashId(value) { + + companion object { + @JvmStatic + fun from(hash: io.infinitape.etherjar.domain.BlockHash): BlockId { + return BlockId(hash.bytes) + } + + @JvmStatic + fun from(block: BlockJson<*>): BlockId { + return from(block.hash) + } + + @JvmStatic + fun from(id: String): BlockId { + return from(io.infinitape.etherjar.domain.BlockHash.from(id)) + } + } + + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt new file mode 100644 index 00000000..206b45dc --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +open class HashId( + val value: ByteArray +) { + + companion object { + val HEX_DIGITS = "0123456789abcdef".toCharArray() + } + + override fun toString(): String { + return toHex() + } + + fun toHex(): String { + val hex = CharArray(value.size * 2 + 2) + hex[0] = '0' + hex[1] = 'x' + var i = 0 + var j = 2 + while (i < value.size) { + hex[j++] = HEX_DIGITS[0xF0 and value[i].toInt() ushr 4] + hex[j++] = HEX_DIGITS[0x0F and value[i].toInt()] + i++ + } + return String(hex) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is HashId) return false + + if (!value.contentEquals(other.value)) return false + + return true + } + + override fun hashCode(): Int { + return value.contentHashCode() + } + + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt new file mode 100644 index 00000000..73108460 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream + +class RawJsonBuilder { + + companion object { + private val log = LoggerFactory.getLogger(RawJsonBuilder::class.java) + + private val START = "{\"jsonrpc\":\"2.0\"".toByteArray() + private val ID_START = "\"id\":".toByteArray() + private val RESULT_START = "\"result\":".toByteArray() + private val COMMA = ",".toByteArray() + private val END = "}".toByteArray() + } + + fun write(id: Int, data: ByteArray): ByteArray { + val buf = ByteArrayOutputStream(data.size + 100) + buf.write(START) + buf.write(COMMA) + buf.write(ID_START) + buf.write(id.toString().toByteArray()); + buf.write(COMMA) + buf.write(RESULT_START) + buf.write(data) + buf.write(END) + + return buf.toByteArray() + } + + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt new file mode 100644 index 00000000..7294a676 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +abstract class SourceContainer( + val json: ByteArray? +) { + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is SourceContainer) return false + + if (json != null) { + if (other.json == null) return false + if (!json.contentEquals(other.json)) return false + } else if (other.json != null) return false + + return true + } + + override fun hashCode(): Int { + return json?.contentHashCode() ?: 0 + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt new file mode 100644 index 00000000..494f1966 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +import com.fasterxml.jackson.databind.ObjectMapper +import io.infinitape.etherjar.rpc.json.TransactionJson + +class TxContainer( + val height: Long, + val hash: TxId, + val blockId: BlockId?, + json: ByteArray? +) : SourceContainer(json) { + + companion object { + @JvmStatic + fun from(tx: TransactionJson, objectMapper: ObjectMapper): TxContainer { + return TxContainer( + tx.blockNumber, + TxId.from(tx.hash), + BlockId.from(tx.blockHash), + objectMapper.writeValueAsBytes(tx) + ) + } + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + if (!super.equals(other)) return false + + other as TxContainer + + if (height != other.height) return false + if (hash != other.hash) return false + if (blockId != other.blockId) return false + + return true + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + height.hashCode() + result = 31 * result + hash.hashCode() + return result + } + + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt new file mode 100644 index 00000000..1dd1b976 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2020 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.data + +import io.infinitape.etherjar.domain.TransactionId +import io.infinitape.etherjar.rpc.json.TransactionJson + +class TxId( + value: ByteArray +) : HashId(value) { + + companion object { + @JvmStatic + fun from(id: TransactionId): TxId { + return TxId(id.bytes) + } + + @JvmStatic + fun from(tx: TransactionJson): TxId { + return from(tx.hash) + } + + @JvmStatic + fun from(id: String): TxId { + return from(TransactionId.from(id)) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt index 34920888..151c4df2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt @@ -17,30 +17,27 @@ package io.emeraldpay.dshackle.quorum import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.RpcException -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson open class AlwaysQuorum: CallQuorum { private var resolved = false private var result: ByteArray? = null - override fun init(head: Head>) { + override fun init(head: Head) { } override fun isResolved(): Boolean { return resolved } - override fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean { + override fun record(response: ByteArray, upstream: Upstream<*>): Boolean { result = response resolved = true return true } - override fun record(error: RpcException, upstream: Upstream<*, *>) { + override fun record(error: RpcException, upstream: Upstream<*>) { } override fun getResult(): ByteArray? { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt index b33565ba..445bd10d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt @@ -17,10 +17,7 @@ package io.emeraldpay.dshackle.quorum import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.JacksonRpcConverter -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson open class BroadcastQuorum( jacksonRpcConverter: JacksonRpcConverter, @@ -31,7 +28,7 @@ open class BroadcastQuorum( private var txid: String? = null private var calls = 0 - override fun init(head: Head>) { + override fun init(head: Head) { } override fun isResolved(): Boolean { @@ -42,7 +39,7 @@ open class BroadcastQuorum( return result } - override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*, *>) { + override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*>) { calls++ if (txid == null && responseValue != null) { txid = responseValue @@ -50,7 +47,7 @@ open class BroadcastQuorum( } } - override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) { + override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) { // can be "message: known transaction: TXID", "Transaction with the same hash was already imported" or "message: Nonce too low" calls++ if (result == null) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt index ac20594b..e31e95f6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt @@ -27,11 +27,11 @@ import java.util.function.Predicate interface CallQuorum { - fun init(head: Head>) + fun init(head: Head) fun isResolved(): Boolean - fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean - fun record(error: RpcException, upstream: Upstream<*, *>) + fun record(response: ByteArray, upstream: Upstream<*>): Boolean + fun record(error: RpcException, upstream: Upstream<*>) fun getResult(): ByteArray? companion object { @@ -41,8 +41,8 @@ interface CallQuorum { } } - fun asReducer(): BiFunction>, CallQuorum> { - return BiFunction>, CallQuorum> { a, b -> + fun asReducer(): BiFunction>, CallQuorum> { + return BiFunction>, CallQuorum> { a, b -> a.record(b.t1, b.t2) return@BiFunction a } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt index 7f064ce7..059d845b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt @@ -17,11 +17,8 @@ package io.emeraldpay.dshackle.quorum import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.JacksonRpcConverter import io.infinitape.etherjar.rpc.RpcException -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson open class NonEmptyQuorum( jacksonRpcConverter: JacksonRpcConverter, @@ -31,14 +28,14 @@ open class NonEmptyQuorum( private var result: ByteArray? = null private var tries: Int = 0 - override fun init(head: Head>) { + override fun init(head: Head) { } override fun isResolved(): Boolean { return result != null || tries >= maxTries } - override fun recordValue(response: ByteArray, responseValue: Any?, upstream: Upstream<*, *>) { + override fun recordValue(response: ByteArray, responseValue: Any?, upstream: Upstream<*>) { tries++ if (responseValue != null) { result = response @@ -49,10 +46,10 @@ open class NonEmptyQuorum( return result } - override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) { + override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) { } - override fun record(error: RpcException, upstream: Upstream<*, *>) { + override fun record(error: RpcException, upstream: Upstream<*>) { } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt index 954bae99..16e9f577 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt @@ -17,12 +17,9 @@ package io.emeraldpay.dshackle.quorum import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.hex.HexQuantity import io.infinitape.etherjar.rpc.JacksonRpcConverter import io.infinitape.etherjar.rpc.RpcException -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock @@ -37,7 +34,7 @@ open class NonceQuorum( private var receivedTimes = 0 private var errors = 0 - override fun init(head: Head>) { + override fun init(head: Head) { } override fun isResolved(): Boolean { @@ -46,7 +43,7 @@ open class NonceQuorum( } } - override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*, *>) { + override fun recordValue(response: ByteArray, responseValue: String?, upstream: Upstream<*>) { val value = responseValue?.let { str -> HexQuantity.from(str).value.toLong() } @@ -65,11 +62,11 @@ open class NonceQuorum( return result } - override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) { + override fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) { errors++ } - override fun record(error: RpcException, upstream: Upstream<*, *>) { + override fun record(error: RpcException, upstream: Upstream<*>) { errors++ } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt index cc3405e4..3226bc67 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt @@ -17,24 +17,21 @@ package io.emeraldpay.dshackle.quorum import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.RpcException -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import java.util.concurrent.atomic.AtomicReference class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum { private val result: AtomicReference = AtomicReference() - override fun init(head: Head>) { + override fun init(head: Head) { } override fun isResolved(): Boolean { return result.get() != null } - override fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean { + override fun record(response: ByteArray, upstream: Upstream<*>): Boolean { val lagging = upstream.getLag() > maxLag if (!lagging) { result.set(response) @@ -43,7 +40,7 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum { return false } - override fun record(error: RpcException, upstream: Upstream<*, *>) { + override fun record(error: RpcException, upstream: Upstream<*>) { } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt index 9ae8cab9..413a39e3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt @@ -31,7 +31,7 @@ abstract class ValueAwareQuorum( return jacksonRpcConverter.fromJson(response.inputStream(), clazz) } - override fun record(response: ByteArray, upstream: Upstream<*, *>): Boolean { + override fun record(response: ByteArray, upstream: Upstream<*>): Boolean { try { val value = extractValue(response, clazz) recordValue(response, value, upstream) @@ -43,12 +43,12 @@ abstract class ValueAwareQuorum( return isResolved(); } - override fun record(error: RpcException, upstream: Upstream<*, *>) { + override fun record(error: RpcException, upstream: Upstream<*>) { recordError(null, error.rpcMessage, upstream) } - abstract fun recordValue(response: ByteArray, responseValue: T?, upstream: Upstream<*, *>) + abstract fun recordValue(response: ByteArray, responseValue: T?, upstream: Upstream<*>) - abstract fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*, *>) + abstract fun recordError(response: ByteArray?, errorMessage: String?, upstream: Upstream<*>) } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt index fc8baef3..6cc41c17 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt @@ -95,10 +95,10 @@ open class NativeCall( val upstream = upstreams.getUpstream(chain) ?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain))) - return prepareCall(request, upstream as AggregatedUpstream>) + return prepareCall(request, upstream as AggregatedUpstream) } - fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: AggregatedUpstream>): Flux> { + fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: AggregatedUpstream): Flux> { return request.itemsList.toFlux().map { val method = it.method val params = it.payload.toStringUtf8() @@ -205,7 +205,7 @@ open class NativeCall( } open class CallContext(val id: Int, - val upstream: AggregatedUpstream>, + val upstream: AggregatedUpstream, val matcher: Selector.Matcher, val callQuorum: CallQuorum, val payload: T) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt index c6c289fe..a10c07ea 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt @@ -19,6 +19,7 @@ import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.BlockchainType +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.Upstreams import io.emeraldpay.grpc.Chain import io.infinitape.etherjar.rpc.json.BlockJson @@ -51,23 +52,19 @@ class StreamHead( } } - fun asProto(chain: Chain, block: Any): BlockchainOuterClass.ChainHead { + fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead { if (BlockchainType.fromBlockchain(chain) == BlockchainType.ETHEREUM) { - if (BlockJson::class.java.isAssignableFrom(block.javaClass)) { - return asEthereumProto(chain, block as BlockJson) - } else { - throw IllegalArgumentException("Invalid block type: ${block.javaClass}") - } + return asEthereumProto(chain, block) } throw IllegalArgumentException("Unsupported blockchain ${chain}") } - fun asEthereumProto(chain: Chain, block: BlockJson): BlockchainOuterClass.ChainHead { + fun asEthereumProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead { return BlockchainOuterClass.ChainHead.newBuilder() .setChainValue(chain.id) - .setHeight(block.number) - .setTimestamp(block.timestamp.toEpochMilli()) - .setWeight(ByteString.copyFrom(block.totalDifficulty.toByteArray())) + .setHeight(block.height) + .setTimestamp(block.timestamp!!.toEpochMilli()) + .setWeight(ByteString.copyFrom(block.difficulty.toByteArray())) .setBlockId(block.hash.toHex().substring(2)) .build() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt index 5fc42eee..96bacdde 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt @@ -45,7 +45,7 @@ class SubscribeStatus( } } - fun chainStatus(chain: Chain, ups: List>): BlockchainOuterClass.ChainStatus { + fun chainStatus(chain: Chain, ups: List>): BlockchainOuterClass.ChainStatus { val available = ups.map { u -> u.getStatus() }.min() ?: UpstreamAvailability.UNAVAILABLE @@ -59,6 +59,6 @@ class SubscribeStatus( .build() } - class ChainSubscription(val chain: Chain, val up: AggregatedUpstream<*, *>, val avail: UpstreamAvailability) + class ChainSubscription(val chain: Chain, val up: AggregatedUpstream<*>, val avail: UpstreamAvailability) } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt index 555501b0..11d2a902 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt @@ -187,7 +187,7 @@ class TrackEthereumAddress( } fun getBalance(addr: SimpleAddress): Mono { - val up = upstreams.getUpstream(addr.chain) as AggregatedUpstream>? + val up = upstreams.getUpstream(addr.chain) as AggregatedUpstream? ?: return Mono.error(SilentException.UnsupportedBlockchain(addr.chain)) return up.getApi(Selector.empty) .flatMap { api -> api.executeAndConvert(Commands.eth().getBalance(addr.address, BlockTag.LATEST)) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt index ac2b9981..c15f0048 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt @@ -213,7 +213,7 @@ class TrackEthereumTx( } private fun loadWeight(tx: TxDetails): Mono { - val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream>? + val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream? ?: return Mono.error(SilentException.UnsupportedBlockchain(tx.chain)) return upstream.getApi(Selector.empty) .flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(tx.status.blockHash)) } @@ -224,7 +224,7 @@ class TrackEthereumTx( } } - fun updateFromBlock(upstream: Upstream>, tx: TxDetails, it: TransactionJson): Mono { + fun updateFromBlock(upstream: Upstream, tx: TxDetails, it: TransactionJson): Mono { return if (it.blockNumber != null && it.blockHash != null && it.blockHash != ZERO_BLOCK) { val updated = tx.withStatus( blockHash = it.blockHash, @@ -235,11 +235,11 @@ class TrackEthereumTx( ) upstream.getHead().getFlux().next().map { head -> val height = updated.status.height - if (height == null || head.number < height) { + if (height == null || head.height < height) { updated } else { updated.withStatus( - confirmations = head.number - height + 1 + confirmations = head.height - height + 1 ) } }.doOnError { t -> @@ -255,7 +255,7 @@ class TrackEthereumTx( private fun checkForUpdate(tx: TxDetails): Mono { val initialStatus = tx.status - val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream>? + val upstream = upstreams.getUpstream(tx.chain) as AggregatedUpstream? ?: return Mono.error(SilentException.UnsupportedBlockchain(tx.chain)) val execution = upstream.getApi(Selector.empty) .flatMap { api -> api.executeAndConvert(Commands.eth().getTransaction(tx.txid)) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index e159053c..de3ade53 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -144,7 +144,8 @@ open class ConfiguredUpstreams( val wsApi = EthereumWs( endpoint.url, endpoint.origin ?: URI("http://localhost"), - rpcApi!! + rpcApi!!, + objectMapper ) endpoint.basicAuth?.let { auth -> wsApi.basicAuth = auth @@ -159,7 +160,8 @@ open class ConfiguredUpstreams( config.id!!, chain, rpcApi!!, wsApi, options, QuorumForLabels.QuorumItem(1, config.labels), - methods) + methods, + objectMapper) ethereumUpstream.start() currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED)) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt index c2535ae1..cee4ad6e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt @@ -31,7 +31,7 @@ class UpstreamChange( /** * Corresponding upstream */ - val upstream: Upstream<*, *>, + val upstream: Upstream<*>, /** * Type of the change */ diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AggregatedUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AggregatedUpstream.kt index a2061497..30d803ac 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AggregatedUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AggregatedUpstream.kt @@ -34,18 +34,18 @@ import kotlin.concurrent.withLock /** * Aggregation of multiple upstreams responding to a single blockchain */ -abstract class AggregatedUpstream( +abstract class AggregatedUpstream( private val objectMapper: ObjectMapper, val caches: Caches -) : Upstream, Lifecycle { +) : Upstream, Lifecycle { private var cacheSubscription: Disposable? = null - var cache: CachingEthereumApi = CachingEthereumApi.empty() + var cache: CachingEthereumApi = CachingEthereumApi.empty(objectMapper) private val reconfigLock = ReentrantLock() private var callMethods: CallMethods? = null - abstract fun getAll(): List> - abstract fun addUpstream(upstream: Upstream) + abstract fun getAll(): List> + abstract fun addUpstream(upstream: Upstream) abstract fun getApis(matcher: Selector.Matcher): ApiSource fun onUpstreamsUpdated() { @@ -101,12 +101,12 @@ abstract class AggregatedUpstream( // -------------------------------------------------------------------------------------------------------- - class UpstreamStatus(val upstream: Upstream, val status: UpstreamAvailability, val ts: Instant = Instant.now()) + class UpstreamStatus(val upstream: Upstream, val status: UpstreamAvailability, val ts: Instant = Instant.now()) - class FilterBestAvailability() : Predicate> { - private val lastRef = AtomicReference>() + class FilterBestAvailability() : Predicate { + private val lastRef = AtomicReference() - override fun test(t: UpstreamStatus<*>): Boolean { + override fun test(t: UpstreamStatus): Boolean { val last = lastRef.get() val changed = last == null || t.status > last.status diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingEthereumApi.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingEthereumApi.kt index 4893069e..d60daf40 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingEthereumApi.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CachingEthereumApi.kt @@ -17,13 +17,11 @@ package io.emeraldpay.dshackle.upstream import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.cache.Caches +import io.emeraldpay.dshackle.data.* import io.emeraldpay.dshackle.upstream.ethereum.EmptyEthereumHead import io.emeraldpay.dshackle.upstream.ethereum.EthereumApi import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.hex.HexQuantity -import io.infinitape.etherjar.rpc.json.ResponseJson import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import java.math.BigInteger @@ -42,11 +40,13 @@ open class CachingEthereumApi( * Create caching API with empty memory-only cache */ @JvmStatic - fun empty(): CachingEthereumApi { - return CachingEthereumApi(ObjectMapper(), Caches.default(), EmptyEthereumHead()) + fun empty(objectMapper: ObjectMapper): CachingEthereumApi { + return CachingEthereumApi(objectMapper, Caches.default(objectMapper), EmptyEthereumHead()) } } + private val rawJsonBuilder = RawJsonBuilder() + private val cacheBlocks = caches.getBlocksByHash() private val cacheBlocksByHeight = caches.getBlocksByHeight() private val cacheTx = caches.getTxByHash() @@ -62,7 +62,7 @@ open class CachingEthereumApi( cacheBlocks } Mono.just(params[0]) - .map { BlockHash.from(it as String) } + .map { BlockId.from(it as String) } .flatMap(cache::read) .transform(converter(id)) .transform(finalizer()) @@ -93,14 +93,15 @@ open class CachingEthereumApi( return when (method) { "eth_blockNumber" -> head.getFlux().next() - .map { HexQuantity.from(it.number).toHex() } - .map(toJson(id)) + .map { HexQuantity.from(it.height).toHex() } + .map { objectMapper.writeValueAsBytes(it) } + .map(bytesToJson(id)) "eth_getBlockByHash" -> readBlockByHash(id, method, params) "eth_getBlockByNumber" -> readBlockByNumber(id, method, params) "eth_getTransactionByHash" -> if (params.size == 1) Mono.just(params[0]) - .map { TransactionId.from(it as String) } + .map { TxId.from(it as String) } .flatMap(cacheTx::read) .transform(converter(id)) .transform(finalizer()) @@ -113,9 +114,9 @@ open class CachingEthereumApi( /** * Convert to JSON RPC response */ - fun converter(id: Int): Function, out Mono> { + fun converter(id: Int): Function, out Mono> { return Function { mono -> - mono.map(toJson(id)) + mono.map(containerToJson(id)) } } @@ -131,12 +132,15 @@ open class CachingEthereumApi( } } - fun toJson(id: Int): Function { + fun bytesToJson(id: Int): Function { return Function { data -> - val resp = ResponseJson() - resp.id = id - resp.result = data - objectMapper.writer().writeValueAsBytes(resp) + rawJsonBuilder.write(id, data) + } + } + + fun containerToJson(id: Int): Function { + return Function { data -> + rawJsonBuilder.write(id, data.json!!) } } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainUpstreams.kt index 75788f70..aedfdfc2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainUpstreams.kt @@ -26,24 +26,24 @@ import reactor.core.publisher.Mono /** * General interface to upstream(s) to a single chain */ -abstract class ChainUpstreams( +abstract class ChainUpstreams( val chain: Chain, - private val upstreams: MutableList>, + private val upstreams: MutableList>, caches: Caches, objectMapper: ObjectMapper -) : AggregatedUpstream(objectMapper, caches), Lifecycle { +) : AggregatedUpstream(objectMapper, caches), Lifecycle { private val log = LoggerFactory.getLogger(ChainUpstreams::class.java) private var seq = 0 - protected var lagObserver: HeadLagObserver? = null + protected var lagObserver: HeadLagObserver? = null private var subscription: Disposable? = null open fun init() { onUpstreamsUpdated() } - abstract fun updateHead(): Head - abstract fun setHead(head: Head) + abstract fun updateHead(): Head + abstract fun setHead(head: Head) override fun getId(): String { return "!all:${chain.chainCode}" @@ -72,11 +72,11 @@ abstract class ChainUpstreams( lagObserver?.stop() } - override fun getAll(): List> { + override fun getAll(): List> { return upstreams } - override fun addUpstream(upstream: Upstream) { + override fun addUpstream(upstream: Upstream) { upstreams.add(upstream) setHead(updateHead()) onUpstreamsUpdated() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentUpstreams.kt index f88b5089..0ad5cc24 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentUpstreams.kt @@ -46,7 +46,7 @@ class CurrentUpstreams( private val log = LoggerFactory.getLogger(CurrentUpstreams::class.java) - private val chainMapping = ConcurrentHashMap>() + private val chainMapping = ConcurrentHashMap>() private val chainsBus = TopicProcessor.create() private val callTargets = HashMap() private val updateLock = ReentrantLock() @@ -55,8 +55,8 @@ class CurrentUpstreams( updateLock.withLock { val chain = change.chain val up = change.upstream - .cast(EthereumUpstream::class.java, EthereumApi::class.java, BlockJson::class.java) as Upstream> - val current = chainMapping[chain] as ChainUpstreams>? + .cast(EthereumUpstream::class.java, EthereumApi::class.java) as Upstream + val current = chainMapping[chain] as ChainUpstreams? if (change.type == UpstreamChange.ChangeType.REMOVED) { current?.removeUpstream(up.getId()) log.info("Upstream ${change.upstream.getId()} with chain $chain has been removed") @@ -84,7 +84,7 @@ class CurrentUpstreams( } } - override fun getUpstream(chain: Chain): AggregatedUpstream<*, *>? { + override fun getUpstream(chain: Chain): AggregatedUpstream<*>? { return chainMapping[chain] } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt index 9cfc6d26..95a196cc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt @@ -19,10 +19,10 @@ import reactor.core.publisher.Flux import reactor.core.publisher.TopicProcessor import java.util.concurrent.atomic.AtomicReference -abstract class DefaultUpstream( +abstract class DefaultUpstream( defaultLag: Long, defaultAvail: UpstreamAvailability -) : Upstream { +) : Upstream { constructor() : this(Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt index 11c577fe..223d4729 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt @@ -26,7 +26,7 @@ import kotlin.math.roundToLong import kotlin.random.Random class FilteredApis( - allUpstreams: List>, + allUpstreams: List>, private val matcher: Selector.Matcher, pos: Int, private val repeatLimit: Long, @@ -38,15 +38,15 @@ class FilteredApis( private const val MAX_WAIT_MILLIS = 5000L } - constructor(allUpstreams: List>, + constructor(allUpstreams: List>, matcher: Selector.Matcher, pos: Int) : this(allUpstreams, matcher, pos, 10, 7) - constructor(allUpstreams: List>, + constructor(allUpstreams: List>, matcher: Selector.Matcher) : this(allUpstreams, matcher, 0, 10, 10) private val delay: Int - private val upstreams: List> + private val upstreams: List> private val control = EmitterProcessor.create(32, false) @@ -81,7 +81,7 @@ class FilteredApis( }.let { Flux.concat(it) } Flux.concat(first, retries) - .filter(Upstream::isAvailable) + .filter(Upstream::isAvailable) .filter(matcher::matches) .flatMap { it.getApi(matcher) } .zipWith(control).map { it.t1 } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt index 8914c2ad..142d0d6b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt @@ -15,9 +15,9 @@ */ package io.emeraldpay.dshackle.upstream +import io.emeraldpay.dshackle.data.BlockContainer import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -interface Head { - fun getFlux(): Flux +interface Head { + fun getFlux(): Flux } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt index c6ca3d6f..67fc7f1f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.upstream +import io.emeraldpay.dshackle.data.BlockContainer import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import reactor.core.Disposable @@ -26,9 +27,9 @@ import reactor.util.function.Tuples * Observer group of upstreams and defined a distance in blocks (lag) between a leader (best height/difficulty) and * other upstreams. */ -abstract class HeadLagObserver( - private val master: Head, - private val followers: Collection> +abstract class HeadLagObserver( + private val master: Head, + private val followers: Collection> ) : Lifecycle { private val log = LoggerFactory.getLogger(HeadLagObserver::class.java) @@ -56,7 +57,7 @@ abstract class HeadLagObserver( } } - fun probeFollowers(top: B): Flux>> { + fun probeFollowers(top: BlockContainer): Flux>> { return Flux.fromIterable(followers) .parallel(followers.size) .flatMap { mapLagging(top, it, getCurrentBlocks(it)) } @@ -64,9 +65,9 @@ abstract class HeadLagObserver( .onErrorContinue { t, _ -> log.warn("Failed to update lagging distance", t) } } - abstract fun getCurrentBlocks(up: Upstream): Flux + abstract fun getCurrentBlocks(up: Upstream): Flux - fun mapLagging(top: B, up: Upstream, blocks: Flux): Flux>> { + fun mapLagging(top: BlockContainer, up: Upstream, blocks: Flux): Flux>> { return blocks .map { extractDistance(top, it) } .takeUntil { lag -> lag <= 0L } @@ -76,9 +77,9 @@ abstract class HeadLagObserver( } } - abstract fun extractDistance(top: B, curr: B): Long + abstract fun extractDistance(top: BlockContainer, curr: BlockContainer): Long - fun forkDistance(top: B, curr: B): Long { + fun forkDistance(top: BlockContainer, curr: BlockContainer): Long { //TODO look for common ancestor? though it may be a corruption return 6 } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt index 6b521a7e..9620f4be 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt @@ -95,13 +95,13 @@ class Selector { } interface Matcher { - fun matches(up: Upstream): Boolean + fun matches(up: Upstream): Boolean } class MultiMatcher( private val matchers: Collection ): Matcher { - override fun matches(up: Upstream): Boolean { + override fun matches(up: Upstream): Boolean { return matchers.all { it.matches(up) } } @@ -113,13 +113,13 @@ class Selector { class MethodMatcher( val method: String ): Matcher { - override fun matches(up: Upstream): Boolean { + override fun matches(up: Upstream): Boolean { return up.getMethods().isAllowed(method) } } abstract class LabelSelectorMatcher: Matcher { - override fun matches(up: Upstream): Boolean { + override fun matches(up: Upstream): Boolean { return up.getLabels().any(this::matches) } @@ -128,7 +128,7 @@ class Selector { } class EmptyMatcher: Matcher { - override fun matches(up: Upstream): Boolean { + override fun matches(up: Upstream): Boolean { return true } } @@ -143,7 +143,7 @@ class Selector { return null } - override fun matches(up: Upstream): Boolean { + override fun matches(up: Upstream): Boolean { return true } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt index 33d213da..9c20020a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt @@ -17,16 +17,14 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.upstream.calls.CallMethods -import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi -import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead import reactor.core.publisher.Flux import reactor.core.publisher.Mono -interface Upstream { +interface Upstream { fun isAvailable(): Boolean fun getStatus(): UpstreamAvailability fun observeStatus(): Flux - fun getHead(): Head + fun getHead(): Head fun getApi(matcher: Selector.Matcher): Mono fun getOptions(): UpstreamsConfig.Options fun setLag(lag: Long) @@ -35,5 +33,5 @@ interface Upstream { fun getMethods(): CallMethods fun getId(): String - fun , TA : UpstreamApi, BA> cast(selfType: Class, upstreamType: Class, blockType: Class): T + fun , TA : UpstreamApi> cast(selfType: Class, upstreamType: Class): T } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt index 13db5657..f410f53f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstreams.kt @@ -20,7 +20,7 @@ import io.emeraldpay.grpc.Chain import reactor.core.publisher.Flux interface Upstreams { - fun getUpstream(chain: Chain): AggregatedUpstream<*, *>? + fun getUpstream(chain: Chain): AggregatedUpstream<*>? fun getAvailable(): List fun observeChains(): Flux fun getDefaultMethods(chain: Chain): CallMethods diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt index df9c9509..487119d3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt @@ -1,8 +1,6 @@ package io.emeraldpay.dshackle.upstream.ethereum -import io.infinitape.etherjar.domain.TransactionId -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson +import io.emeraldpay.dshackle.data.BlockContainer import org.slf4j.LoggerFactory import reactor.core.Disposable import reactor.core.publisher.Flux @@ -13,39 +11,39 @@ import java.util.concurrent.atomic.AtomicReference open class DefaultEthereumHead: EthereumHead { private val log = LoggerFactory.getLogger(DefaultEthereumHead::class.java) - private val head = AtomicReference>(null) - private val stream: TopicProcessor> = TopicProcessor.create() + private val head = AtomicReference(null) + private val stream: TopicProcessor = TopicProcessor.create() - fun follow(source: Flux>): Disposable { + fun follow(source: Flux): Disposable { return source.distinctUntilChanged { it.hash }.filter { block -> val curr = head.get() - curr == null || curr.totalDifficulty < block.totalDifficulty + curr == null || curr.difficulty < block.difficulty } - .subscribe { block -> - val prev = head.getAndUpdate { curr -> - if (curr == null || curr.totalDifficulty < block.totalDifficulty) { - block - } else { - curr - } + .subscribe { block -> + val prev = head.getAndUpdate { curr -> + if (curr == null || curr.difficulty < block.difficulty) { + block + } else { + curr + } } if (prev == null || prev.hash != block.hash) { - log.debug("New block ${block.number} ${block.hash}") + log.debug("New block ${block.height} ${block.hash}") stream.onNext(block) } } } - override fun getFlux(): Flux> { + override fun getFlux(): Flux { return Flux.merge( Mono.justOrEmpty(head.get()), Flux.from(stream) ).onBackpressureLatest() } - fun getCurrent(): BlockJson? { + fun getCurrent(): BlockContainer? { return head.get() } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EmptyEthereumHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EmptyEthereumHead.kt index e276f38c..d715abd4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EmptyEthereumHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EmptyEthereumHead.kt @@ -15,14 +15,12 @@ */ package io.emeraldpay.dshackle.upstream.ethereum -import io.infinitape.etherjar.domain.TransactionId -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson +import io.emeraldpay.dshackle.data.BlockContainer import reactor.core.publisher.Flux class EmptyEthereumHead : EthereumHead { - override fun getFlux(): Flux> { + override fun getFlux(): Flux { return Flux.empty() } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumApi.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumApi.kt index 56c58a6e..4a93d139 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumApi.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumApi.kt @@ -34,7 +34,7 @@ abstract class EthereumApi( } private val jacksonRpcConverter = JacksonRpcConverter(objectMapper) - var upstream: Upstream>? = null + var upstream: Upstream? = null fun execute(rpcCall: RpcCall): Mono { return execute(0, rpcCall.method, rpcCall.params as List) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainUpstreams.kt index db324c10..239b790e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainUpstreams.kt @@ -33,7 +33,7 @@ class EthereumChainUpstreams( val upstreams: MutableList, caches: Caches, objectMapper: ObjectMapper -) : ChainUpstreams>(chain, upstreams as MutableList>>, caches, objectMapper) { +) : ChainUpstreams(chain, upstreams as MutableList>, caches, objectMapper) { companion object { private val log = LoggerFactory.getLogger(EthereumChainUpstreams::class.java) @@ -56,7 +56,7 @@ class EthereumChainUpstreams( return head!! } - override fun setHead(head: Head>) { + override fun setHead(head: Head) { this.head = head as EthereumHead } @@ -76,7 +76,7 @@ class EthereumChainUpstreams( val newHead = EthereumHeadMerge(upstreams.map { it.getHead() }).apply { this.start() } - val lagObserver = EthereumHeadLagObserver(newHead, upstreams).apply { + val lagObserver = EthereumHeadLagObserver(newHead, upstreams as Collection>).apply { this.start() } this.lagObserver = lagObserver @@ -93,7 +93,7 @@ class EthereumChainUpstreams( override fun printStatus() { var height: Long? = null try { - height = getHead().getFlux().next().block(Duration.ofSeconds(1))?.number + height = getHead().getFlux().next().block(Duration.ofSeconds(1))?.height } catch (e: IllegalStateException) { //timout } catch (e: Exception) { @@ -110,18 +110,14 @@ class EthereumChainUpstreams( } @SuppressWarnings("unchecked") - override fun , TA : UpstreamApi, BA> cast(selfType: Class, upstreamType: Class, blockType: Class): T { + override fun , TA : UpstreamApi> cast(selfType: Class, upstreamType: Class): T { if (!selfType.isAssignableFrom(this.javaClass)) { throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") } if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) { throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType") } - if (!blockType.isAssignableFrom(BlockJson::class.java)) { - throw ClassCastException("Cannot cast ${BlockJson::class.java} to $blockType") - } return this as T } - } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHead.kt index 7841abe9..0f5719c7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHead.kt @@ -20,5 +20,5 @@ import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionRefJson -interface EthereumHead: Head> { +interface EthereumHead : Head { } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt index 0664f1d4..47c88390 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt @@ -17,31 +17,30 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.upstream.HeadLagObserver import io.emeraldpay.dshackle.upstream.Upstream -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson +import io.emeraldpay.dshackle.data.BlockContainer import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import java.time.Duration class EthereumHeadLagObserver( master: EthereumHead, - followers: Collection>> -) : HeadLagObserver>(master, followers) { + followers: Collection> +) : HeadLagObserver(master, followers) { companion object { private val log = LoggerFactory.getLogger(EthereumHeadLagObserver::class.java) } - override fun getCurrentBlocks(up: Upstream>): Flux> { + override fun getCurrentBlocks(up: Upstream): Flux { val head = up.getHead() - return Flux.from(head.getFlux()).take(Duration.ofSeconds(1)) + return head.getFlux().take(Duration.ofSeconds(1)) } - override fun extractDistance(top: BlockJson, curr: BlockJson): Long { + override fun extractDistance(top: BlockContainer, curr: BlockContainer): Long { return when { - curr.number > top.number -> if (curr.totalDifficulty >= top.totalDifficulty) 0 else forkDistance(top, curr) - curr.number == top.number -> if (curr.totalDifficulty == top.totalDifficulty) 0 else forkDistance(top, curr) - else -> top.number - curr.number + curr.height > top.height -> if (curr.difficulty >= top.difficulty) 0 else forkDistance(top, curr) + curr.height == top.height -> if (curr.difficulty == top.difficulty) 0 else forkDistance(top, curr) + else -> top.height - curr.height } } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt index 1e092f16..c1553572 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt @@ -15,18 +15,10 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Defaults -import io.emeraldpay.dshackle.cache.Caches -import io.emeraldpay.dshackle.cache.CachesEnabled -import io.emeraldpay.dshackle.reader.EmptyReader -import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.CachingEthereumApi -import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.rpc.Batch +import io.emeraldpay.dshackle.data.BlockContainer import io.infinitape.etherjar.rpc.Commands -import io.infinitape.etherjar.rpc.ReactorBatch -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import org.springframework.scheduling.concurrent.CustomizableThreadFactory @@ -38,8 +30,9 @@ import java.time.Duration import java.util.concurrent.Executors class EthereumRpcHead( - private val api: DirectEthereumApi, - private val interval: Duration = Duration.ofSeconds(10) + private val api: DirectEthereumApi, + private val objectMapper: ObjectMapper, + private val interval: Duration = Duration.ofSeconds(10) ): DefaultEthereumHead(), Lifecycle { companion object { @@ -67,6 +60,9 @@ class EthereumRpcHead( .subscribeOn(scheduler) .timeout(Defaults.timeout, Mono.error(Exception("Block data not received"))) } + .map { + BlockContainer.from(it, objectMapper) + } .onErrorContinue { err, _ -> log.debug("RPC error ${err.message}") } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt index 40ad8fd7..b405a039 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.config.UpstreamsConfig @@ -38,12 +39,13 @@ open class EthereumUpstream( private val ethereumWs: EthereumWs? = null, private val options: UpstreamsConfig.Options, val node: QuorumForLabels.QuorumItem, - private val targets: CallMethods -) : DefaultUpstream>(), Upstream>, CachesEnabled, Lifecycle { + private val targets: CallMethods, + private val objectMapper: ObjectMapper +) : DefaultUpstream(), Upstream, CachesEnabled, Lifecycle { - constructor(id: String, chain: Chain, api: DirectEthereumApi) : this(id, chain, api, null, + constructor(id: String, chain: Chain, api: DirectEthereumApi, objectMapper: ObjectMapper) : this(id, chain, api, null, UpstreamsConfig.Options.getDefaults(), QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()), - DirectCallMethods()) + DirectCallMethods(), objectMapper) private val log = LoggerFactory.getLogger(EthereumUpstream::class.java) @@ -97,7 +99,7 @@ open class EthereumUpstream( this.start() } // receive bew blocks through Websockets, but periodically verify with RPC - val rpc = EthereumRpcHead(api, Duration.ofSeconds(30)).apply { + val rpc = EthereumRpcHead(api, objectMapper, Duration.ofSeconds(30)).apply { this.start() } EthereumHeadMerge(listOf(rpc, ws)).apply { @@ -105,7 +107,7 @@ open class EthereumUpstream( } } else { log.warn("Setting up upstream $id with RPC-only access, less effective than WS+RPC") - EthereumRpcHead(api).apply { + EthereumRpcHead(api, objectMapper).apply { this.start() } } @@ -136,16 +138,13 @@ open class EthereumUpstream( } @Suppress("unchecked") - override fun , TA : UpstreamApi, BA> cast(selfType: Class, upstreamType: Class, blockType: Class): T { + override fun , TA : UpstreamApi> cast(selfType: Class, upstreamType: Class): T { if (!selfType.isAssignableFrom(this.javaClass)) { throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") } if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) { throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType") } - if (!blockType.isAssignableFrom(BlockJson::class.java)) { - throw ClassCastException("Cannot cast ${BlockJson::class.java} to $blockType") - } return this as T } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWs.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWs.kt index ae1cb898..3d7ac724 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWs.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWs.kt @@ -15,14 +15,15 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.config.AuthConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.reader.Reader -import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.Commands import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionRefJson @@ -38,17 +39,18 @@ import java.time.Duration class EthereumWs( private val uri: URI, private val origin: URI, - private val api: EthereumApi + private val api: EthereumApi, + private val objectMapper: ObjectMapper ): CachesEnabled { private val log = LoggerFactory.getLogger(EthereumWs::class.java) private val topic = TopicProcessor - .builder>() + .builder() .name("new-blocks") .build() var basicAuth: AuthConfig.ClientBasicAuth? = null - private var blockCache: Reader> = EmptyReader() + private var blockCache: Reader = EmptyReader() fun connect() { log.info("Connecting to WebSocket: $uri") @@ -68,24 +70,33 @@ class EthereumWs( } fun onNewBlock(block: BlockJson) { - if (block.totalDifficulty == null || block.transactions == null) { + // WS returns incomplete blocks + if (block.difficulty == null || block.transactions == null) { Mono.just(block.hash).flatMap { hash -> - // first check in cache, if empty then check api - blockCache.read(hash) - .switchIfEmpty(api.executeAndConvert(Commands.eth().getBlock(hash))) - }.repeatWhenEmpty { n -> - Repeat.times(10) - .exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(250)) - .apply(n) - } + val hash = BlockId.from(hash) + // first check in cache, if empty then check api + blockCache.read(hash) + .switchIfEmpty(request(hash)) + }.repeatWhenEmpty { n -> + Repeat.times(10) + .exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(250)) + .apply(n) + } .timeout(Defaults.timeout, Mono.empty()) .subscribe(topic::onNext) + } else { - topic.onNext(block) + topic.onNext(BlockContainer.from(block, objectMapper)) } } - fun getFlux(): Flux> { + fun request(hash: BlockId): Mono { + return api + .executeAndConvert(Commands.eth().getBlock(io.infinitape.etherjar.domain.BlockHash(hash.value))) + .map { BlockContainer.from(it, objectMapper) } + } + + fun getFlux(): Flux { return Flux.from(this.topic) .onBackpressureLatest() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt index bc239863..99ef6ee1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt @@ -24,6 +24,8 @@ import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.* import io.emeraldpay.dshackle.upstream.calls.CallMethods @@ -36,8 +38,6 @@ import io.emeraldpay.grpc.Chain import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.* import io.infinitape.etherjar.rpc.emerald.ReactorEmeraldClient -import io.infinitape.etherjar.rpc.json.BlockJson -import io.infinitape.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import reactor.core.Disposable @@ -46,6 +46,7 @@ import reactor.core.publisher.Mono import reactor.core.publisher.toMono import java.math.BigInteger import java.time.Duration +import java.time.Instant import java.util.* import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicReference @@ -58,7 +59,7 @@ open class EthereumGrpcUpstream( private val blockchainStub: ReactorBlockchainGrpc.ReactorBlockchainStub, private val objectMapper: ObjectMapper, private val rpcClient: ReactorEmeraldClient -) : DefaultUpstream>(), CachesEnabled, Lifecycle { +) : DefaultUpstream(), CachesEnabled, Lifecycle { private var allLabels: Collection = ArrayList() private val log = LoggerFactory.getLogger(EthereumGrpcUpstream::class.java) @@ -117,19 +118,24 @@ open class EthereumGrpcUpstream( internal fun observeHead(flux: Flux) { val base = flux.map { value -> - val block = BlockJson() - block.number = value.height - block.totalDifficulty = BigInteger(1, value.weight.toByteArray()) - block.hash = BlockHash.from("0x"+value.blockId) + val block = BlockContainer( + value.height, + BlockId.from(BlockHash.from("0x" + value.blockId)), + BigInteger(1, value.weight.toByteArray()), + Instant.ofEpochMilli(value.timestamp), + false, + null + ) block }.distinctUntilChanged { it.hash }.filter { block -> val curr = head.getCurrent() - curr == null || curr.totalDifficulty < block.totalDifficulty + curr == null || curr.difficulty < block.difficulty }.flatMap { getApi(Selector.EmptyMatcher()) - .flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(it.hash)) } + .flatMap { api -> api.executeAndConvert(Commands.eth().getBlock(BlockHash(it.hash.value))) } + .map { BlockContainer.from(it, objectMapper) } .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) .doOnError { t -> setStatus(UpstreamAvailability.UNAVAILABLE) @@ -216,16 +222,13 @@ open class EthereumGrpcUpstream( } @SuppressWarnings("unchecked") - override fun , TA : UpstreamApi, BA> cast(selfType: Class, upstreamType: Class, blockType: Class): T { + override fun , TA : UpstreamApi> cast(selfType: Class, upstreamType: Class): T { if (!selfType.isAssignableFrom(this.javaClass)) { throw ClassCastException("Cannot cast ${this.javaClass} to $selfType") } if (!upstreamType.isAssignableFrom(EthereumApi::class.java)) { throw ClassCastException("Cannot cast ${EthereumApi::class.java} to $upstreamType") } - if (!blockType.isAssignableFrom(BlockJson::class.java)) { - throw ClassCastException("Cannot cast ${BlockJson::class.java} to $blockType") - } return this as T } diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy index 275c4d9f..a3dbce1f 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/BlockByHeightSpec.groovy @@ -1,15 +1,23 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification +import java.time.Instant +import java.time.temporal.ChronoUnit + class BlockByHeightSpec extends Specification { String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33" + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Fetch with all data available"() { setup: def blocks = new BlocksMemCache() @@ -18,16 +26,22 @@ class BlockByHeightSpec extends Specification { def block = new BlockJson() block.number = 100 block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block.uncles = [] + block.transactions = [] - blocks.add(block) - heights.add(block) + BlockContainer.from(block, objectMapper).with { + blocks.add(it) + heights.add(it) + } def blocksByHeight = new BlockByHeight(heights, blocks) when: def act = blocksByHeight.read(100).block() then: - act == block + objectMapper.readValue(act.json, BlockJson) == block } def "Fetch correct blocks if multiple"() { @@ -38,26 +52,40 @@ class BlockByHeightSpec extends Specification { def block1 = new BlockJson() block1.number = 100 block1.hash = BlockHash.from(hash1) + block1.totalDifficulty = BigInteger.ONE + block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block1.uncles = [] + block1.transactions = [] + def block2 = new BlockJson() block2.number = 101 block2.hash = BlockHash.from(hash2) + block2.totalDifficulty = BigInteger.ONE + block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block2.uncles = [] + block2.transactions = [] - blocks.add(block1) - heights.add(block1) - blocks.add(block2) - heights.add(block2) + + BlockContainer.from(block1, objectMapper).with { + blocks.add(it) + heights.add(it) + } + BlockContainer.from(block2, objectMapper).with { + blocks.add(it) + heights.add(it) + } def blocksByHeight = new BlockByHeight(heights, blocks) when: def act = blocksByHeight.read(100).block() then: - act == block1 + objectMapper.readValue(act.json, BlockJson) == block1 when: act = blocksByHeight.read(101).block() then: - act == block2 + objectMapper.readValue(act.json, BlockJson) == block2 } def "Fetch last block if updated"() { @@ -68,21 +96,34 @@ class BlockByHeightSpec extends Specification { def block1 = new BlockJson() block1.number = 100 block1.hash = BlockHash.from(hash1) + block1.totalDifficulty = BigInteger.ONE + block1.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block1.uncles = [] + block1.transactions = [] + def block2 = new BlockJson() block2.number = 100 block2.hash = BlockHash.from(hash2) + block2.totalDifficulty = BigInteger.ONE + block2.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block2.uncles = [] + block2.transactions = [] - blocks.add(block1) - heights.add(block1) - blocks.add(block2) - heights.add(block2) + BlockContainer.from(block1, objectMapper).with { + blocks.add(it) + heights.add(it) + } + BlockContainer.from(block2, objectMapper).with { + blocks.add(it) + heights.add(it) + } def blocksByHeight = new BlockByHeight(heights, blocks) when: def act = blocksByHeight.read(100).block() then: - act == block2 + objectMapper.readValue(act.json, BlockJson) == block2 } def "Fetch nothing if block expired"() { @@ -93,9 +134,13 @@ class BlockByHeightSpec extends Specification { def block = new BlockJson() block.number = 100 block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() // add only to heights - heights.add(block) + BlockContainer.from(block, objectMapper).with { + heights.add(it) + } def blocksByHeight = new BlockByHeight(heights, blocks) @@ -113,9 +158,13 @@ class BlockByHeightSpec extends Specification { def block = new BlockJson() block.number = 100 block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() // add only to blocks - blocks.add(block) + BlockContainer.from(block, objectMapper).with { + blocks.add(it) + } def blocksByHeight = new BlockByHeight(heights, blocks) diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy index eacfc945..5bbe6db4 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy @@ -15,12 +15,18 @@ */ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash -import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification +import java.time.Instant +import java.time.temporal.ChronoUnit + class BlocksMemCacheSpec extends Specification { String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" @@ -28,18 +34,24 @@ class BlocksMemCacheSpec extends Specification { String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Add and read"() { setup: def cache = new BlocksMemCache() def block = new BlockJson() block.number = 100 block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block.uncles = [] + block.transactions = [] when: - cache.add(block) - def act = cache.read(BlockHash.from(hash1)).block() + cache.add(BlockContainer.from(block, objectMapper)) + def act = cache.read(BlockId.from(hash1)).block() then: - act == block + objectMapper.readValue(act.json, BlockJson) == block } def "Keeps only configured amount"() { @@ -48,17 +60,22 @@ class BlocksMemCacheSpec extends Specification { [hash1] when: - [hash1, hash2, hash3, hash4].eachWithIndex{ String hash, int i -> + [hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i -> def block = new BlockJson() block.number = 100 + i block.hash = BlockHash.from(hash) - cache.add(block) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() + block.uncles = [] + block.transactions = [] + + cache.add(BlockContainer.from(block, objectMapper)) } - def act1 = cache.read(BlockHash.from(hash1)).block() - def act2 = cache.read(BlockHash.from(hash2)).block() - def act3 = cache.read(BlockHash.from(hash3)).block() - def act4 = cache.read(BlockHash.from(hash4)).block() + def act1 = cache.read(BlockId.from(hash1)).block() + def act2 = cache.read(BlockId.from(hash2)).block() + def act3 = cache.read(BlockId.from(hash3)).block() + def act4 = cache.read(BlockId.from(hash4)).block() then: act2.hash.toHex() == hash2 act3.hash.toHex() == hash3 diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy index 3f42c820..029c4d47 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksRedisCacheSpec.groovy @@ -1,5 +1,8 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.test.IntegrationTestingCommons import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.grpc.Chain @@ -24,6 +27,7 @@ class BlocksRedisCacheSpec extends Specification { String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" + ObjectMapper objectMapper = TestingCommons.objectMapper() def setup() { RedisClient client = IntegrationTestingCommons.redis() @@ -40,15 +44,17 @@ class BlocksRedisCacheSpec extends Specification { def block = new BlockJson() block.number = 100 block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block.totalDifficulty = BigInteger.ONE block.hash = BlockHash.from(hash1) block.transactions = [] block.uncles = [] when: - cache.add(block).subscribe() - def act = cache.read(BlockHash.from(hash1)).block() + cache.add(BlockContainer.from(block, objectMapper)).subscribe() + def act = cache.read(BlockId.from(hash1)).block() then: - act == block + act != null + objectMapper.readValue(act.json, BlockJson) == block } def "Evict existing block"() { @@ -59,19 +65,20 @@ class BlocksRedisCacheSpec extends Specification { def block = new BlockJson() block.number = 100 block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block.totalDifficulty = BigInteger.ONE block.hash = BlockHash.from(hash2) block.transactions = [] block.uncles = [] when: - cache.add(block).subscribe() - def act = cache.read(BlockHash.from(hash2)).block() + cache.add(BlockContainer.from(block, objectMapper)).subscribe() + def act = cache.read(BlockId.from(hash2)).block() then: - act == block + objectMapper.readValue(act.json, BlockJson) == block when: - cache.evict(block.hash).subscribe() - act = cache.read(BlockHash.from(hash2)).block() + cache.evict(BlockId.from(block.hash)).subscribe() + act = cache.read(BlockId.from(hash2)).block() then: act == null @@ -85,22 +92,25 @@ class BlocksRedisCacheSpec extends Specification { def block = new BlockJson() block.number = 100 block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block.totalDifficulty = BigInteger.ONE block.hash = BlockHash.from(hash2) block.transactions = [] block.uncles = [] when: - cache.add(block).subscribe() - def act = cache.read(BlockHash.from(hash2)).block() + cache.add(BlockContainer.from(block, objectMapper)).subscribe() + def act = cache.read(BlockId.from(hash2)).block() then: - act == block + act != null + objectMapper.readValue(act.json, BlockJson) == block when: - cache.evict(BlockHash.from(hash3)).subscribe() - act = cache.read(BlockHash.from(hash2)).block() + cache.evict(BlockId.from(hash3)).subscribe() + act = cache.read(BlockId.from(hash2)).block() then: - act == block + act != null + objectMapper.readValue(act.json, BlockJson) == block } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/CachesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/CachesSpec.groovy index 571205c6..3be4ba34 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/CachesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/CachesSpec.groovy @@ -1,5 +1,9 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson @@ -7,11 +11,14 @@ import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification +import java.time.Instant + class CachesSpec extends Specification { String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33" + ObjectMapper objectMapper = TestingCommons.objectMapper() def "Evict txes if block updated"() { setup: @@ -19,6 +26,7 @@ class CachesSpec extends Specification { HeightCache heightCache = Mock() BlocksMemCache blocksCache = Mock() def caches = Caches.newBuilder() + .setObjectMapper(objectMapper) .setTxByHash(txCache) .setBlockByHeight(heightCache) .setBlockByHash(blocksCache) @@ -27,10 +35,18 @@ class CachesSpec extends Specification { def block1 = new BlockJson() block1.number = 100 block1.hash = BlockHash.from(hash1) + block1.totalDifficulty = BigInteger.ONE + block1.timestamp = Instant.now() + block1.transactions = [] + block1 = BlockContainer.from(block1, objectMapper) def block2 = new BlockJson() block2.number = 100 block2.hash = BlockHash.from(hash2) + block2.totalDifficulty = BigInteger.ONE + block2.timestamp = Instant.now() + block2.transactions = [] + block2 = BlockContainer.from(block2, objectMapper) when: caches.cache(Caches.Tag.LATEST, block1) @@ -53,6 +69,7 @@ class CachesSpec extends Specification { HeightCache heightCache = Mock() BlocksMemCache blocksCache = Mock() def caches = Caches.newBuilder() + .setObjectMapper(objectMapper) .setTxByHash(txCache) .setBlockByHeight(heightCache) .setBlockByHash(blocksCache) @@ -61,10 +78,16 @@ class CachesSpec extends Specification { def block1 = new BlockJson() block1.number = 100 block1.hash = BlockHash.from(hash1) + block1.totalDifficulty = BigInteger.ONE + block1.timestamp = Instant.now() + block1 = BlockContainer.from(block1, objectMapper) def block2 = new BlockJson() block2.number = 100 block2.hash = BlockHash.from(hash2) + block2.totalDifficulty = BigInteger.ONE + block2.timestamp = Instant.now() + block2 = BlockContainer.from(block2, objectMapper) when: caches.cache(Caches.Tag.LATEST, block1) @@ -87,6 +110,7 @@ class CachesSpec extends Specification { HeightCache heightCache = Mock() BlocksMemCache blocksCache = Mock() def caches = Caches.newBuilder() + .setObjectMapper(TestingCommons.objectMapper()) .setTxByHash(txCache) .setBlockByHeight(heightCache) .setBlockByHash(blocksCache) @@ -95,13 +119,15 @@ class CachesSpec extends Specification { def block = new BlockJson() block.number = 100 block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() block.transactions = [ new TransactionRefJson(TransactionId.from(hash1)), new TransactionRefJson(TransactionId.from(hash2)), ] when: - caches.cache(Caches.Tag.REQUESTED, block) + caches.cache(Caches.Tag.REQUESTED, BlockContainer.from(block, objectMapper)) then: 0 * txCache.add(_) } @@ -112,6 +138,7 @@ class CachesSpec extends Specification { HeightCache heightCache = Mock() BlocksMemCache blocksCache = Mock() def caches = Caches.newBuilder() + .setObjectMapper(TestingCommons.objectMapper()) .setTxByHash(txCache) .setBlockByHeight(heightCache) .setBlockByHash(blocksCache) @@ -134,12 +161,15 @@ class CachesSpec extends Specification { def block = new BlockJson() block.number = 100 block.hash = BlockHash.from(hash1) + block.totalDifficulty = BigInteger.ONE block.transactions = [tx1, tx2] + block.timestamp = Instant.now() + block = BlockContainer.from(block, objectMapper) when: caches.cache(Caches.Tag.REQUESTED, block) then: - 1 * txCache.add(tx1) - 1 * txCache.add(tx2) + 1 * txCache.add(TxContainer.from(tx1, objectMapper)) + 1 * txCache.add(TxContainer.from(tx2, objectMapper)) } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksWithTxCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/EthereumBlocksWithTxCacheSpec.groovy similarity index 60% rename from src/test/groovy/io/emeraldpay/dshackle/cache/BlocksWithTxCacheSpec.groovy rename to src/test/groovy/io/emeraldpay/dshackle/cache/EthereumBlocksWithTxCacheSpec.groovy index e2482976..2d89ad38 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/BlocksWithTxCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/EthereumBlocksWithTxCacheSpec.groovy @@ -1,5 +1,10 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson @@ -7,7 +12,9 @@ import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification -class BlocksWithTxCacheSpec extends Specification { +import java.time.Instant + +class EthereumBlocksWithTxCacheSpec extends Specification { // sorted String hash1 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" @@ -15,6 +22,8 @@ class BlocksWithTxCacheSpec extends Specification { String hash3 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" String hash4 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" + ObjectMapper objectMapper = TestingCommons.objectMapper() + def tx1 = new TransactionJson().with { it.blockNumber = 100 it.blockHash = BlockHash.from(hash1) @@ -50,6 +59,8 @@ class BlocksWithTxCacheSpec extends Specification { def block1 = new BlockJson().with { it.number = 100 it.hash = BlockHash.from(hash1) + it.totalDifficulty = BigInteger.ONE + it.timestamp = Instant.now() it.transactions = [ new TransactionRefJson(tx1.hash), new TransactionRefJson(tx2.hash) @@ -61,6 +72,8 @@ class BlocksWithTxCacheSpec extends Specification { def block2 = new BlockJson().with { it.number = 101 it.hash = BlockHash.from(hash3) + it.totalDifficulty = BigInteger.ONE + it.timestamp = Instant.now() it.transactions = [ new TransactionRefJson(tx3.hash) ] @@ -71,6 +84,8 @@ class BlocksWithTxCacheSpec extends Specification { def block3 = new BlockJson().with { it.number = 102 it.hash = BlockHash.from(hash4) + it.totalDifficulty = BigInteger.ONE + it.timestamp = Instant.now() it.transactions = [] it } @@ -81,21 +96,26 @@ class BlocksWithTxCacheSpec extends Specification { def txes = new TxMemCache() def blocks = new BlocksMemCache() - txes.add(tx1) - txes.add(tx2) - txes.add(tx3) - txes.add(tx4) - blocks.add(block1) - blocks.add(block2) - blocks.add(block3) + txes.add(TxContainer.from(tx1, objectMapper)) + txes.add(TxContainer.from(tx2, objectMapper)) + txes.add(TxContainer.from(tx3, objectMapper)) + txes.add(TxContainer.from(tx4, objectMapper)) + blocks.add(BlockContainer.from(block1, objectMapper)) + blocks.add(BlockContainer.from(block2, objectMapper)) + blocks.add(BlockContainer.from(block3, objectMapper)) - def full = new BlocksWithTxCache(blocks, txes) + def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes) when: - def act = full.read(block1.hash).block() + def act = full.read(BlockId.from(block1.hash)).block() then: act != null + + when: + act = objectMapper.readValue(act.json, BlockJson) + + then: act.hash == BlockHash.from(hash1) act.number == 100 act.transactions.size() == 2 @@ -119,9 +139,15 @@ class BlocksWithTxCacheSpec extends Specification { // request second block when: - act = full.read(block2.hash).block() + act = full.read(BlockId.from(block2.hash)).block() then: act != null + + when: + act = objectMapper.readValue(act.json, BlockJson) + + then: + act.hash == BlockHash.from(hash3) act.number == 101 act.transactions.size() == 1 @@ -136,23 +162,23 @@ class BlocksWithTxCacheSpec extends Specification { def txes = new TxMemCache() def blocks = new BlocksMemCache() - txes.add(tx1) - txes.add(tx2) - txes.add(tx3) - txes.add(tx4) - blocks.add(block1) - blocks.add(block2) - blocks.add(block3) + txes.add(TxContainer.from(tx1, objectMapper)) + txes.add(TxContainer.from(tx2, objectMapper)) + txes.add(TxContainer.from(tx3, objectMapper)) + txes.add(TxContainer.from(tx4, objectMapper)) + blocks.add(BlockContainer.from(block1, objectMapper)) + blocks.add(BlockContainer.from(block2, objectMapper)) + blocks.add(BlockContainer.from(block3, objectMapper)) - def full = new BlocksWithTxCache(blocks, txes) + def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes) when: - def act = full.read(block3.hash).block() + def act = full.read(BlockId.from(block3.hash)).block() then: act != null - act.hash == BlockHash.from(hash4) - act.number == 102 + act.hash == BlockId.from(hash4) + act.height == 102 act.transactions.size() == 0 } @@ -161,13 +187,13 @@ class BlocksWithTxCacheSpec extends Specification { def txes = new TxMemCache() def blocks = new BlocksMemCache() - txes.add(tx1) - blocks.add(block1) //missing tx2 in cache + txes.add(TxContainer.from(tx1, objectMapper)) + blocks.add(BlockContainer.from(block1, objectMapper)) //missing tx2 in cache - def full = new BlocksWithTxCache(blocks, txes) + def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes) when: - def act = full.read(block1.hash).block() + def act = full.read(BlockId.from(block1.hash)).block() then: act == null @@ -178,14 +204,14 @@ class BlocksWithTxCacheSpec extends Specification { def txes = new TxMemCache() def blocks = new BlocksMemCache() - txes.add(tx1) - txes.add(tx2) - txes.add(tx3) + txes.add(TxContainer.from(tx1, objectMapper)) + txes.add(TxContainer.from(tx2, objectMapper)) + txes.add(TxContainer.from(tx3, objectMapper)) - def full = new BlocksWithTxCache(blocks, txes) + def full = new EthereumBlocksWithTxCache(objectMapper, blocks, txes) when: - def act = full.read(block1.hash).block() + def act = full.read(BlockId.from(block1.hash)).block() then: act == null diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/HeightCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/HeightCacheSpec.groovy index ff4be861..e711438e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/HeightCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/HeightCacheSpec.groovy @@ -1,10 +1,15 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.json.BlockJson import io.infinitape.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification +import java.time.Instant + class HeightCacheSpec extends Specification { String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" @@ -12,16 +17,20 @@ class HeightCacheSpec extends Specification { String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Add and read"() { setup: def cache = new HeightCache() when: - [hash1, hash2, hash3, hash4].eachWithIndex{ String hash, int i -> + [hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i -> def block = new BlockJson() block.number = 100 + i block.hash = BlockHash.from(hash) - cache.add(block) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() + cache.add(BlockContainer.from(block, objectMapper)) } def act1 = cache.read(100).block() @@ -41,11 +50,13 @@ class HeightCacheSpec extends Specification { [hash1] when: - [hash1, hash2, hash3, hash4].eachWithIndex{ String hash, int i -> + [hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i -> def block = new BlockJson() block.number = 100 + i block.hash = BlockHash.from(hash) - cache.add(block) + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() + cache.add(BlockContainer.from(block, objectMapper)) } def act1 = cache.read(100).block() diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/TxMemCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/TxMemCacheSpec.groovy index 1e292b67..5b310d7e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/TxMemCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/TxMemCacheSpec.groovy @@ -1,5 +1,11 @@ package io.emeraldpay.dshackle.cache +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.domain.TransactionId import io.infinitape.etherjar.rpc.json.BlockJson @@ -7,6 +13,8 @@ import io.infinitape.etherjar.rpc.json.TransactionJson import io.infinitape.etherjar.rpc.json.TransactionRefJson import spock.lang.Specification +import java.time.Instant + class TxMemCacheSpec extends Specification { String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab" @@ -14,6 +22,8 @@ class TxMemCacheSpec extends Specification { String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Add and read"() { setup: def cache = new TxMemCache() @@ -23,10 +33,10 @@ class TxMemCacheSpec extends Specification { tx.blockNumber = 100 when: - cache.add(tx) - def act = cache.read(TransactionId.from(hash1)).block() + cache.add(TxContainer.from(tx, objectMapper)) + def act = cache.read(TxId.from(hash1)).block() then: - act == tx + objectMapper.readValue(act.json, TransactionJson.class) == tx } def "Keeps only configured amount"() { @@ -34,18 +44,18 @@ class TxMemCacheSpec extends Specification { def cache = new TxMemCache(3) when: - [hash1, hash2, hash3, hash4].eachWithIndex{ String hash, int i -> + [hash1, hash2, hash3, hash4].eachWithIndex { String hash, int i -> def tx = new TransactionJson() tx.blockNumber = 100 + i tx.blockHash = BlockHash.from(hash) tx.hash = TransactionId.from(hash) - cache.add(tx) + cache.add(TxContainer.from(tx, objectMapper)) } - def act1 = cache.read(TransactionId.from(hash1)).block() - def act2 = cache.read(TransactionId.from(hash2)).block() - def act3 = cache.read(TransactionId.from(hash3)).block() - def act4 = cache.read(TransactionId.from(hash4)).block() + def act1 = cache.read(TxId.from(hash1)).block() + def act2 = cache.read(TxId.from(hash2)).block() + def act3 = cache.read(TxId.from(hash3)).block() + def act4 = cache.read(TxId.from(hash4)).block() then: act2.hash.toHex() == hash2 act3.hash.toHex() == hash3 @@ -63,22 +73,22 @@ class TxMemCacheSpec extends Specification { tx.blockNumber = 100 tx.blockHash = BlockHash.from(hash1) tx.hash = TransactionId.from(hash) - cache.add(tx) + cache.add(TxContainer.from(tx, objectMapper)) } - [hash3, hash4].eachWithIndex{ String hash, int i -> + [hash3, hash4].eachWithIndex { String hash, int i -> def tx = new TransactionJson() tx.blockNumber = 101 tx.blockHash = BlockHash.from(hash2) tx.hash = TransactionId.from(hash) - cache.add(tx) + cache.add(TxContainer.from(tx, objectMapper)) } - cache.evict(BlockHash.from(hash1)) + cache.evict(BlockId.from(hash1)) - def act1 = cache.read(TransactionId.from(hash1)).block() - def act2 = cache.read(TransactionId.from(hash2)).block() - def act3 = cache.read(TransactionId.from(hash3)).block() - def act4 = cache.read(TransactionId.from(hash4)).block() + def act1 = cache.read(TxId.from(hash1)).block() + def act2 = cache.read(TxId.from(hash2)).block() + def act3 = cache.read(TxId.from(hash3)).block() + def act4 = cache.read(TxId.from(hash4)).block() then: act1 == null @@ -97,30 +107,32 @@ class TxMemCacheSpec extends Specification { tx.blockNumber = 100 tx.blockHash = BlockHash.from(hash1) tx.hash = TransactionId.from(hash) - cache.add(tx) + cache.add(TxContainer.from(tx, objectMapper)) } [hash3, hash4].eachWithIndex{ String hash, int i -> def tx = new TransactionJson() tx.blockNumber = 100 tx.blockHash = BlockHash.from(hash2) tx.hash = TransactionId.from(hash) - cache.add(tx) + cache.add(TxContainer.from(tx, objectMapper)) } def block = new BlockJson() block.hash = BlockHash.from(hash1) block.number = 100 + block.totalDifficulty = BigInteger.ONE + block.timestamp = Instant.now() block.transactions = [ new TransactionRefJson(TransactionId.from(hash1)), new TransactionRefJson(TransactionId.from(hash2)), ] - cache.evict(block) + cache.evict(BlockContainer.from(block, objectMapper)) - def act1 = cache.read(TransactionId.from(hash1)).block() - def act2 = cache.read(TransactionId.from(hash2)).block() - def act3 = cache.read(TransactionId.from(hash3)).block() - def act4 = cache.read(TransactionId.from(hash4)).block() + def act1 = cache.read(TxId.from(hash1)).block() + def act2 = cache.read(TxId.from(hash2)).block() + def act3 = cache.read(TxId.from(hash3)).block() + def act4 = cache.read(TxId.from(hash4)).block() then: act1 == null diff --git a/src/test/groovy/io/emeraldpay/dshackle/cache/TxRedisCacheSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/cache/TxRedisCacheSpec.groovy index dc605646..ca0d73de 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/cache/TxRedisCacheSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/cache/TxRedisCacheSpec.groovy @@ -1,5 +1,9 @@ package io.emeraldpay.dshackle.cache + +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.test.IntegrationTestingCommons import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.grpc.Chain @@ -26,6 +30,8 @@ class TxRedisCacheSpec extends Specification { String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b" TxRedisCache cache + def objectMapper = TestingCommons.objectMapper() + def setup() { RedisClient client = IntegrationTestingCommons.redis() StatefulRedisConnection connection = client.connect(); @@ -39,6 +45,7 @@ class TxRedisCacheSpec extends Specification { def block = new BlockJson() block.number = 100 block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block.totalDifficulty = BigInteger.ONE block.hash = BlockHash.from(hash1) block.transactions = [] block.uncles = [] @@ -51,10 +58,11 @@ class TxRedisCacheSpec extends Specification { tx.nonce = 0 when: - cache.add(tx, block).subscribe() - def act = cache.read(TransactionId.from(hash1)).block() + cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block, objectMapper)).subscribe() + def act = cache.read(TxId.from(hash1)).block() then: - act == tx + act != null + objectMapper.readValue(act.json, TransactionJson) == tx } def "Evict single tx"() { @@ -62,6 +70,7 @@ class TxRedisCacheSpec extends Specification { def block = new BlockJson() block.number = 100 block.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block.totalDifficulty = BigInteger.ONE block.hash = BlockHash.from(hash2) block.transactions = [] block.uncles = [] @@ -74,14 +83,15 @@ class TxRedisCacheSpec extends Specification { tx.nonce = 0 when: - cache.add(tx, block).subscribe() - def act = cache.read(tx.hash).block() + cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block, objectMapper)).subscribe() + def act = cache.read(TxId.from(tx.hash)).block() then: - act == tx + act != null + objectMapper.readValue(act.json, TransactionJson) == tx when: - cache.evict(tx.hash).subscribe() - act = cache.read(tx.hash).block() + cache.evict(TxId.from(tx.hash)).subscribe() + act = cache.read(TxId.from(tx.hash)).block() then: act == null } @@ -92,6 +102,7 @@ class TxRedisCacheSpec extends Specification { block1.hash = BlockHash.from(hash1) block1.number = 100 block1.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block1.totalDifficulty = BigInteger.ONE block1.transactions = [ new TransactionRefJson(TransactionId.from(hash1)), new TransactionRefJson(TransactionId.from(hash2)), @@ -100,6 +111,7 @@ class TxRedisCacheSpec extends Specification { block2.hash = BlockHash.from(hash2) block2.number = 101 block2.timestamp = Instant.now().minusSeconds(100).truncatedTo(ChronoUnit.SECONDS) + block2.totalDifficulty = BigInteger.ONE block2.transactions = [ new TransactionRefJson(TransactionId.from(hash3)), new TransactionRefJson(TransactionId.from(hash4)), @@ -112,7 +124,7 @@ class TxRedisCacheSpec extends Specification { tx.hash = TransactionId.from(hash) tx.value = Wei.ofEthers(i) tx.nonce = 0 - cache.add(tx, block1).subscribe() + cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block1, objectMapper)).subscribe() } [hash3, hash4].eachWithIndex{ String hash, int i -> def tx = new TransactionJson() @@ -121,16 +133,16 @@ class TxRedisCacheSpec extends Specification { tx.hash = TransactionId.from(hash) tx.value = Wei.ofEthers(i) tx.nonce = 0 - cache.add(tx, block2).subscribe() + cache.add(TxContainer.from(tx, objectMapper), BlockContainer.from(block2, objectMapper)).subscribe() } - cache.evict(block1).subscribe() + cache.evict(BlockContainer.from(block1, objectMapper)).subscribe() - def act1 = cache.read(TransactionId.from(hash1)).block() - def act2 = cache.read(TransactionId.from(hash2)).block() - def act3 = cache.read(TransactionId.from(hash3)).block() - def act4 = cache.read(TransactionId.from(hash4)).block() + def act1 = cache.read(TxId.from(hash1)).block() + def act2 = cache.read(TxId.from(hash2)).block() + def act3 = cache.read(TxId.from(hash3)).block() + def act4 = cache.read(TxId.from(hash4)).block() then: act1 == null diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy index 2280fd03..d6b3304f 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy @@ -15,10 +15,13 @@ */ package io.emeraldpay.dshackle.rpc +import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.test.EthereumUpstreamMock +import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.UpstreamsMock import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi import io.emeraldpay.dshackle.upstream.Upstream @@ -37,6 +40,8 @@ import java.time.Instant class StreamHeadSpec extends Specification { + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Errors on unavailable chain"() { setup: def upstreams = new UpstreamsMock(Chain.ETHEREUM, Stub(EthereumUpstream)) @@ -83,9 +88,9 @@ class StreamHeadSpec extends Specification { ) then: StepVerifier.create(flux.take(2)) - .then { upstream.nextBlock(blocks[0]) } + .then { upstream.nextBlock(BlockContainer.from(blocks[0], objectMapper)) } .expectNext(heads[0]) - .then { upstream.nextBlock(blocks[1]) } + .then { upstream.nextBlock(BlockContainer.from(blocks[1], objectMapper)) } .expectNext(heads[1]) .expectComplete() .verify(Duration.ofSeconds(1)) diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumAddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumAddressSpec.groovy index 0e41d17c..764c176f 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumAddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumAddressSpec.groovy @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.UpstreamsMock import io.emeraldpay.dshackle.upstream.Upstreams @@ -32,6 +33,8 @@ import reactor.test.StepVerifier import spock.lang.Specification import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit class TrackEthereumAddressSpec extends Specification { @@ -94,6 +97,7 @@ class TrackEthereumAddressSpec extends Specification { it.number = 1 it.totalDifficulty = 100 it.hash = BlockHash.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22") + it.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) return it } @@ -115,7 +119,7 @@ class TrackEthereumAddressSpec extends Specification { assert trackAddress.isTracked(Chain.ETHEREUM, Address.from(address1)) } .then { - upstreamMock.nextBlock(block2) + upstreamMock.nextBlock(BlockContainer.from(block2, TestingCommons.objectMapper())) } .expectNext(exp2) .thenCancel() diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy index 6d843110..c12b618c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy @@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.rpc import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.test.UpstreamsMock import io.emeraldpay.dshackle.upstream.Upstreams @@ -63,6 +64,7 @@ class TrackEthereumTxSpec extends Specification { it.timestamp = Instant.ofEpochMilli(156400200000) it.number = 108 it.totalDifficulty = BigInteger.valueOf(800) + it.transactions = [] it } @@ -75,15 +77,17 @@ class TrackEthereumTxSpec extends Specification { it } + blockJson.transactions = [new TransactionRefJson(txJson.hash)] + def exp1 = BlockchainOuterClass.TxStatus.newBuilder() - .setTxId(txId) - .setBroadcasted(true) - .setMined(true) - .setConfirmations(8 + 1) - .setBlock( - Common.BlockInfo.newBuilder() - .setHeight(blockJson.number) - .setWeight(ByteString.copyFrom(blockJson.totalDifficulty.toByteArray())) + .setTxId(txId) + .setBroadcasted(true) + .setMined(true) + .setConfirmations(8 + 1) + .setBlock( + Common.BlockInfo.newBuilder() + .setHeight(blockJson.number) + .setWeight(ByteString.copyFrom(blockJson.totalDifficulty.toByteArray())) .setBlockId(blockJson.hash.toHex().substring(2)) .setTimestamp(blockJson.timestamp.toEpochMilli()) ).build() @@ -96,7 +100,7 @@ class TrackEthereumTxSpec extends Specification { apiMock.answer("eth_getTransactionByHash", [txId], txJson) apiMock.answer("eth_getBlockByHash", [blockJson.hash.toHex(), false], blockJson) - upstreamMock.nextBlock(blockHeadJson) + upstreamMock.nextBlock(BlockContainer.from(blockHeadJson, TestingCommons.objectMapper())) when: def flux = trackTx.add(Mono.just(req)) @@ -292,7 +296,7 @@ class TrackEthereumTxSpec extends Specification { def nextBlock = { int i -> return { println("block $i"); - upstreamMock.nextBlock(blocks[i]) + upstreamMock.nextBlock(BlockContainer.from(blocks[i], TestingCommons.objectMapper())) } as Runnable } diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumHeadMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumHeadMock.groovy index 865d28c4..fa737f51 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumHeadMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumHeadMock.groovy @@ -15,26 +15,25 @@ */ package io.emeraldpay.dshackle.test +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead -import io.infinitape.etherjar.domain.TransactionId -import io.infinitape.etherjar.rpc.json.BlockJson import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.core.publisher.TopicProcessor class EthereumHeadMock implements EthereumHead { - private TopicProcessor> bus = TopicProcessor.create() - private BlockJson latest + private TopicProcessor bus = TopicProcessor.create() + private BlockContainer latest - void nextBlock(BlockJson block) { + void nextBlock(BlockContainer block) { assert block != null latest = block bus.onNext(block) } @Override - Flux> getFlux() { + Flux getFlux() { return Flux.concat(Mono.justOrEmpty(latest), bus).distinctUntilChanged() } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy index 586a7d5b..22fdd3e5 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.test import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods @@ -47,12 +48,12 @@ class EthereumUpstreamMock extends EthereumUpstream { EthereumUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull DirectEthereumApi api, CallMethods methods) { super(id, chain, api, null, UpstreamsConfig.Options.getDefaults(), new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()), - methods) + methods, TestingCommons.objectMapper()) setLag(0) setStatus(UpstreamAvailability.OK) } - void nextBlock(BlockJson block) { + void nextBlock(BlockContainer block) { ethereumHeadMock.nextBlock(block) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy index 23043b5b..4af923b6 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy @@ -76,7 +76,7 @@ class TestingCommons { } static AggregatedUpstream aggregatedUpstream(EthereumUpstream up) { - return new EthereumChainUpstreams(Chain.ETHEREUM, [up], Caches.default(), objectMapper()) + return new EthereumChainUpstreams(Chain.ETHEREUM, [up], Caches.default(objectMapper()), objectMapper()) } static CachesFactory emptyCaches() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy index f0070cc8..8de5ad34 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy @@ -41,7 +41,7 @@ class UpstreamsMock implements Upstreams { AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull Upstream up) { if (!upstreams.containsKey(chain)) { - upstreams[chain] = new EthereumChainUpstreams(chain, [up], Caches.default(), TestingCommons.objectMapper()) + upstreams[chain] = new EthereumChainUpstreams(chain, [up], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper()) } else { upstreams[chain].addUpstream(up) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/AggregatedUpstreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/AggregatedUpstreamSpec.groovy index b930c609..c2fffdcf 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/AggregatedUpstreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/AggregatedUpstreamSpec.groovy @@ -31,7 +31,7 @@ class AggregatedUpstreamSpec extends Specification { setup: def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test1", "eth_test2"])) def up2 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test2", "eth_test3"])) - def aggr = new EthereumChainUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(), TestingCommons.objectMapper()) + def aggr = new EthereumChainUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper()) when: aggr.onUpstreamsUpdated() def act = aggr.getMethods() diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/CachingEthereumApiSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/CachingEthereumApiSpec.groovy index a8747cc6..659d8a85 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/CachingEthereumApiSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/CachingEthereumApiSpec.groovy @@ -1,11 +1,14 @@ package io.emeraldpay.dshackle.upstream +import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.cache.BlockByHeight import io.emeraldpay.dshackle.cache.BlocksMemCache import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.HeightCache import io.emeraldpay.dshackle.cache.TxMemCache -import io.emeraldpay.dshackle.reader.EmptyReader +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.ethereum.EthereumHead import io.infinitape.etherjar.domain.BlockHash @@ -18,34 +21,47 @@ import reactor.test.StepVerifier import spock.lang.Specification import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit class CachingEthereumApiSpec extends Specification { + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Get blockNumber from head"() { setup: def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.default(), + objectMapper, + Caches.default(objectMapper), head ) - 1 * head.getFlux() >> Flux.just(new BlockJson(number: 100)) + 1 * head.getFlux() >> Flux.just(BlockContainer.from( + new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + difficulty: 1, + totalDifficulty: BigInteger.ONE, + timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS) + ), + objectMapper + )) when: - def act = api.execute(1, "eth_blockNumber", []).map { new String(it)} + def act = api.execute(1, "eth_blockNumber", []).map { new String(it) } then: StepVerifier.create(act) - .expectNext('{"jsonrpc":"2.0","id":1,"result":"0x64"}') - .expectComplete() - .verify(Duration.ofSeconds(3)) + .expectNext('{"jsonrpc":"2.0","id":1,"result":"0x64"}') + .expectComplete() + .verify(Duration.ofSeconds(3)) } def "Return empty if block is not cached"() { setup: def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.default(), + objectMapper, + Caches.default(objectMapper), head ) when: @@ -62,18 +78,26 @@ class CachingEthereumApiSpec extends Specification { def cache = new BlocksMemCache(); def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.newBuilder().setBlockByHash(cache).build(), + objectMapper, + Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(cache).build(), head ) - cache.add(new BlockJson(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"))) + cache.add(BlockContainer.from( + new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + totalDifficulty: BigInteger.ONE, + timestamp: Instant.ofEpochSecond(0x5e95313a) + ), + objectMapper + )) when: - def act = api.execute(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).map { new String(it)} + def act = api.execute(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).map { new String(it) } then: StepVerifier.create(act) - .expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","transactions":[],"uncles":[]}}') + .expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","timestamp":"0x5e95313a","transactions":[],"totalDifficulty":"0x1","uncles":[]}}') .expectComplete() .verify(Duration.ofSeconds(3)) } @@ -84,20 +108,25 @@ class CachingEthereumApiSpec extends Specification { def heightCache = new HeightCache() def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.newBuilder().setBlockByHash(blocksCache).setBlockByHeight(heightCache).build(), + objectMapper, + Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setBlockByHeight(heightCache).build(), head ) - def block = new BlockJson(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) - heightCache.add(block) - blocksCache.add(block) + def block = new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + totalDifficulty: BigInteger.ONE, + timestamp: Instant.ofEpochSecond(0x5e95313a) + ) + heightCache.add(BlockContainer.from(block, objectMapper)) + blocksCache.add(BlockContainer.from(block, objectMapper)) when: - def act = api.execute(1, "eth_getBlockByNumber", ["0x64", false]).map { new String(it)} + def act = api.execute(1, "eth_getBlockByNumber", ["0x64", false]).map { new String(it) } then: StepVerifier.create(act) - .expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","transactions":[],"uncles":[]}}') + .expectNext('{"jsonrpc":"2.0","id":1,"result":{"number":"0x64","hash":"0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58","timestamp":"0x5e95313a","transactions":[],"totalDifficulty":"0x1","uncles":[]}}') .expectComplete() .verify(Duration.ofSeconds(3)) } @@ -108,18 +137,23 @@ class CachingEthereumApiSpec extends Specification { def txCache = Mock(TxMemCache) def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).build(), + objectMapper, + Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).build(), head ) - def block = new BlockJson(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) + def block = new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + totalDifficulty: BigInteger.ONE, + timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS) + ) when: def act = api.readBlockByHash(1, "eth_getBlockByHash", ["0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58", false]).block() then: act != null - 1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) + 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper)) 0 * txCache.read(_) } @@ -129,11 +163,16 @@ class CachingEthereumApiSpec extends Specification { def txCache = Mock(TxMemCache) def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).build(), + objectMapper, + Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).build(), head ) - def block = new BlockJson(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) + def block = new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + totalDifficulty: BigInteger.ONE, + timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS) + ) block.transactions = [ new TransactionRefJson(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) ] @@ -143,8 +182,8 @@ class CachingEthereumApiSpec extends Specification { then: act == null - 1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) - 1 * txCache.read(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty() + 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper)) + 1 * txCache.read(TxId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty() } def "Uses base cache when requested, by height"() { @@ -154,19 +193,24 @@ class CachingEthereumApiSpec extends Specification { def heightCache = Mock(HeightCache) def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(), + objectMapper, + Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(), head ) - def block = new BlockJson(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) + def block = new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + totalDifficulty: BigInteger.ONE, + timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS) + ) when: def act = api.readBlockByNumber(1, "eth_getBlockByNumber", ["0x64", false]).block() then: act != null - 1 * heightCache.read(100) >> Mono.just(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) - 1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) + 1 * heightCache.read(100) >> Mono.just(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) + 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper)) 0 * txCache.read(_) } @@ -177,11 +221,16 @@ class CachingEthereumApiSpec extends Specification { def heightCache = Mock(HeightCache) def head = Mock(EthereumHead.class) def api = new CachingEthereumApi( - TestingCommons.objectMapper(), - Caches.newBuilder().setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(), + objectMapper, + Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).setTxByHash(txCache).setBlockByHeight(heightCache).build(), head ) - def block = new BlockJson(number: 100, hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) + def block = new BlockJson( + number: 100, + hash: BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58"), + totalDifficulty: BigInteger.ONE, + timestamp: Instant.now().truncatedTo(ChronoUnit.SECONDS) + ) block.transactions = [ new TransactionRefJson(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) ] @@ -191,8 +240,8 @@ class CachingEthereumApiSpec extends Specification { then: act == null - 1 * heightCache.read(100) >> Mono.just(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) - 1 * blocksCache.read(BlockHash.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(block) - 1 * txCache.read(TransactionId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty() + 1 * heightCache.read(100) >> Mono.just(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) + 1 * blocksCache.read(BlockId.from("0x5b4590a9905fa1c9cc273f32e6dc63b4c512f0ee14edc6fa41c26b416a7b5d58")) >> Mono.just(BlockContainer.from(block, objectMapper)) + 1 * txCache.read(TxId.from("0x0500219f2b147f3013e9030d585e8e5d45401ebd2620a42c879c0d5d1b754073")) >> Mono.empty() } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy index 0ff1f6e7..5dc17780 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy @@ -53,7 +53,7 @@ class FilteredApisSpec extends Specification { (EthereumWs) null, new UpstreamsConfig.Options(), new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)), - ethereumTargets + ethereumTargets, TestingCommons.objectMapper() ) } def matcher = new Selector.LabelMatcher("test", ["foo"]) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy index 6e642416..cb935599 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy @@ -15,23 +15,31 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.json.BlockJson import reactor.core.publisher.Flux import reactor.test.StepVerifier import spock.lang.Specification +import java.time.Instant + class DefaultEthereumHeadSpec extends Specification { DefaultEthereumHead head = new DefaultEthereumHead() + ObjectMapper objectMapper = TestingCommons.objectMapper() def blocks = (10L..20L).collect { i -> - new BlockJson().with { - it.number = 10000L + i - it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec89152" + i) - it.totalDifficulty = 11 * i - return it - } + BlockContainer.from( + new BlockJson().with { + it.number = 10000L + i + it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec89152" + i) + it.totalDifficulty = 11 * i + it.timestamp = Instant.now() + return it + }, objectMapper) } def "Starts to follow"() { @@ -80,12 +88,14 @@ class DefaultEthereumHeadSpec extends Specification { def "Ignores less difficult"() { when: - def block3less = new BlockJson().with { - it.number = blocks[3].number - it.hash = blocks[3].hash - it.totalDifficulty = blocks[3].totalDifficulty - 1 - return it - } + def block3less = BlockContainer.from( + new BlockJson().with { + it.number = blocks[3].height + it.hash = BlockHash.from(blocks[3].hash.value) + it.totalDifficulty = blocks[3].difficulty - 1 + it.timestamp = Instant.now() + return it + }, objectMapper) head.follow(Flux.just(blocks[0], blocks[3], block3less)) def act = head.flux then: @@ -97,12 +107,14 @@ class DefaultEthereumHeadSpec extends Specification { def "Replaces with more difficult"() { when: - def block3less = new BlockJson().with { - it.number = blocks[3].number - it.hash = blocks[3].hash - it.totalDifficulty = blocks[3].totalDifficulty + 1 - return it - } + def block3less = BlockContainer.from( + new BlockJson().with { + it.number = blocks[3].height + it.hash = BlockHash.from(blocks[3].hash.value) + it.totalDifficulty = blocks[3].difficulty + 1 + it.timestamp = Instant.now() + return it + }, objectMapper) head.follow(Flux.just(blocks[0], blocks[3], block3less)) def act = head.flux then: diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserverSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserverSpec.groovy index 746833ca..c25b0e2e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserverSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserverSpec.groovy @@ -15,8 +15,12 @@ */ package io.emeraldpay.dshackle.upstream.ethereum +import com.fasterxml.jackson.databind.ObjectMapper +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.HeadLagObserver import io.emeraldpay.dshackle.upstream.Upstream +import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.json.BlockJson import reactor.core.publisher.Flux import reactor.core.publisher.TopicProcessor @@ -25,9 +29,12 @@ import reactor.util.function.Tuples import spock.lang.Specification import java.time.Duration +import java.time.Instant class EthereumHeadLagObserverSpec extends Specification { + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Updates lag distance"() { setup: EthereumHead master = Mock() @@ -43,11 +50,15 @@ class EthereumHeadLagObserverSpec extends Specification { } def blocks = [100, 101, 102].collect { i -> - return new BlockJson().with { - it.number = i - it.totalDifficulty = 2000 + i - return it - } + return BlockContainer.from( + new BlockJson().with { + it.number = i + it.totalDifficulty = 2000 + i + it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915" + i) + it.timestamp = Instant.now() + return it + }, + objectMapper) } def masterBus = TopicProcessor.create() @@ -83,11 +94,15 @@ class EthereumHeadLagObserverSpec extends Specification { Upstream up = Mock() def blocks = [100, 101, 102].collect { i -> - return new BlockJson().with { - it.number = i - it.totalDifficulty = 2000 + i - return it - } + return BlockContainer.from( + new BlockJson().with { + it.number = i + it.totalDifficulty = 2000 + i + it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915" + i) + it.timestamp = Instant.now() + return it + }, + objectMapper) } def upblocks = Flux.fromIterable(blocks) @@ -109,14 +124,18 @@ class EthereumHeadLagObserverSpec extends Specification { def top = new BlockJson().with { it.number = topHeight it.totalDifficulty = topDiff + it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123") + it.timestamp = Instant.now() return it } def curr = new BlockJson().with { it.number = currHeight it.totalDifficulty = currDiff + it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123") + it.timestamp = Instant.now() return it } - delta as Long == observer.extractDistance(top, curr) + delta as Long == observer.extractDistance(BlockContainer.from(top, objectMapper), BlockContainer.from(curr, objectMapper)) where: topHeight | topDiff | currHeight | currDiff | delta 100 | 1000 | 100 | 1000 | 0 diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsSpec.groovy index af9a8709..68c2f8c3 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsSpec.groovy @@ -1,8 +1,11 @@ package io.emeraldpay.dshackle.upstream.ethereum +import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.cache.BlocksMemCache import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.HeightCache +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.test.TestingCommons import io.infinitape.etherjar.domain.BlockHash import io.infinitape.etherjar.rpc.ReactorRpcClient @@ -19,26 +22,30 @@ import java.time.temporal.ChronoUnit class EthereumWsSpec extends Specification { + ObjectMapper objectMapper = TestingCommons.objectMapper() + def "Uses cache to fetch block"() { setup: ReactorRpcClient rpcClient = Stub(ReactorRpcClient) def apiMock = TestingCommons.api(rpcClient) - def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock) + def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock, objectMapper) def blocksCache = Mock(BlocksMemCache) - def caches = Caches.newBuilder().setBlockByHash(blocksCache).build() + def caches = Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).build() ws.setCaches(caches) def block = new BlockJson() + block.number = 100 block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200") block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) + block.totalDifficulty = BigInteger.ONE when: ws.onNewBlock(block) then: - 1 * blocksCache.read(BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")) >> Mono.just(block) + 1 * blocksCache.read(BlockId.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200")) >> Mono.just(BlockContainer.from(block, objectMapper)) StepVerifier.create(ws.flux.take(1)) - .expectNext(block) + .expectNext(BlockContainer.from(block, objectMapper)) .expectComplete() .verify(Duration.ofSeconds(1)) } @@ -47,16 +54,18 @@ class EthereumWsSpec extends Specification { setup: ReactorRpcClient rpcClient = Stub(ReactorRpcClient) def apiMock = TestingCommons.api(rpcClient) - def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock) + def ws = new EthereumWs(new URI("http://localhost"), new URI("http://localhost"), apiMock, objectMapper) def blocksCache = Mock(BlocksMemCache) - def caches = Caches.newBuilder().setBlockByHash(blocksCache).build() + def caches = Caches.newBuilder().setObjectMapper(objectMapper).setBlockByHash(blocksCache).build() ws.setCaches(caches) def block = new BlockJson() + block.number = 100 block.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200") block.timestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS) block.transactions = [] block.uncles = [] + block.totalDifficulty = BigInteger.ONE apiMock.answerOnce("eth_getBlockByHash", ["0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915200", false], block) @@ -66,7 +75,7 @@ class EthereumWsSpec extends Specification { then: 1 * blocksCache.read(_) >> Mono.empty() StepVerifier.create(ws.flux.take(1)) - .expectNext(block) + .expectNext(BlockContainer.from(block, objectMapper)) .expectComplete() .verify(Duration.ofSeconds(1)) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy index 58cb81cd..0775bdbe 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstreamSpec.groovy @@ -20,6 +20,7 @@ import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainGrpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common +import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.test.MockServer import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.UpstreamAvailability @@ -32,6 +33,7 @@ import io.infinitape.etherjar.rpc.json.BlockJson import spock.lang.Specification import java.time.Duration +import java.time.Instant import java.util.concurrent.CompletableFuture class EthereumGrpcUpstreamSpec extends Specification { @@ -48,6 +50,7 @@ class EthereumGrpcUpstreamSpec extends Specification { it.number = 650246 it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) + it.timestamp = Instant.now() return it } api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) @@ -81,7 +84,7 @@ class EthereumGrpcUpstreamSpec extends Specification { then: callData.chain == Chain.ETHEREUM.id upstream.status == UpstreamAvailability.OK - h.hash == BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") + h.hash == BlockId.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") } def "Follows difficulty, ignores less difficult"() { @@ -91,12 +94,14 @@ class EthereumGrpcUpstreamSpec extends Specification { it.number = 650246 it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) + it.timestamp = Instant.now() return it } def block2 = new BlockJson().with { it.number = 650247 it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") it.totalDifficulty = new BigInteger("35bbde5595de6455", 16) + it.timestamp = Instant.now() return it } api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) @@ -136,8 +141,8 @@ class EthereumGrpcUpstreamSpec extends Specification { def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block() then: upstream.status == UpstreamAvailability.OK - h.hash == BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") - h.number == 650246 + h.hash == BlockId.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") + h.height == 650246 } def "Follows difficulty"() { @@ -150,12 +155,14 @@ class EthereumGrpcUpstreamSpec extends Specification { it.number = 650246 it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7") it.totalDifficulty = new BigInteger("35bbde5595de6456", 16) + it.timestamp = Instant.now() return it } def block2 = new BlockJson().with { it.number = 650247 it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") it.totalDifficulty = new BigInteger("35bbde5595de6457", 16) + it.timestamp = Instant.now() return it } api.answer("eth_getBlockByHash", [block1.hash.toHex(), false], block1) @@ -197,7 +204,7 @@ class EthereumGrpcUpstreamSpec extends Specification { def h = upstream.head.getFlux().take(Duration.ofSeconds(1)).last().block() then: upstream.status == UpstreamAvailability.OK - h.hash == BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") - h.number == 650247 + h.hash == BlockId.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec891521a") + h.height == 650247 } }