StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

problem: different places to fetch from upstream

Browse Source 
solution: single access reader for Ethereum upstreams
This commit is contained in:
Igor Artamonov
2020-05-04 21:51:42 -04:00
parent 3a13fb3bd5
commit 0ef46b2980
25 changed files with 748 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/test/groovy/io/emeraldpay/dshackle/reader/RekeyingReaderSpec.groovy Normal file
View File

@@ -0,0 +1,40 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.reader
import reactor.core.publisher.Mono
import spock.lang.Specification
import java.util.function.Function
class RekeyingReaderSpec extends Specification {
def "Simple rekey"() {
setup:
Reader<Long, Long> r = new Reader<Long, Long>() {
@Override
Mono<Long> read(Long key) {
return Mono.just(key * 2)
}
}
Function<String, Long> f = { String s -> Long.parseLong(s) }
when:
def rekey = new RekeyingReader(f, r)
def act = rekey.read("4").block()
then:
act == 8
}
}

40
src/test/groovy/io/emeraldpay/dshackle/reader/TransformingReaderSpec.groovy Normal file
View File

@@ -0,0 +1,40 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.reader
import reactor.core.publisher.Mono
import spock.lang.Specification
import java.util.function.Function
class TransformingReaderSpec extends Specification {
def "Transform"() {
Reader<String, String> r = new Reader<String, String>() {
@Override
Mono<String> read(String key) {
return Mono.just(key + "2")
}
}
Function<String, Long> f = { String s -> Long.parseLong(s) }
when:
def transform = new TransformingReader(r, f)
def act = transform.read("4").block()
then:
act == 42L
}
}

8
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinTxSpec.groovy
View File

@@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinData
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinReader
import io.emeraldpay.dshackle.upstream.bitcoin.DirectBitcoinApi
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream
import io.emeraldpay.dshackle.upstream.bitcoin.CachingMempoolData
@@ -46,7 +46,7 @@ class TrackBitcoinTxSpec extends Specification {
])
}
BitcoinUpstream upstream = Mock(BitcoinUpstream) {
_ * getData() >> Mock(BitcoinData) {
_ * getData() >> Mock(BitcoinReader) {
_ * getMempool() >> mempoolAccess
}
}
@@ -72,7 +72,7 @@ class TrackBitcoinTxSpec extends Specification {
])
}
BitcoinUpstream upstream = Mock(BitcoinUpstream) {
_ * getData() >> Mock(BitcoinData) {
_ * getData() >> Mock(BitcoinReader) {
_ * getMempool() >> mempoolAccess
}
}
@@ -231,7 +231,7 @@ class TrackBitcoinTxSpec extends Specification {
BitcoinUpstream upstream = Mock(BitcoinUpstream) {
_ * getApi(_) >> Mono.just(api)
_ * getHead() >> head
_ * getData() >> Mock(BitcoinData) {
_ * getData() >> Mock(BitcoinReader) {
_ * getMempool() >> mempoolAccess
}
}

10
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumAddressSpec.groovy
View File

@@ -19,14 +19,18 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.test.UpstreamsMock
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.dshackle.upstream.ethereum.EthereumReader
import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.domain.Address
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.rpc.ReactorRpcClient
import io.infinitape.etherjar.rpc.RpcCall
import io.infinitape.etherjar.rpc.json.BlockJson
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.TopicProcessor
import reactor.core.scheduler.Schedulers
@@ -111,12 +115,12 @@ class TrackEthereumAddressSpec extends Specification {
def flux = trackAddress.subscribe(req)
then:
StepVerifier.create(flux)
.expectNext(exp1)
.expectNext(exp1).as("First block")
.then {
upstreamMock.nextBlock(BlockContainer.from(block2, TestingCommons.objectMapper()))
}
.expectNext(exp2)
.expectNext(exp2).as("Second block")
.thenCancel()
.verify(Duration.ofSeconds(3))
.verify(Duration.ofSeconds(1))
}
}

8
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy
View File

@@ -27,8 +27,7 @@ import io.emeraldpay.dshackle.test.UpstreamsMock
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.dshackle.upstream.ethereum.EthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainUpstreams
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWs
import io.emeraldpay.dshackle.upstream.ethereum.AggregatedEthereumUpstreams
import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
@@ -39,7 +38,6 @@ import io.infinitape.etherjar.rpc.json.TransactionRefJson
import reactor.core.publisher.Flux
import reactor.test.StepVerifier
import reactor.test.scheduler.VirtualTimeScheduler
import spock.lang.Ignore
import spock.lang.Specification
import java.time.Duration
@@ -68,7 +66,7 @@ class TrackEthereumTxSpec extends Specification {
}
def blockHeadJson = new BlockJson().with {
it.hash = BlockHash.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22")
it.hash = BlockHash.from("0x552d882f16f34a9dba2ccdc05c0a6a27c22a0e65cbc1b52a8ca60562112c6060")
it.timestamp = Instant.ofEpochMilli(156400200000)
it.number = 108
it.totalDifficulty = BigInteger.valueOf(800)
@@ -123,7 +121,7 @@ class TrackEthereumTxSpec extends Specification {
def apiMock = TestingCommons.api(Stub(ReactorRpcClient))
def upstreamMock = TestingCommons.upstream(apiMock)
Upstreams upstreams = new UpstreamsMock(Chain.ETHEREUM, upstreamMock)
((EthereumChainUpstreams) upstreams.getUpstream(Chain.ETHEREUM)).head = Mock(Head) {
((AggregatedEthereumUpstreams) upstreams.getUpstream(Chain.ETHEREUM)).head = Mock(Head) {
_ * getFlux() >> Flux.empty()
}
TrackEthereumTx trackTx = new TrackEthereumTx(upstreams)

6
src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy
View File

@@ -25,15 +25,13 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesFactory
import io.emeraldpay.dshackle.config.CacheConfig
import io.emeraldpay.dshackle.upstream.AggregatedUpstream
import io.emeraldpay.dshackle.upstream.ChainUpstreams
import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainUpstreams
import io.emeraldpay.dshackle.upstream.ethereum.AggregatedEthereumUpstreams
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.rpc.JacksonRpcConverter
import io.infinitape.etherjar.rpc.ReactorRpcClient
import org.springframework.core.env.StandardEnvironment
import java.text.SimpleDateFormat
@@ -77,7 +75,7 @@ class TestingCommons {
}
static AggregatedUpstream aggregatedUpstream(EthereumUpstream up) {
return new EthereumChainUpstreams(Chain.ETHEREUM, [up], Caches.default(objectMapper()), objectMapper())
return new AggregatedEthereumUpstreams(Chain.ETHEREUM, [up], Caches.default(objectMapper()), objectMapper())
}
static CachesFactory emptyCaches() {

34
src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy
View File

@@ -16,13 +16,15 @@
*/
package io.emeraldpay.dshackle.test
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.upstream.AggregatedUpstream
import io.emeraldpay.dshackle.upstream.ChainUpstreams
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainUpstreams
import io.emeraldpay.dshackle.upstream.ethereum.AggregatedEthereumUpstreams
import io.emeraldpay.dshackle.upstream.ethereum.EthereumReader
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.grpc.Chain
import org.jetbrains.annotations.NotNull
import reactor.core.publisher.Flux
@@ -30,7 +32,7 @@ import reactor.core.publisher.Flux
class UpstreamsMock implements Upstreams {
private Map<Chain, DefaultEthereumMethods> target = [:]
private Map<Chain, AggregatedUpstream> upstreams = [:]
private Map<Chain, AggregatedEthereumUpstreamsMock> upstreams = [:]
UpstreamsMock(Chain chain, Upstream up) {
addUpstream(chain, up)
@@ -40,15 +42,20 @@ class UpstreamsMock implements Upstreams {
addUpstream(chain2, up2)
}
AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull Upstream up) {
AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull EthereumUpstream up) {
if (!upstreams.containsKey(chain)) {
upstreams[chain] = new EthereumChainUpstreams(chain, [up], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
upstreams[chain] = new AggregatedEthereumUpstreamsMock(chain, [up], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
upstreams[chain].start()
} else {
upstreams[chain].addUpstream(up)
}
return upstreams[chain]
}
void setReader(@NotNull Chain chain, EthereumReader reader) {
upstreams[chain].customReader = reader
}
@Override
AggregatedUpstream getUpstream(@NotNull Chain chain) {
return upstreams[chain]
@@ -78,4 +85,21 @@ class UpstreamsMock implements Upstreams {
return upstreams.containsKey(chain)
}
static class AggregatedEthereumUpstreamsMock extends AggregatedEthereumUpstreams {
EthereumReader customReader = null
AggregatedEthereumUpstreamsMock(@NotNull Chain chain, @NotNull List<EthereumUpstream> upstreams, @NotNull Caches caches, @NotNull ObjectMapper objectMapper) {
super(chain, upstreams, caches, objectMapper)
}
@Override
EthereumReader getReader() {
if (customReader != null) {
return customReader
}
return super.getReader()
}
}
}

4
src/test/groovy/io/emeraldpay/dshackle/upstream/AggregatedUpstreamSpec.groovy
View File

@@ -22,7 +22,7 @@ import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainUpstreams
import io.emeraldpay.dshackle.upstream.ethereum.AggregatedEthereumUpstreams
import io.emeraldpay.grpc.Chain
import spock.lang.Specification
@@ -32,7 +32,7 @@ class AggregatedUpstreamSpec extends Specification {
setup:
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test1", "eth_test2"]))
def up2 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, Stub(DirectEthereumApi), new DirectCallMethods(["eth_test2", "eth_test3"]))
def aggr = new EthereumChainUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
def aggr = new AggregatedEthereumUpstreams(Chain.ETHEREUM, [up1, up2], Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
when:
aggr.onUpstreamsUpdated()
def act = aggr.getMethods()

237
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumReaderSpec.groovy Normal file
View File

@@ -0,0 +1,237 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.cache.BlocksMemCache
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.TxMemCache
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.domain.Address
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.domain.Wei
import io.infinitape.etherjar.rpc.ReactorRpcClient
import io.infinitape.etherjar.rpc.json.BlockJson
import io.infinitape.etherjar.rpc.json.TransactionJson
import io.infinitape.etherjar.rpc.json.TransactionRefJson
import reactor.core.publisher.Mono
import spock.lang.Specification
import java.time.Instant
class EthereumReaderSpec extends Specification {
def blockId = BlockId.from("f85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2")
def blockJson = new BlockJson<TransactionRefJson>().tap { blockJson ->
blockJson.hash = BlockHash.from(blockId.value)
blockJson.totalDifficulty = BigInteger.ONE
blockJson.number = 101
blockJson.timestamp = Instant.ofEpochSecond(100000000)
blockJson.transactions = []
blockJson.uncles = []
}
def txId = BlockId.from("a38e7b4d456777c94b46c61a1e4cf52fbdd92acc4444719d1fad77005698c221")
def txJson = new TransactionJson().tap { json ->
json.hash = TransactionId.from(txId.value)
json.blockHash = blockJson.hash
json.blockNumber = blockJson.number
}
def "Block by Id reads from cache"() {
setup:
def memCache = Mock(BlocksMemCache) {
1 * read(blockId) >> Mono.just(BlockContainer.from(blockJson, TestingCommons.objectMapper()))
}
def caches = Caches.newBuilder()
.setBlockByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def reader = new EthereumReader(Stub(Upstream), caches, TestingCommons.objectMapper())
when:
def act = reader.blocksById().read(blockId).block()
then:
act == blockJson
}
def "Block by Id reads from api if cache is empty"() {
setup:
def memCache = Mock(BlocksMemCache) {
1 * read(blockId) >> Mono.empty()
}
def caches = Caches.newBuilder()
.setBlockByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def rpcClient = Stub(ReactorRpcClient)
def api = TestingCommons.api(rpcClient)
api.answer("eth_getBlockByHash", ["0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2", false], blockJson)
def upstream = TestingCommons.aggregatedUpstream(api)
def reader = new EthereumReader(upstream, caches, TestingCommons.objectMapper())
when:
def act = reader.blocksById().read(blockId).block()
then:
act == blockJson
}
def "Block by Id reads from api if cache failed"() {
setup:
def memCache = Mock(BlocksMemCache) {
1 * read(blockId) >> Mono.error(new IllegalStateException("Test error"))
}
def caches = Caches.newBuilder()
.setBlockByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def rpcClient = Stub(ReactorRpcClient)
def api = TestingCommons.api(rpcClient)
api.answer("eth_getBlockByHash", ["0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2", false], blockJson)
def upstream = TestingCommons.aggregatedUpstream(api)
def reader = new EthereumReader(upstream, caches, TestingCommons.objectMapper())
when:
def act = reader.blocksById().read(blockId).block()
then:
act == blockJson
}
def "Block by Hash reads from cache"() {
setup:
def memCache = Mock(BlocksMemCache) {
1 * read(blockId) >> Mono.just(BlockContainer.from(blockJson, TestingCommons.objectMapper()))
}
def caches = Caches.newBuilder()
.setBlockByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def reader = new EthereumReader(Stub(Upstream), caches, TestingCommons.objectMapper())
when:
def act = reader.blocksByHash().read(blockJson.hash).block()
then:
act == blockJson
}
def "Block by Hash reads from api if cache is empty"() {
setup:
def memCache = Mock(BlocksMemCache) {
1 * read(blockId) >> Mono.empty()
}
def caches = Caches.newBuilder()
.setBlockByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def rpcClient = Stub(ReactorRpcClient)
def api = TestingCommons.api(rpcClient)
api.answer("eth_getBlockByHash", ["0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2", false], blockJson)
def upstream = TestingCommons.aggregatedUpstream(api)
def reader = new EthereumReader(upstream, caches, TestingCommons.objectMapper())
when:
def act = reader.blocksByHash().read(blockJson.hash).block()
then:
act == blockJson
}
def "Tx by Hash reads from cache"() {
setup:
def memCache = Mock(TxMemCache) {
1 * read(txId) >> Mono.just(TxContainer.from(txJson, TestingCommons.objectMapper()))
}
def caches = Caches.newBuilder()
.setTxByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def reader = new EthereumReader(Stub(Upstream), caches, TestingCommons.objectMapper())
when:
def act = reader.txByHash().read(txJson.hash).block()
then:
act == txJson
}
def "Tx by Hash reads from api if cache is empty"() {
setup:
def memCache = Mock(TxMemCache) {
1 * read(txId) >> Mono.empty()
}
def caches = Caches.newBuilder()
.setTxByHash(memCache)
.setObjectMapper(TestingCommons.objectMapper())
.build()
def rpcClient = Stub(ReactorRpcClient)
def api = TestingCommons.api(rpcClient)
api.answer("eth_getTransactionByHash", [txJson.hash.toHex()], txJson)
def upstream = TestingCommons.aggregatedUpstream(api)
def reader = new EthereumReader(upstream, caches, TestingCommons.objectMapper())
when:
def act = reader.txByHash().read(txJson.hash).block()
then:
act == txJson
}
def "Caches balance until block mined"() {
setup:
def rpcClient = Stub(ReactorRpcClient)
def api = TestingCommons.api(rpcClient)
api.answerOnce("eth_getBalance", ["0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c", "latest"], "0x10")
api.answerOnce("eth_getBalance", ["0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c", "latest"], "0xff")
def upstream = TestingCommons.upstream(api)
def reader = new EthereumReader(upstream, Caches.default(TestingCommons.objectMapper()), TestingCommons.objectMapper())
reader.start()
when:
def act = reader.balance().read(Address.from("0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c")).block()
then:
act == Wei.from("0x10")
when:
//now it should use cached value, without actual request
act = reader.balance().read(Address.from("0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c")).block()
then:
act == Wei.from("0x10")
when:
//move head forward, which should erase cache
def block2 = blockJson.copy().tap {
it.number++
it.totalDifficulty = BigInteger.TWO
}
upstream.nextBlock(BlockContainer.from(block2, TestingCommons.objectMapper()))
act = reader.balance().read(Address.from("0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c")).block()
then:
act == Wei.from("0xff")
}
}