StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solution: cache latest blocks in memory

Browse Source
This commit is contained in:
Igor Artamonov
2019-08-20 23:14:49 -04:00
parent 887cf07e16
commit 8a99212fc5
28 changed files with 476 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/test/groovy/io/emeraldpay/dshackle/cache/BlocksMemCacheSpec.groovy vendored Normal file
View File

@@ -0,0 +1,52 @@
package io.emeraldpay.dshackle.cache
import io.infinitape.etherjar.domain.BlockHash
import io.infinitape.etherjar.domain.TransactionId
import io.infinitape.etherjar.rpc.json.BlockJson
import spock.lang.Specification
class BlocksMemCacheSpec extends Specification {
String hash1 = "0xd3f34def3c56ba4e701540d15edaff9acd2a1c968a7ff83b3300ab5dfd5f6aab"
String hash2 = "0x4aabdaff9acd2f30d15e00ab5dfd5f6c56ba4ea1c968a7ff8d3f34de70153b33"
String hash3 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String hash4 = "0xa4e7a75dfd5f6a83b3304dc56bfa0abfd3fef01540d15edafc9683f9acd2a13b"
def "Add and read"() {
setup:
def cache = new BlocksMemCache()
def block = new BlockJson<TransactionId>()
block.number = 100
block.hash = BlockHash.from(hash1)
when:
cache.add(block)
def act = cache.get(BlockHash.from(hash1)).block()
then:
act == block
}
def "Keeps only configured amount"() {
setup:
def cache = new BlocksMemCache(3)
[hash1]
when:
[hash1, hash2, hash3, hash4].eachWithIndex{ String hash, int i ->
def block = new BlockJson<TransactionId>()
block.number = 100 + i
block.hash = BlockHash.from(hash)
cache.add(block)
}
def act1 = cache.get(BlockHash.from(hash1)).block()
def act2 = cache.get(BlockHash.from(hash2)).block()
def act3 = cache.get(BlockHash.from(hash3)).block()
def act4 = cache.get(BlockHash.from(hash4)).block()
then:
act2.hash.toHex() == hash2
act3.hash.toHex() == hash3
act4.hash.toHex() == hash4
act1 == null
}
}

58
src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy
View File

@@ -1,17 +1,23 @@
package io.emeraldpay.dshackle.rpc
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.test.EthereumApiMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.AggregatedUpstream
import io.emeraldpay.dshackle.upstream.AlwaysQuorum
import io.emeraldpay.dshackle.upstream.CachingEthereumApi
import io.emeraldpay.dshackle.upstream.CallQuorum
import io.emeraldpay.dshackle.upstream.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.EthereumApi
import io.emeraldpay.dshackle.upstream.NonEmptyQuorum
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.Upstreams
import io.emeraldpay.grpc.Chain
import io.infinitape.etherjar.rpc.RpcClient
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import reactor.util.function.Tuples
import spock.lang.Specification
@@ -33,7 +39,9 @@ class NativeCallSpec extends Specification {
apiMock.answer("eth_test", [], "foo")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, [apiMock].multiply(59).iterator(), quorum, Tuples.of("eth_test", []))
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
@@ -59,7 +67,9 @@ class NativeCallSpec extends Specification {
apiMock.answerOnce("eth_test", [], null)
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, [apiMock].multiply(5).iterator(), quorum, Tuples.of("eth_test", []))
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
@@ -85,7 +95,8 @@ class NativeCallSpec extends Specification {
apiMock.answerOnce("eth_test", [], "foo")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, [apiMock].multiply(5).iterator(), quorum, Tuples.of("eth_test", []))
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock), Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
(4..5) * quorum.isResolved()
3 * quorum.record(_, _)
@@ -135,9 +146,10 @@ class NativeCallSpec extends Specification {
def upstreams = Stub(Upstreams)
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def json = [jsonrpc:"2.0", id:1, result: "foo"]
when:
def resp = nativeCall.buildResponse(
new NativeCall.CallContext<byte[]>(1561, [].iterator(), new AlwaysQuorum(), objectMapper.writeValueAsBytes(json))
new NativeCall.CallContext<byte[]>(1561, TestingCommons.aggregatedUpstream(Stub(DirectEthereumApi)), Selector.empty, new AlwaysQuorum(), objectMapper.writeValueAsBytes(json))
)
then:
resp.id == 1561
@@ -191,4 +203,42 @@ class NativeCallSpec extends Specification {
// .expectComplete()
.verify(Duration.ofSeconds(1))
}
def "Calls cache before remote"() {
setup:
def upstreams = Stub(Upstreams)
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def api = Mock(DirectEthereumApi)
def upstream = TestingCommons.aggregatedUpstream(api)
def cacheMock = Mock(CachingEthereumApi)
upstream.cache = cacheMock
def ctx = new NativeCall.CallContext<NativeCall.ParsedCallDetails>(10,
upstream,
Selector.empty, new AlwaysQuorum(),
new NativeCall.ParsedCallDetails("eth_test", []))
when:
nativeCall.fetch(ctx)
then:
1 * cacheMock.execute(10, "eth_test", []) >> Mono.empty()
}
def "Uses cached value"() {
setup:
def upstreams = Stub(Upstreams)
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def upstream = TestingCommons.aggregatedUpstream(Stub(DirectEthereumApi))
def cacheMock = Mock(CachingEthereumApi)
upstream.cache = cacheMock
def ctx = new NativeCall.CallContext<NativeCall.ParsedCallDetails>(10,
upstream,
Selector.empty, new AlwaysQuorum(),
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def act = nativeCall.fetch(ctx)
then:
1 * cacheMock.execute(10, "eth_test", []) >> Mono.just('{"result": "foo"}'.bytes)
new String(act.block().payload) == '{"result": "foo"}'
}
}

3
src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy
View File

@@ -5,6 +5,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.UpstreamsMock
import io.emeraldpay.dshackle.upstream.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.EthereumApi
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.grpc.Chain
@@ -57,7 +58,7 @@ class StreamHeadSpec extends Specification {
.build()
}
def upstream = new EthereumUpstreamMock(Chain.ETHEREUM, Mock(EthereumApi))
def upstream = new EthereumUpstreamMock(Chain.ETHEREUM, Stub(DirectEthereumApi.class))
def upstreams = new UpstreamsMock(Chain.ETHEREUM, upstream)
def streamHead = new StreamHead(upstreams)
when:

5
src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiMock.groovy
View File

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.upstream.DirectCallMethods
import io.emeraldpay.dshackle.upstream.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.EthereumApi
import io.emeraldpay.dshackle.upstream.QuorumBasedMethods
import io.emeraldpay.dshackle.upstream.Upstream
@@ -17,14 +18,14 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
class EthereumApiMock extends EthereumApi {
class EthereumApiMock extends DirectEthereumApi {
private static final Logger log = LoggerFactory.getLogger(this)
List<PredefinedResponse> predefined = []
private ObjectMapper objectMapper
EthereumApiMock(@NotNull RpcClient rpcClient, @NotNull ObjectMapper objectMapper, @NotNull Chain chain) {
super(rpcClient, objectMapper, chain, new DirectCallMethods())
super(rpcClient, objectMapper, new DirectCallMethods())
this.objectMapper = objectMapper
}

3
src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy
View File

@@ -1,5 +1,6 @@
package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.upstream.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.EthereumApi
import io.emeraldpay.dshackle.upstream.EthereumHead
import io.emeraldpay.dshackle.upstream.EthereumUpstream
@@ -13,7 +14,7 @@ class EthereumUpstreamMock extends EthereumUpstream {
EthereumHeadMock ethereumHeadMock = new EthereumHeadMock()
EthereumUpstreamMock(@NotNull Chain chain, @NotNull EthereumApi api) {
EthereumUpstreamMock(@NotNull Chain chain, @NotNull DirectEthereumApi api) {
super(chain, api)
setLag(0)
setStatus(UpstreamAvailability.OK)

10
src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy
View File

@@ -4,6 +4,10 @@ import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.module.SimpleModule
import io.emeraldpay.dshackle.upstream.AggregatedUpstream
import io.emeraldpay.dshackle.upstream.ChainUpstreams
import io.emeraldpay.dshackle.upstream.DirectCallMethods
import io.emeraldpay.dshackle.upstream.DirectEthereumApi
import io.emeraldpay.dshackle.upstream.EthereumApi
import io.emeraldpay.dshackle.upstream.EthereumUpstream
import io.emeraldpay.dshackle.upstream.Upstream
@@ -42,7 +46,11 @@ class TestingCommons {
return new JacksonRpcConverter(objectMapper())
}
static EthereumUpstreamMock upstream(EthereumApi api) {
static EthereumUpstreamMock upstream(DirectEthereumApi api) {
return new EthereumUpstreamMock(Chain.ETHEREUM, api)
}
static AggregatedUpstream aggregatedUpstream(DirectEthereumApi api) {
return new ChainUpstreams(Chain.ETHEREUM, [upstream(api)], new DirectCallMethods(), objectMapper())
}
}

2
src/test/groovy/io/emeraldpay/dshackle/test/UpstreamsMock.groovy
View File

@@ -26,7 +26,7 @@ class UpstreamsMock implements Upstreams {
@Override
AggregatedUpstream addUpstream(@NotNull Chain chain, @NotNull Upstream up) {
if (!upstreams.containsKey(chain)) {
upstreams[chain] = new ChainUpstreams(chain, [up], targetFor(chain))
upstreams[chain] = new ChainUpstreams(chain, [up], targetFor(chain), TestingCommons.objectMapper())
} else {
upstreams[chain].addUpstream(up)
}

17
src/test/groovy/io/emeraldpay/dshackle/upstream/EthereumGrpcTransportSpec.groovy
View File

@@ -23,11 +23,13 @@ class EthereumGrpcTransportSpec extends Specification {
def "Make simple call"() {
setup:
def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM)
def callData = [:]
def otherSideUpstreams = Mock(Upstreams)
def otherSideAggr = Mock(AggregatedUpstream)
def otherSideAggr = TestingCommons.aggregatedUpstream(otherSideApi)
def otherSideNativeCall = new NativeCall(otherSideUpstreams, objectMapper)
def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM)
otherSideApi.upstream = otherSideAggr
def client = mockServer.clientForServer(new ReactorBlockchainGrpc.BlockchainImplBase() {
@@ -47,9 +49,6 @@ class EthereumGrpcTransportSpec extends Specification {
then:
1 * otherSideUpstreams.getUpstream(Chain.ETHEREUM) >> otherSideAggr
1 * otherSideAggr.getApis(_) >> [otherSideApi].iterator()
_ * otherSideAggr.getHead() >> Stub(EthereumHead)
_ * otherSideAggr.getTargets() >> ethereumTargets
status.failed == 0
status.succeed == 1
status.total == 1
@@ -67,11 +66,12 @@ class EthereumGrpcTransportSpec extends Specification {
def "Make few calls"() {
setup:
def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM)
def callData = [:]
def otherSideUpstreams = Mock(Upstreams)
def otherSideAggr = Mock(AggregatedUpstream)
def otherSideAggr = TestingCommons.aggregatedUpstream(otherSideApi)
def otherSideNativeCall = new NativeCall(otherSideUpstreams, objectMapper)
def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM)
otherSideApi.upstream = otherSideAggr
def client = mockServer.clientForServer(new ReactorBlockchainGrpc.BlockchainImplBase() {
@@ -95,9 +95,6 @@ class EthereumGrpcTransportSpec extends Specification {
then:
1 * otherSideUpstreams.getUpstream(Chain.ETHEREUM) >> otherSideAggr
1 * otherSideAggr.getApis(_) >> [otherSideApi].multiply(34).iterator()
_ * otherSideAggr.getHead() >> Stub(EthereumHead)
_ * otherSideAggr.getTargets() >> ethereumTargets
status.failed == 0
status.succeed == 2
status.total == 2

2
src/test/groovy/io/emeraldpay/dshackle/upstream/FilteringApiIteratorSpec.groovy
View File

@@ -23,7 +23,7 @@ class FilteringApiIteratorSpec extends Specification {
].collect {
new EthereumUpstream(
Chain.ETHEREUM,
new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM, ethereumTargets),
new DirectEthereumApi(rpcClient, objectMapper, ethereumTargets),
(EthereumWs) null,
new UpstreamsConfig.Options(),
new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap(it)),