solution: individual class for quorum based requests

This commit is contained in:
Igor Artamonov
2020-05-15 21:06:32 -04:00
parent 73f55d8d81
commit 114f109722
19 changed files with 611 additions and 235 deletions

View File

@@ -0,0 +1,133 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.infinitape.etherjar.rpc.RpcException
import spock.lang.Specification
class NonEmptyQuorumSpec extends Specification {
def "Fail if too many errors"() {
setup:
def q = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstream1 = Stub(Upstream)
def upstream2 = Stub(Upstream)
def upstream3 = Stub(Upstream)
when:
q.init(Stub(Head))
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream1)
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream2)
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream3)
then:
q.isFailed()
!q.isResolved()
}
def "Fail first if not error"() {
setup:
def q = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstream1 = Stub(Upstream)
def upstream2 = Stub(Upstream)
def upstream3 = Stub(Upstream)
when:
q.init(Stub(Head))
then:
!q.isResolved()
!q.isFailed()
when:
q.record('"0x11"'.bytes, upstream1)
then:
q.isResolved()
!q.isFailed()
}
def "Fail second if first is error"() {
setup:
def q = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstream1 = Stub(Upstream)
def upstream2 = Stub(Upstream)
def upstream3 = Stub(Upstream)
when:
q.init(Stub(Head))
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream1)
then:
!q.isFailed()
!q.isResolved()
when:
q.record('"0x11"'.bytes, upstream2)
then:
q.isResolved()
!q.isFailed()
}
def "Fail second if first is null"() {
setup:
def q = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstream1 = Stub(Upstream)
def upstream2 = Stub(Upstream)
def upstream3 = Stub(Upstream)
when:
q.init(Stub(Head))
then:
!q.isResolved()
!q.isFailed()
when:
q.record('null'.bytes, upstream2)
then:
!q.isFailed()
!q.isResolved()
when:
q.record('"0x11"'.bytes, upstream2)
then:
q.isResolved()
!q.isFailed()
}
}

View File

@@ -96,4 +96,36 @@ class NonceQuorumSpec extends Specification {
q.isResolved()
objectMapper.readValue(q.result, Object) == "0x11"
}
def "Fail if too many errors"() {
setup:
def q = Spy(new NonceQuorum(objectMapper, 3))
def upstream1 = Stub(Upstream)
def upstream2 = Stub(Upstream)
def upstream3 = Stub(Upstream)
when:
q.init(Stub(Head))
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream1)
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream2)
then:
!q.isResolved()
!q.isFailed()
when:
q.record(new RpcException(1, "Internal"), upstream3)
then:
q.isFailed()
!q.isResolved()
}
}

View File

@@ -0,0 +1,206 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.FilteredApis
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import spock.lang.Specification
import java.time.Duration
class QuorumRpcReaderSpec extends Specification {
def "always-quorum - get the result if ok"() {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
1 * getApi() >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_test", [])) >> Mono.just(JsonRpcResponse.ok("1"))
}
}
def apis = new FilteredApis(
[up], Selector.empty
)
def reader = new QuorumRpcReader(apis, new AlwaysQuorum())
when:
def act = reader.read(new JsonRpcRequest("eth_test", []))
.map {
new String(it.value)
}
then:
StepVerifier.create(act)
.expectNext("1")
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "always-quorum - retry upstream error"() {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getApi() >> Mock(Reader) {
2 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.error(1, "test")),
Mono.just(JsonRpcResponse.ok("1"))
]
}
}
def apis = new FilteredApis(
[up], Selector.empty
)
def reader = new QuorumRpcReader(apis, new AlwaysQuorum())
when:
def act = reader.read(new JsonRpcRequest("eth_test", []))
.map {
new String(it.value)
}
then:
StepVerifier.create(act)
.expectNext("1")
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "non-empty-quorum - get the second result if first is null"() {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getApi() >> Mock(Reader) {
2 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.ok("null")),
Mono.just(JsonRpcResponse.ok("1"))
]
}
}
def apis = new FilteredApis(
[up], Selector.empty
)
def reader = new QuorumRpcReader(apis, new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
when:
def act = reader.read(new JsonRpcRequest("eth_test", []))
.map {
new String(it.value)
}
then:
StepVerifier.create(act)
.expectNext("1")
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "non-empty-quorum - get the second result if first is error"() {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getApi() >> Mock(Reader) {
2 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.error(1, "test")),
Mono.just(JsonRpcResponse.ok("1"))
]
}
}
def apis = new FilteredApis(
[up], Selector.empty
)
def reader = new QuorumRpcReader(apis, new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
when:
def act = reader.read(new JsonRpcRequest("eth_test", []))
.map {
new String(it.value)
}
then:
StepVerifier.create(act)
.expectNext("1")
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "non-empty-quorum - get the third result if first two are not ok"() {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getApi() >> Mock(Reader) {
3 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.ok("null")),
Mono.just(JsonRpcResponse.error(1, "test")),
Mono.just(JsonRpcResponse.ok("1"))
]
}
}
def apis = new FilteredApis(
[up], Selector.empty
)
def reader = new QuorumRpcReader(apis, new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
when:
def act = reader.read(new JsonRpcRequest("eth_test", []))
.map {
new String(it.value)
}
then:
StepVerifier.create(act)
.expectNext("1")
.expectComplete()
.verify(Duration.ofSeconds(1))
}
def "non-empty-quorum - no result if all failed"() {
setup:
def up = Mock(Upstream) {
_ * isAvailable() >> true
_ * getApi() >> Mock(Reader) {
3 * read(new JsonRpcRequest("eth_test", [])) >>> [
Mono.just(JsonRpcResponse.ok("null")),
Mono.just(JsonRpcResponse.error(1, "test")),
Mono.just(JsonRpcResponse.ok("null"))
]
}
}
def apis = new FilteredApis(
[up], Selector.empty
)
def reader = new QuorumRpcReader(apis, new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
when:
def act = reader.read(new JsonRpcRequest("eth_test", []))
.map {
new String(it.value)
}
then:
StepVerifier.create(act)
.expectComplete()
.verify(Duration.ofSeconds(1))
}
}

View File

@@ -89,5 +89,10 @@ class ValueAwareQuorumSpec extends Specification {
byte[] getResult() {
return new byte[0]
}
@Override
boolean isFailed() {
return false
}
}
}

View File

@@ -19,6 +19,8 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.quorum.BroadcastQuorum
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
import io.emeraldpay.dshackle.quorum.QuorumRpcReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
@@ -96,126 +98,43 @@ class NativeCallSpec extends Specification {
def "Quorum is applied"() {
setup:
def quorum = Spy(new AlwaysQuorum())
def upstreams = Stub(MultistreamHolder)
def apiMock = TestingCommons.api()
def quorum = new AlwaysQuorum()
apiMock.answer("eth_test", [], "foo")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
def nativeCall = new NativeCall(Stub(MultistreamHolder), TestingCommons.objectMapper())
nativeCall.quorumReaderFactory = Mock(QuorumReaderFactory) {
1 * create(_, _) >> Mock(Reader) {
1 * read(_) >> Mono.just(new QuorumRpcReader.Result("\"foo\"".bytes, 1))
}
}
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(TestingCommons.api()), Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(1))
def act = objectMapper.readValue(resp.payload, Object)
then:
act == "foo"
1 * quorum.record(_, _)
1 * quorum.getResult()
}
def "Quorum may return not first received value"() {
setup:
def quorum = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstreams = Stub(MultistreamHolder)
def apiMock = TestingCommons.api()
apiMock.answerOnce("eth_test", [], null)
apiMock.answerOnce("eth_test", [], "bar")
apiMock.answerOnce("eth_test", [], null)
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
def act = objectMapper.readValue(resp.payload, Object)
then:
act == "bar"
2 * quorum.record(_, _)
1 * quorum.getResult()
}
def "Have pause between repeats"() {
setup:
def quorum = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstreams = Stub(MultistreamHolder)
def apiMock = TestingCommons.api()
apiMock.answerOnce("eth_test", [], null)
apiMock.answerOnce("eth_test", [], "bar")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def t1 = System.currentTimeMillis()
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
def delta = System.currentTimeMillis() - t1
def act = objectMapper.readValue(resp.payload, Object)
then:
delta > 95 // should be 100, but sometimes gives less ???
act == "bar"
}
def "One call has no pause"() {
setup:
def quorum = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def upstreams = Stub(MultistreamHolder)
ReactorRpcClient rpcClient = Stub(ReactorRpcClient)
def apiMock = TestingCommons.api()
apiMock.answerOnce("eth_test", [], "bar")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def t1 = System.currentTimeMillis()
nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
def delta = System.currentTimeMillis() - t1
then:
delta < 50
}
def "Returns error if no quorum"() {
setup:
def quorum = Spy(new NonEmptyQuorum(TestingCommons.objectMapper(), 3))
def quorum = new AlwaysQuorum()
def upstreams = Stub(MultistreamHolder)
ReactorRpcClient rpcClient = Stub(ReactorRpcClient)
def apiMock = TestingCommons.api()
apiMock.answer("eth_test", [], null, 3)
apiMock.answerOnce("eth_test", [], "foo")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock), Selector.empty, quorum,
def nativeCall = new NativeCall(Stub(MultistreamHolder), TestingCommons.objectMapper())
nativeCall.quorumReaderFactory = Mock(QuorumReaderFactory) {
1 * create(_, _) >> Mock(Reader) {
1 * read(_) >> Mono.empty()
}
}
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(TestingCommons.api()), Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
3 * quorum.record(_, _)
1 * quorum.getResult()
when:
def resp = nativeCall.executeOnRemote(call)
then:
StepVerifier.create(resp)
.expectErrorMatches({t -> t instanceof NativeCall.CallFailure && t.id == 1})
.verify(Duration.ofSeconds(1))
.expectErrorMatches({ t -> t instanceof NativeCall.CallFailure && t.id == 1 })
.verify(Duration.ofSeconds(1))
}
def "Packs call exception into response with id"() {
@@ -351,59 +270,4 @@ class NativeCallSpec extends Specification {
1 * cacheMock.execute(10, "eth_test", []) >> Mono.just('{"result": "foo"}'.bytes)
new String(act.block().payload) == '{"result": "foo"}'
}
def "Retries on error"() {
setup:
def quorum = Spy(new AlwaysQuorum())
def upstreams = Stub(MultistreamHolder)
def apiMock = TestingCommons.api()
apiMock.answer("eth_test", [], null, 1, new TimeoutException("test 1"))
apiMock.answer("eth_test", [], null, 1, new TimeoutException("test 2"))
apiMock.answerOnce("eth_test", [], "bar")
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_test", []))
when:
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
def act = objectMapper.readValue(resp.payload, Object)
then:
act == "bar"
1 * quorum.record(_, _)
1 * quorum.getResult()
}
def "Send raw retries 3 times"() {
setup:
def quorum = Spy(new BroadcastQuorum(TestingCommons.objectMapper(), 3))
def upstreams = Stub(MultistreamHolder)
def apiMock = TestingCommons.api()
apiMock.answer("eth_sendRawTransaction", ["0x1234"],
"0x4b66b555df9faed6f0711f2104d183736c8e2dc7434626dd2622e243f041d41b", 1)
apiMock.answer("eth_sendRawTransaction", ["0x1234"], null, 10,
new RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Transaction with the same hash was already imported"))
// apiMock.answer("eth_sendRawTransaction", ["0x1234"],
// new RpcResponseError(RpcResponseError.CODE_INVALID_REQUEST, "Transaction with the same hash was already imported"), 10)
def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
def call = new NativeCall.CallContext(1, TestingCommons.aggregatedUpstream(apiMock),
Selector.empty, quorum,
new NativeCall.ParsedCallDetails("eth_sendRawTransaction", ["0x1234"]))
when:
def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
def act = objectMapper.readValue(resp.payload, Object)
then:
act == "0x4b66b555df9faed6f0711f2104d183736c8e2dc7434626dd2622e243f041d41b"
1 * quorum.record(_ as byte[], _)
2 * quorum.record(_ as RpcException, _)
}
}

View File

@@ -68,9 +68,9 @@ class FilteredApisSpec extends Specification {
iter.request(10)
then:
StepVerifier.create(iter)
.expectNext(upstreams[0].api)
.expectNext(upstreams[2].api)
.expectNext(upstreams[3].api)
.expectNext(upstreams[0])
.expectNext(upstreams[2])
.expectNext(upstreams[3])
.expectComplete()
.verify(Duration.ofSeconds(1))
@@ -79,9 +79,9 @@ class FilteredApisSpec extends Specification {
iter.request(10)
then:
StepVerifier.create(iter)
.expectNext(upstreams[2].api)
.expectNext(upstreams[3].api)
.expectNext(upstreams[0].api)
.expectNext(upstreams[2])
.expectNext(upstreams[3])
.expectNext(upstreams[0])
.expectComplete()
.verify(Duration.ofSeconds(1))
@@ -90,12 +90,12 @@ class FilteredApisSpec extends Specification {
iter.request(10)
then:
StepVerifier.create(iter)
.expectNext(upstreams[2].api)
.expectNext(upstreams[3].api)
.expectNext(upstreams[0].api)
.expectNext(upstreams[2].api)
.expectNext(upstreams[3].api)
.expectNext(upstreams[0].api)
.expectNext(upstreams[2])
.expectNext(upstreams[3])
.expectNext(upstreams[0])
.expectNext(upstreams[2])
.expectNext(upstreams[3])
.expectNext(upstreams[0])
.expectComplete()
.verify(Duration.ofSeconds(1))
}
@@ -154,13 +154,13 @@ class FilteredApisSpec extends Specification {
apis.request(10)
return apis
})
.expectNext(api1, api2).as("Batch 1")
.expectNext(up1, up2).as("Batch 1")
.expectNoEvent(Duration.ofMillis(100)).as("Wait 1")
.expectNext(api1, api2).as("Batch 2")
.expectNoEvent(Duration.ofMillis(400)).as("Wait 2")
.expectNext(api1, api2).as("Batch 3")
.expectNoEvent(Duration.ofMillis(900)).as("Wait 3")
.expectNext(api1, api2).as("Batch 4")
.expectNext(up1, up2).as("Batch 2")
.expectNoEvent(Duration.ofMillis(400)).as("Wait 2")
.expectNext(up1, up2).as("Batch 3")
.expectNoEvent(Duration.ofMillis(900)).as("Wait 3")
.expectNext(up1, up2).as("Batch 4")
.expectComplete()
.verify(Duration.ofSeconds(10))
}
@@ -178,7 +178,7 @@ class FilteredApisSpec extends Specification {
act.request(10)
then:
StepVerifier.create(act)
.expectNext(apis[2], apis[3], apis[4], apis[5], apis[0], apis[1])
.expectNext(ups[2], ups[3], ups[4], ups[5], ups[0], ups[1])
.expectComplete()
.verify(Duration.ofSeconds(1))
}