solution: use quorum for native calls

src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy

@@ -0,0 +1,195 @@
+package io.emeraldpay.dshackle.rpc
+
+import io.emeraldpay.api.proto.BlockchainOuterClass
+import io.emeraldpay.api.proto.Common
+import io.emeraldpay.dshackle.test.EthereumApiMock
+import io.emeraldpay.dshackle.test.TestingCommons
+import io.emeraldpay.dshackle.upstream.AlwaysQuorum
+import io.emeraldpay.dshackle.upstream.CallQuorum
+import io.emeraldpay.dshackle.upstream.EthereumApi
+import io.emeraldpay.dshackle.upstream.NonEmptyQuorum
+import io.emeraldpay.dshackle.upstream.Upstream
+import io.emeraldpay.dshackle.upstream.Upstreams
+import io.emeraldpay.grpc.Chain
+import io.infinitape.etherjar.rpc.RpcClient
+import reactor.test.StepVerifier
+import reactor.util.function.Tuples
+import spock.lang.Specification
+
+import java.time.Duration
+
+class NativeCallSpec extends Specification {
+
+    def objectMapper = TestingCommons.objectMapper()
+
+    def "Quorum is applied"() {
+        setup:
+        def quorum = Spy(new AlwaysQuorum())
+        def upstreams = Stub(Upstreams)
+        RpcClient rpcClient = Stub(RpcClient)
+        def upstream = Stub(Upstream)
+        def apiMock = TestingCommons.api(rpcClient, upstream)
+
+        apiMock.answer("eth_test", [], "foo")
+
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+        def call = new NativeCall.CallContext(1, [apiMock].multiply(5).iterator(), quorum, Tuples.of("eth_test", []))
+
+
+        when:
+        def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
+        def act = objectMapper.readValue(resp.payload, Map)
+        then:
+        act == [jsonrpc:"2.0", id:1, result: "foo"]
+        (2..3) * quorum.isResolved() // 2 times during api call (before and after) + 1 time in a filter after
+        1 * quorum.record(_, _)
+        1 * quorum.getResult()
+    }
+
+    def "Quorum may return not first received value"() {
+        setup:
+        def quorum = Spy(new NonEmptyQuorum(TestingCommons.rpcConverter(), 3))
+
+        def upstreams = Stub(Upstreams)
+        RpcClient rpcClient = Stub(RpcClient)
+        def upstream = Stub(Upstream)
+        def apiMock = TestingCommons.api(rpcClient, upstream)
+
+        apiMock.answerOnce("eth_test", [], null)
+        apiMock.answerOnce("eth_test", [], "bar")
+        apiMock.answerOnce("eth_test", [], null)
+
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+        def call = new NativeCall.CallContext(1, [apiMock].multiply(5).iterator(), quorum, Tuples.of("eth_test", []))
+
+
+        when:
+        def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(2))
+        def act = objectMapper.readValue(resp.payload, Map)
+        then:
+        act == [jsonrpc:"2.0", id:1, result: "bar"]
+        (3..4) * quorum.isResolved()
+        2 * quorum.record(_, _)
+        1 * quorum.getResult()
+    }
+
+    def "Returns error if no quorum"() {
+        setup:
+        def quorum = Spy(new NonEmptyQuorum(TestingCommons.rpcConverter(), 3))
+
+        def upstreams = Stub(Upstreams)
+        RpcClient rpcClient = Stub(RpcClient)
+        def upstream = Stub(Upstream)
+        def apiMock = TestingCommons.api(rpcClient, upstream)
+
+        apiMock.answer("eth_test", [], null, 3)
+        apiMock.answerOnce("eth_test", [], "foo")
+
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+        def call = new NativeCall.CallContext(1, [apiMock].multiply(5).iterator(), quorum, Tuples.of("eth_test", []))
+
+        (4..5) * quorum.isResolved()
+        3 * quorum.record(_, _)
+        1 * quorum.getResult()
+
+        when:
+        def resp = nativeCall.executeOnRemote(call)
+        then:
+        StepVerifier.create(resp)
+            .expectErrorMatches({t -> t instanceof NativeCall.CallFailure && t.id == 1})
+            .verify(Duration.ofSeconds(1))
+    }
+
+    def "Packs call exception into response with id"() {
+        setup:
+        def upstreams = Stub(Upstreams)
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+        when:
+        def resp = nativeCall.processException(new NativeCall.CallFailure(5, new IllegalArgumentException("test test")))
+        then:
+        StepVerifier.create(resp)
+                .expectNext(BlockchainOuterClass.NativeCallReplyItem.newBuilder()
+                        .setSucceed(false)
+                        .setId(5)
+                        .build())
+                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+
+    def "Packs unknown exception into response"() {
+        setup:
+        def upstreams = Stub(Upstreams)
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+        when:
+        def resp = nativeCall.processException(new IllegalArgumentException("test test"))
+        then:
+        StepVerifier.create(resp)
+                .expectNext(BlockchainOuterClass.NativeCallReplyItem.newBuilder()
+                        .setSucceed(false)
+                        .build())
+                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+
+    def "Builds normal response"() {
+        setup:
+        def upstreams = Stub(Upstreams)
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+        def json = [jsonrpc:"2.0", id:1, result: "foo"]
+        when:
+        def resp = nativeCall.buildResponse(
+                new NativeCall.CallContext<byte[]>(1561, [].iterator(), new AlwaysQuorum(), objectMapper.writeValueAsBytes(json))
+        )
+        then:
+        resp.id == 1561
+        resp.succeed
+        objectMapper.readValue(resp.payload.toByteArray(), Map.class) == [jsonrpc:"2.0", id:1, result: "foo"]
+    }
+
+    def "Returns error for invalid chain"() {
+        setup:
+        def upstreams = Stub(Upstreams)
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+
+        def req = BlockchainOuterClass.NativeCallRequest.newBuilder()
+            .setChainValue(0)
+            .addAllItems([1, 2].collect { id ->
+                return BlockchainOuterClass.NativeCallItem.newBuilder()
+                .setId(id)
+                .setTarget("eth_test")
+                .build()
+            })
+            .build()
+        when:
+        def resp = nativeCall.prepareCall(req)
+        then:
+        StepVerifier.create(resp)
+                .expectErrorMatches({t -> t instanceof NativeCall.CallFailure && t.id == 0})
+//                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+
+    def "Returns error for unsupported chain"() {
+        setup:
+        def upstreams =  Mock(Upstreams)
+        def nativeCall = new NativeCall(upstreams, TestingCommons.objectMapper())
+
+        def req = BlockchainOuterClass.NativeCallRequest.newBuilder()
+                .setChainValue(Chain.TESTNET_MORDEN.id)
+                .addAllItems([1, 2].collect { id ->
+                    return BlockchainOuterClass.NativeCallItem.newBuilder()
+                            .setId(id)
+                            .setTarget("eth_test")
+                            .build()
+                })
+                .build()
+        1 * upstreams.getUpstream(Chain.TESTNET_MORDEN) >> null
+        when:
+        def resp = nativeCall.prepareCall(req)
+        then:
+        StepVerifier.create(resp)
+                .expectErrorMatches({t -> t instanceof NativeCall.CallFailure && t.id == 0})
+//                .expectComplete()
+                .verify(Duration.ofSeconds(1))
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/rpc/TrackAddressSpec.groovy

@@ -37,7 +37,7 @@ class TrackAddressSpec extends Specification {
 
 
     def setup() {
-        availableChains = new AvailableChains()
+        availableChains = new AvailableChains(TestingCommons.objectMapper())
         upstreams = Mock(Upstreams)
         trackAddress = new TrackAddress(upstreams, availableChains, Schedulers.immediate())
     }
@@ -61,7 +61,7 @@ class TrackAddressSpec extends Specification {
             .build()
 
         def upstreamMock = Mock(AggregatedUpstreams)
-        def apiMock = new EthereumApiMock(Mock(RpcClient), TestingCommons.objectMapper(), Chain.ETHEREUM)
+        def apiMock = TestingCommons.api(Stub(RpcClient), upstreamMock)
         apiMock.answer("eth_getBalance", ["0xe2c8fa8120d813cd0b5e6add120295bf20cfa09f", "latest"], "0x499602D2")
         _ * upstreams.getUpstream(Chain.ETHEREUM) >> upstreamMock
         _ * upstreamMock.getApi(_) >> apiMock
@@ -103,7 +103,7 @@ class TrackAddressSpec extends Specification {
         def blocksBus = TopicProcessor.create()
         def upstreamMock = Mock(AggregatedUpstreams)
         def headMock = Mock(EthereumHead)
-        def apiMock = new EthereumApiMock(Mock(RpcClient), TestingCommons.objectMapper(), Chain.ETHEREUM)
+        def apiMock = TestingCommons.api(Stub(RpcClient), upstreamMock)
         apiMock.answerOnce("eth_getBalance", ["0xe2c8fa8120d813cd0b5e6add120295bf20cfa09f", "latest"], "0x499602D2")
         apiMock.answerOnce("eth_getBalance", ["0xe2c8fa8120d813cd0b5e6add120295bf20cfa09f", "latest"], "0xff98")
         _ * upstreams.getUpstream(Chain.ETHEREUM) >> upstreamMock

src/test/groovy/io/emeraldpay/dshackle/rpc/TrackTxSpec.groovy

@@ -25,7 +25,7 @@ import java.time.Duration
 
 class TrackTxSpec extends Specification {
 
-    AvailableChains availableChains = new AvailableChains()
+    AvailableChains availableChains = new AvailableChains(TestingCommons.objectMapper())
     Upstreams upstreams
     TrackTx trackTx
 
@@ -94,7 +94,7 @@ class TrackTxSpec extends Specification {
         def blocksBus = TopicProcessor.create()
         def headMock = Mock(EthereumHead)
 
-        def apiMock = new EthereumApiMock(Mock(RpcClient), TestingCommons.objectMapper(), Chain.ETHEREUM)
+        def apiMock = TestingCommons.api(Stub(RpcClient), upstreamMock)
         apiMock.answer("eth_getTransactionByHash", [txId], txJson)
         apiMock.answer("eth_getBlockByHash", [blockJson.hash.toHex(), false], blockJson)
 
@@ -171,7 +171,7 @@ class TrackTxSpec extends Specification {
         def blocksBus = TopicProcessor.create()
         def headMock = Mock(EthereumHead)
 
-        def apiMock = new EthereumApiMock(Mock(RpcClient), TestingCommons.objectMapper(), Chain.ETHEREUM)
+        def apiMock = TestingCommons.api(Stub(RpcClient), upstreamMock)
         apiMock.answerOnce("eth_getTransactionByHash", [txId], null)
         apiMock.answerOnce("eth_getTransactionByHash", [txId], txJsonBroadcasted)
         apiMock.answer("eth_getTransactionByHash", [txId], txJsonMined)

src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiMock.groovy

@@ -4,6 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.google.protobuf.ByteString
 import io.emeraldpay.api.proto.BlockchainOuterClass
 import io.emeraldpay.dshackle.upstream.EthereumApi
+import io.emeraldpay.dshackle.upstream.EthereumTargets
+import io.emeraldpay.dshackle.upstream.Upstream
+import io.emeraldpay.dshackle.upstream.Upstreams
 import io.emeraldpay.grpc.Chain
 import io.grpc.stub.StreamObserver
 import io.infinitape.etherjar.rpc.RpcClient
@@ -20,8 +23,8 @@ class EthereumApiMock extends EthereumApi {
     List<PredefinedResponse> predefined = []
     private ObjectMapper objectMapper
 
-    EthereumApiMock(@NotNull RpcClient rpcClient, @NotNull ObjectMapper objectMapper, @NotNull Chain chain) {
-        super(rpcClient, objectMapper, chain)
+    EthereumApiMock(@NotNull RpcClient rpcClient, @NotNull ObjectMapper objectMapper, @NotNull Chain chain, Upstream upstream) {
+        super(rpcClient, objectMapper, chain, new EthereumTargets(objectMapper, chain), upstream)
         this.objectMapper = objectMapper
     }
 

src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy

@@ -4,8 +4,18 @@ import com.fasterxml.jackson.core.Version
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.module.SimpleModule
+import io.emeraldpay.dshackle.upstream.Upstream
+import io.emeraldpay.grpc.Chain
+import io.infinitape.etherjar.rpc.Batch
+import io.infinitape.etherjar.rpc.ExecutableBatch
+import io.infinitape.etherjar.rpc.JacksonRpcConverter
+import io.infinitape.etherjar.rpc.RpcCall
+import io.infinitape.etherjar.rpc.RpcClient
+import io.infinitape.etherjar.rpc.transport.BatchStatus
+import spock.mock.MockingApi
 
 import java.text.SimpleDateFormat
+import java.util.concurrent.CompletableFuture
 
 class TestingCommons {
 
@@ -21,4 +31,12 @@ class TestingCommons {
 
         return objectMapper
     }
+
+    static EthereumApiMock api(RpcClient rpcClient, Upstream upstream) {
+        return new EthereumApiMock(rpcClient, objectMapper(), Chain.ETHEREUM, upstream)
+    }
+
+    static JacksonRpcConverter rpcConverter() {
+        return new JacksonRpcConverter(objectMapper())
+    }
 }

src/test/groovy/io/emeraldpay/dshackle/upstream/BroadcastQuorumSpec.groovy

@@ -0,0 +1,78 @@
+package io.emeraldpay.dshackle.upstream
+
+import io.emeraldpay.dshackle.test.TestingCommons
+import io.infinitape.etherjar.domain.TransactionId
+import io.infinitape.etherjar.rpc.json.BlockJson
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Mono
+import spock.lang.Specification
+
+class BroadcastQuorumSpec extends Specification {
+
+    def rpcConverted = TestingCommons.rpcConverter()
+    def objectMapper = TestingCommons.objectMapper()
+
+    def "Resolved with first after 3 tries"() {
+        setup:
+        def q = Spy(new BroadcastQuorum(rpcConverted, 3))
+        def upstream1 = Stub(Upstream)
+        def upstream2 = Stub(Upstream)
+        def upstream3 = Stub(Upstream)
+
+        when:
+        q.init(Stub(Head))
+        then:
+        !q.isResolved()
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c"]), upstream1)
+        then:
+        !q.isResolved()
+        1 * q.recordValue(_, "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c", _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c"]), upstream2)
+        then:
+        !q.isResolved()
+        1 * q.recordValue(_, "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c", _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([error: [message: "Nonce too low"]]), upstream3)
+        then:
+        1 * q.recordError(_, _, _)
+        q.isResolved()
+        objectMapper.readValue(q.result, Map) == [result: "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c"]
+    }
+
+    def "Remembers first response"() {
+        setup:
+        def q = Spy(new BroadcastQuorum(rpcConverted, 3))
+        def upstream1 = Stub(Upstream)
+        def upstream2 = Stub(Upstream)
+        def upstream3 = Stub(Upstream)
+
+        when:
+        q.init(Stub(Head))
+        then:
+        !q.isResolved()
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([error: [message: "Internal error"]]), upstream1)
+        then:
+        !q.isResolved()
+        1 * q.recordError(_, _, _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c"]), upstream2)
+        then:
+        !q.isResolved()
+        1 * q.recordValue(_, "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c", _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([error: [message: "Nonce too low"]]), upstream3)
+        then:
+        1 * q.recordError(_, _, _)
+        q.isResolved()
+        objectMapper.readValue(q.result, Map) == [result: "0xeaa972c0d8d1ecd3e34fbbef6d34e06670e745c788bdba31c4234a1762f0378c"]
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/upstream/EthereumGrpcTransportSpec.groovy

@@ -19,6 +19,7 @@ class EthereumGrpcTransportSpec extends Specification {
 
     MockServer mockServer = new MockServer()
     ObjectMapper objectMapper = TestingCommons.objectMapper()
+    def ethereumTargets = new EthereumTargets(objectMapper, Chain.ETHEREUM)
 
     def "Make simple call"() {
         setup:
@@ -26,7 +27,7 @@ class EthereumGrpcTransportSpec extends Specification {
         def otherSideUpstreams = Mock(Upstreams)
         def otherSideAggr = Mock(AggregatedUpstreams)
         def otherSideNativeCall = new NativeCall(otherSideUpstreams, objectMapper)
-        def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM)
+        def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM, otherSideAggr)
 
         def client = mockServer.clientForServer(new ReactorBlockchainGrpc.BlockchainImplBase() {
             @Override
@@ -45,7 +46,9 @@ class EthereumGrpcTransportSpec extends Specification {
 
         then:
         1 * otherSideUpstreams.getUpstream(Chain.ETHEREUM) >> otherSideAggr
-        1 * otherSideAggr.getApi(_) >> otherSideApi
+        1 * otherSideAggr.getApis(_) >> [otherSideApi].iterator()
+        _ * otherSideAggr.getHead() >> Stub(EthereumHead)
+        _ * otherSideAggr.getTargets() >> ethereumTargets
         status.failed == 0
         status.succeed == 1
         status.total == 1
@@ -67,7 +70,7 @@ class EthereumGrpcTransportSpec extends Specification {
         def otherSideUpstreams = Mock(Upstreams)
         def otherSideAggr = Mock(AggregatedUpstreams)
         def otherSideNativeCall = new NativeCall(otherSideUpstreams, objectMapper)
-        def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM)
+        def otherSideApi = new EthereumApiMock(Mock(RpcClient), objectMapper, Chain.ETHEREUM, otherSideAggr)
 
         def client = mockServer.clientForServer(new ReactorBlockchainGrpc.BlockchainImplBase() {
             @Override
@@ -90,7 +93,9 @@ class EthereumGrpcTransportSpec extends Specification {
 
         then:
         1 * otherSideUpstreams.getUpstream(Chain.ETHEREUM) >> otherSideAggr
-        1 * otherSideAggr.getApi(_) >> otherSideApi
+        1 * otherSideAggr.getApis(_) >> [otherSideApi].multiply(3).iterator()
+        _ * otherSideAggr.getHead() >> Stub(EthereumHead)
+        _ * otherSideAggr.getTargets() >> ethereumTargets
         status.failed == 0
         status.succeed == 2
         status.total == 2

src/test/groovy/io/emeraldpay/dshackle/upstream/FilteringApiIteratorSpec.groovy

@@ -10,52 +10,32 @@ class FilteringApiIteratorSpec extends Specification {
 
     def rpcClient = new DefaultRpcClient(null)
     def objectMapper = TestingCommons.objectMapper()
+    def ethereumTargets = new EthereumTargets(objectMapper, Chain.ETHEREUM)
 
     def "Verifies labels"() {
         setup:
-        def upstreams = [
-               new EthereumUpstream(
-                       Chain.ETHEREUM,
-                       new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM),
-                       (EthereumWs)null,
-                       new UpstreamsConfig.Options(),
-                       new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap([test: "foo"]))
-               ),
-               new EthereumUpstream(
-                       Chain.ETHEREUM,
-                       new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM),
-                       (EthereumWs)null,
-                       new UpstreamsConfig.Options(),
-                       new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap([test: "bar"]))
-               ),
-               new EthereumUpstream(
-                       Chain.ETHEREUM,
-                       new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM),
-                       (EthereumWs)null,
-                       new UpstreamsConfig.Options(),
-                       new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap([test: "foo", test2: "baz"]))
-               ),
-               new EthereumUpstream(
-                       Chain.ETHEREUM,
-                       new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM),
-                       (EthereumWs)null,
-                       new UpstreamsConfig.Options(),
-                       new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap([test: "foo"]))
-               ),
-               new EthereumUpstream(
-                       Chain.ETHEREUM,
-                       new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM),
-                       (EthereumWs)null,
-                       new UpstreamsConfig.Options(),
-                       new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap([test: "baz"]))
-               )
-        ]
+        List<EthereumUpstream> upstreams = [
+                [test: "foo"],
+                [test: "bar"],
+                [test: "foo", test2: "baz"],
+                [test: "foo"],
+                [test: "baz"]
+        ].collect {
+            new EthereumUpstream(
+                    Chain.ETHEREUM,
+                    new EthereumApi(rpcClient, objectMapper, Chain.ETHEREUM, ethereumTargets, null),
+                    (EthereumWs) null,
+                    new UpstreamsConfig.Options(),
+                    new NodeDetailsList.NodeDetails(1, UpstreamsConfig.Labels.fromMap(it)),
+                    ethereumTargets
+            )
+        }
         def matcher = new Selector.LabelMatcher("test", ["foo"])
         upstreams.forEach {
             it.setStatus(UpstreamAvailability.OK)
         }
         when:
-        def iter = new FilteringApiIterator(upstreams, 3, 0, matcher)
+        def iter = new FilteringApiIterator(upstreams, 0, matcher, 1)
         then:
         iter.hasNext()
         iter.next() == upstreams[0].api
@@ -66,16 +46,18 @@ class FilteringApiIteratorSpec extends Specification {
         !iter.hasNext()
 
         when:
-        iter = new FilteringApiIterator(upstreams, 2, 1, matcher)
+        iter = new FilteringApiIterator(upstreams, 1, matcher, 1)
         then:
         iter.hasNext()
         iter.next() == upstreams[2].api
         iter.hasNext()
         iter.next() == upstreams[3].api
+        iter.hasNext()
+        iter.next() == upstreams[0].api
         !iter.hasNext()
 
         when:
-        iter = new FilteringApiIterator(upstreams, 3, 1, matcher)
+        iter = new FilteringApiIterator(upstreams, 1,  matcher, 2)
         then:
         iter.hasNext()
         iter.next() == upstreams[2].api
@@ -83,6 +65,12 @@ class FilteringApiIteratorSpec extends Specification {
         iter.next() == upstreams[3].api
         iter.hasNext()
         iter.next() == upstreams[0].api
+        iter.hasNext()
+        iter.next() == upstreams[2].api
+        iter.hasNext()
+        iter.next() == upstreams[3].api
+        iter.hasNext()
+        iter.next() == upstreams[0].api
         !iter.hasNext()
     }
 }

src/test/groovy/io/emeraldpay/dshackle/upstream/GrpcUpstreamSpec.groovy

@@ -23,12 +23,13 @@ class GrpcUpstreamSpec extends Specification {
 
     MockServer mockServer = new MockServer()
     ObjectMapper objectMapper = TestingCommons.objectMapper()
+    def ethereumTargets = new EthereumTargets(objectMapper, Chain.ETHEREUM)
 
     def "Subscribe to head"() {
         setup:
         def callData = [:]
         def chain = Chain.ETHEREUM
-        def api = new EthereumApiMock(Mock(RpcClient), objectMapper, chain)
+        def api = TestingCommons.api(Stub(RpcClient), Stub(Upstream))
         def block1 = new BlockJson().with {
             it.number = 650246
             it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
@@ -54,7 +55,7 @@ class GrpcUpstreamSpec extends Specification {
                 )
             }
         })
-        def upstream = new GrpcUpstream(chain, client, objectMapper)
+        def upstream = new GrpcUpstream(chain, client, objectMapper, ethereumTargets)
         when:
         upstream.connect()
         def h = upstream.head.head.block(Duration.ofSeconds(1))
@@ -69,7 +70,7 @@ class GrpcUpstreamSpec extends Specification {
         def callData = [:]
         def finished = new CompletableFuture<Boolean>()
         def chain = Chain.ETHEREUM
-        def api = new EthereumApiMock(Mock(RpcClient), objectMapper, chain)
+        def api = TestingCommons.api(Stub(RpcClient), Stub(Upstream))
         def block1 = new BlockJson().with {
             it.number = 650246
             it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
@@ -109,7 +110,7 @@ class GrpcUpstreamSpec extends Specification {
                 finished.complete(true)
             }
         })
-        def upstream = new GrpcUpstream(chain, client, objectMapper)
+        def upstream = new GrpcUpstream(chain, client, objectMapper, ethereumTargets)
         when:
         upstream.connect()
         finished.get()
@@ -125,7 +126,7 @@ class GrpcUpstreamSpec extends Specification {
         def callData = [:]
         def finished = new CompletableFuture<Boolean>()
         def chain = Chain.ETHEREUM
-        def api = new EthereumApiMock(Mock(RpcClient), objectMapper, chain)
+        def api = TestingCommons.api(Stub(RpcClient), Stub(Upstream))
         def block1 = new BlockJson().with {
             it.number = 650246
             it.hash = BlockHash.from("0x50d26e119968e791970d84a7bf5d0ec474d3ec2ef85d5ec8915210ac6bc09ad7")
@@ -165,7 +166,7 @@ class GrpcUpstreamSpec extends Specification {
                 finished.complete(true)
             }
         })
-        def upstream = new GrpcUpstream(chain, client, objectMapper)
+        def upstream = new GrpcUpstream(chain, client, objectMapper, ethereumTargets)
         when:
         upstream.connect()
         finished.get()

src/test/groovy/io/emeraldpay/dshackle/upstream/NonceQuorumSpec.groovy

@@ -0,0 +1,80 @@
+package io.emeraldpay.dshackle.upstream
+
+import io.emeraldpay.dshackle.test.TestingCommons
+import spock.lang.Specification
+
+class NonceQuorumSpec extends Specification {
+
+    def rpcConverted = TestingCommons.rpcConverter()
+    def objectMapper = TestingCommons.objectMapper()
+
+    def "Gets max value"() {
+        setup:
+        def q = Spy(new NonceQuorum(rpcConverted, 3))
+        def upstream1 = Stub(Upstream)
+        def upstream2 = Stub(Upstream)
+        def upstream3 = Stub(Upstream)
+
+        when:
+        q.init(Stub(Head))
+        then:
+        !q.isResolved()
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0x10"]), upstream1)
+        then:
+        !q.isResolved()
+        1 * q.recordValue(_, "0x10", _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0x11"]), upstream2)
+        then:
+        !q.isResolved()
+        1 * q.recordValue(_, "0x11", _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0x10"]), upstream3)
+        then:
+        1 * q.recordValue(_, "0x10", _)
+        q.isResolved()
+        objectMapper.readValue(q.result, Map) == [result: "0x11"]
+    }
+
+    def "Ignores errors"() {
+        setup:
+        def q = Spy(new NonceQuorum(rpcConverted, 3))
+        def upstream1 = Stub(Upstream)
+        def upstream2 = Stub(Upstream)
+        def upstream3 = Stub(Upstream)
+
+        when:
+        q.init(Stub(Head))
+        then:
+        !q.isResolved()
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([error: [error: "Internal"]]), upstream1)
+        then:
+        !q.isResolved()
+        1 * q.recordError(_, _, _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0x11"]), upstream2)
+        then:
+        !q.isResolved()
+        1 * q.recordValue(_, "0x11", _)
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0x10"]), upstream3)
+        then:
+        1 * q.recordValue(_, "0x10", _)
+        !q.isResolved()
+
+        when:
+        q.record(objectMapper.writeValueAsBytes([result: "0x11"]), upstream1)
+        then:
+        1 * q.recordValue(_, "0x11", _)
+        q.isResolved()
+        objectMapper.readValue(q.result, Map) == [result: "0x11"]
+    }
+}