problem: no response from proxy if upstream returned error

fix: #35

src/main/kotlin/io/emeraldpay/dshackle/proxy/WriteRpcJson.kt

@@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.proxy
 import com.fasterxml.jackson.databind.ObjectMapper
 import io.emeraldpay.api.proto.BlockchainOuterClass
 import io.emeraldpay.dshackle.Global
+import io.emeraldpay.dshackle.rpc.NativeCall
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Service
@@ -43,20 +44,29 @@ open class WriteRpcJson() {
      */
     open fun toJsons(call: ProxyCall): Function<Flux<BlockchainOuterClass.NativeCallReplyItem>, Flux<String>> {
         return Function { flux ->
-            flux.flatMap { response ->
+            flux
-                if (!call.ids.containsKey(response.id)) {
+                    .flatMap { response ->
-                    log.warn("ID wasn't requested: ${response.id}")
+                        if (!call.ids.containsKey(response.id)) {
-                    return@flatMap Flux.empty<String>()
+                            log.warn("ID wasn't requested: ${response.id}")
-                }
+                            return@flatMap Flux.empty<String>()
-                val json = toJson(call, response)
+                        }
-                if (json == null) {
+                        val json = toJson(call, response)
-                    Flux.empty<String>()
+                        if (json == null) {
-                } else {
+                            Flux.empty<String>()
-                    Flux.just(json)
+                        } else {
-                }
+                            Flux.just(json)
-            }.onErrorContinue { t, u ->
+                        }
-                log.warn("Failed to convert to JSON", t)
+                    }
-            }
+                    .onErrorResume { t ->
+                        if (t is NativeCall.CallFailure) {
+                            Mono.just(toJson(call, t)!!)
+                        } else {
+                            Mono.empty()
+                        }
+                    }
+                    .onErrorContinue { t, _ ->
+                        log.warn("Failed to convert to JSON", t)
+                    }
         }
     }
 
@@ -70,6 +80,12 @@ open class WriteRpcJson() {
         return objectMapper.writeValueAsString(json)
     }
 
+    fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? {
+        val id = call.ids[error.id] ?: return null;
+        val json = JsonRpcResponse.error(-32002, error.reason.message ?: "", JsonRpcResponse.Id.from(id))
+        return objectMapper.writeValueAsString(json)
+    }
+
     /**
      * Format response as JSON Array, for Batch requests
      */

src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt

@@ -24,16 +24,17 @@ import java.util.concurrent.atomic.AtomicReference
 class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
 
     private val result: AtomicReference<ByteArray> = AtomicReference()
+    private val failed = AtomicReference(false)
 
     override fun init(head: Head) {
     }
 
     override fun isResolved(): Boolean {
-        return result.get() != null
+        return !isFailed() && result.get() != null
     }
 
     override fun isFailed(): Boolean {
-        return false
+        return failed.get()
     }
 
     override fun record(response: ByteArray, upstream: Upstream): Boolean {
@@ -46,6 +47,10 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum {
     }
 
     override fun record(error: RpcException, upstream: Upstream) {
+        val lagging = upstream.getLag() > maxLag
+        if (!lagging && result.get() == null) {
+            failed.set(true)
+        }
     }
 
     override fun getResult(): ByteArray {

src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt

@@ -54,15 +54,40 @@ class QuorumRpcReader(
             }
         }
 
+        val defaultResult: Mono<Result> = Mono.just(quorum).flatMap { q ->
+            if (q.isFailed()) {
+                //TODO record and return actual error details
+                Mono.error<Result>(RpcException(-32000, "Upstream error"))
+            } else {
+                log.warn("Empty result for ${key.method} as ${q}")
+                Mono.empty<Result>()
+            }
+        }
+
         return Flux.from(apis)
+                .takeUntil {
+                    quorum.isFailed() || quorum.isResolved()
+                }
                 .flatMap { api ->
-                    api.getApi().read(key)
+                    api.getApi()
-                            .flatMap(JsonRpcResponse::requireResult)
+                            .read(key)
-                            // on error notify quorum, it may use error message or other details
+                            .flatMap { response ->
-                            .doOnError { err ->
+                                response.requireResult()
-                                if (err is RpcException) {
+                                        .onErrorResume { err ->
-                                    quorum.record(err, api)
+                                            if (err is RpcException) {
-                                }
+                                                // on error notify quorum, it may use error message or other details
+                                                quorum.record(err, api)
+                                                // it it's failed after that, then we don't need more calls, stop api source
+                                                if (quorum.isFailed()) {
+                                                    apis.resolve()
+                                                } else {
+                                                    apis.request(1)
+                                                }
+                                            } else {
+                                                log.warn("Result processing error", err)
+                                            }
+                                            Mono.empty()
+                                        }
                             }
                             .map { Tuples.of(it, api) }
                 }
@@ -95,6 +120,7 @@ class QuorumRpcReader(
                     // TODO find actual quorum number
                     QuorumRpcReader.Result(it.getResult()!!, 1)
                 }
+                .switchIfEmpty(defaultResult)
     }
 
 

src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt

@@ -51,10 +51,12 @@ open class NativeCall(
         return requestMono.flatMapMany(this::prepareCall)
                 .map(this::parseParams)
                 .parallel()
-                .flatMap(this::fetch)
+                .flatMap {
+                    this.fetch(it)
+                            .doOnError { e -> log.warn("Error during native call: ${e.message}") }
+                }
                 .sequential()
                 .map(this::buildResponse)
-                .doOnError { e -> log.warn("Error during native call: ${e.message}") }
                 .onErrorResume(this::processException)
     }
 
@@ -63,17 +65,24 @@ open class NativeCall(
         return it.withPayload(ParsedCallDetails(it.payload.method, params))
     }
 
-    fun buildResponse(it: CallContext<ByteArray>): BlockchainOuterClass.NativeCallReplyItem {
+    fun buildResponse(it: CallResult): BlockchainOuterClass.NativeCallReplyItem {
-        return BlockchainOuterClass.NativeCallReplyItem.newBuilder()
+        val result = BlockchainOuterClass.NativeCallReplyItem.newBuilder()
-                .setSucceed(true)
+                .setSucceed(!it.isError())
                 .setId(it.id)
-                .setPayload(ByteString.copyFrom(it.payload))
+        if (it.isError()) {
-                .build()
+            it.error?.let { error ->
+                result.setErrorMessage(error.message)
+            }
+        } else {
+            result.setPayload(ByteString.copyFrom(it.result))
+        }
+
+        return result.build()
     }
 
     fun processException(it: Throwable?): Mono<BlockchainOuterClass.NativeCallReplyItem> {
-        val id: Int = if (it != null && CallFailure::class.isInstance(it)) {
+        val id: Int = if (it != null && it is CallFailure) {
-            (it as CallFailure).id
+            it.id
         } else {
             log.error("Lost context for a native call", it)
             0
@@ -119,42 +128,52 @@ open class NativeCall(
         }
     }
 
-    fun fetch(ctx: CallContext<ParsedCallDetails>): Mono<CallContext<ByteArray>> {
+    fun fetch(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
         return ctx.upstream.getRoutedApi(ctx.matcher)
                 .flatMap { api ->
                     api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
                             .flatMap(JsonRpcResponse::requireResult)
                             .map {
-                                ctx.withPayload(it)
+                                CallResult.ok(ctx.id, it)
                             }
                 }.switchIfEmpty(
                         Mono.just(ctx).flatMap(this::executeOnRemote)
                 )
-                .onErrorMap {
+                .onErrorResume {
-                    CallFailure(ctx.id, it)
+                    if (it is CallFailure) {
+                        Mono.just(CallResult.fail(it.id, it.reason))
+                    } else {
+                        Mono.just(CallResult.fail(ctx.id, it))
+                    }
                 }
     }
 
-    fun executeOnRemote(ctx: CallContext<ParsedCallDetails>): Mono<CallContext<ByteArray>> {
+    fun executeOnRemote(ctx: CallContext<ParsedCallDetails>): Mono<CallResult> {
         if (!ctx.upstream.getMethods().isAllowed(ctx.payload.method)) {
             return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))
         }
         val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum)
-        return reader.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
+        return reader
+                .read(JsonRpcRequest(ctx.payload.method, ctx.payload.params))
                 .map {
-                    ctx.withPayload(it.value)
+                    CallResult(ctx.id, it.value, null)
                 }
                 .doOnNext {
-                    ctx.upstream.postprocessor
+                    it.result?.let { value ->
-                            .onReceive(ctx.payload.method, ctx.payload.params, it.payload)
+                        ctx.upstream.postprocessor
+                                .onReceive(ctx.payload.method, ctx.payload.params, value)
+                    }
                 }
-                .onErrorMap {
+                .onErrorResume { t ->
-                    log.error("Failed to make a call", it)
+                    val failure = if (t is CallFailure) {
-                    if (it is CallFailure) it
+                        CallResult.fail(t.id, t.reason)
-                    else CallFailure(ctx.id, it)
+                    } else {
+                        CallResult.fail(ctx.id, t)
+                    }
+                    Mono.just(failure)
                 }
                 .switchIfEmpty(
-                        Mono.error(CallFailure(ctx.id, Exception("No response or no available upstream for ${ctx.payload.method}")) as Throwable)
+                        Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}"))
                 )
     }
 
@@ -180,7 +199,39 @@ open class NativeCall(
         }
     }
 
-    open class CallFailure(val id: Int, val reason: Throwable): Exception("Failed to call $id: ${reason.message}")
+    open class CallFailure(val id: Int, val reason: Throwable) : Exception("Failed to call $id: ${reason.message}")
+
+    open class CallError(val id: Int, val message: String) {
+        companion object {
+            fun from(t: Throwable): CallError {
+                return when (t) {
+                    is RpcException -> CallError(t.code, t.rpcMessage)
+                    is CallFailure -> CallError(t.id, t.reason.message ?: "Upstream Error")
+                    else -> CallError(1, t.message ?: "Upstream Error")
+                }
+            }
+        }
+    }
+
+    open class CallResult(val id: Int, val result: ByteArray?, val error: CallError?) {
+        companion object {
+            fun ok(id: Int, result: ByteArray): CallResult {
+                return CallResult(id, result, null)
+            }
+
+            fun fail(id: Int, errorCore: Int, errorMessage: String): CallResult {
+                return CallResult(id, null, CallError(errorCore, errorMessage))
+            }
+
+            fun fail(id: Int, error: Throwable): CallResult {
+                return CallResult(id, null, CallError.from(error))
+            }
+        }
+
+        fun isError(): Boolean {
+            return error != null
+        }
+    }
 
     class RawCallDetails(val method: String, val params: String)
     class ParsedCallDetails(val method: String, val params: List<Any>)

src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcResponse.kt

@@ -136,6 +136,7 @@ class JsonRpcResponse(
         fun isInt(): Boolean
 
         companion object {
+            @JvmStatic
             fun from(id: Any): Id {
                 if (id is Int) {
                     return IntId(id)

src/test/groovy/io/emeraldpay/dshackle/quorum/NotLaggingQuorumSpec.groovy

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.quorum
 
 import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
+import io.infinitape.etherjar.rpc.RpcException
 import spock.lang.Specification
 
 class NotLaggingQuorumSpec extends Specification {
@@ -33,6 +34,7 @@ class NotLaggingQuorumSpec extends Specification {
         then:
         1 * up.getLag() >> 0
         quorum.isResolved()
+        !quorum.isFailed()
         quorum.result == value
     }
 
@@ -47,6 +49,7 @@ class NotLaggingQuorumSpec extends Specification {
         then:
         1 * up.getLag() >> 1
         quorum.isResolved()
+        !quorum.isFailed()
         quorum.result == value
     }
 
@@ -61,5 +64,20 @@ class NotLaggingQuorumSpec extends Specification {
         then:
         1 * up.getLag() >> 2
         !quorum.isResolved()
+        !quorum.isFailed()
+    }
+
+    def "Fails if no lag and error response received"() {
+        setup:
+        def up = Mock(Upstream)
+        def value = "foo".getBytes()
+        def quorum = new NotLaggingQuorum(1)
+
+        when:
+        quorum.record(new RpcException(-100, "test error"), up)
+        then:
+        1 * up.getLag() >> 1
+        !quorum.isResolved()
+        quorum.isFailed()
     }
 }

src/test/groovy/io/emeraldpay/dshackle/quorum/QuorumRpcReaderSpec.groovy

@@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.upstream.Selector
 import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
+import io.infinitape.etherjar.rpc.RpcException
 import reactor.core.publisher.Mono
 import reactor.test.StepVerifier
 import spock.lang.Specification
@@ -60,15 +61,16 @@ class QuorumRpcReaderSpec extends Specification {
 
     def "always-quorum - retry upstream error"() {
         setup:
+        def api = Mock(Reader) {
+            2 * read(new JsonRpcRequest("eth_test", [])) >>> [
+                    Mono.just(JsonRpcResponse.error(1, "test")),
+                    Mono.just(JsonRpcResponse.ok("1"))
+            ]
+        }
         def up = Mock(Upstream) {
             _ * isAvailable() >> true
             _ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
-            _ * getApi() >> Mock(Reader) {
+            _ * getApi() >> api
-                2 * read(new JsonRpcRequest("eth_test", [])) >>> [
-                        Mono.just(JsonRpcResponse.error(1, "test")),
-                        Mono.just(JsonRpcResponse.ok("1"))
-                ]
-            }
         }
         def apis = new FilteredApis(
                 [up], Selector.empty
@@ -180,18 +182,19 @@ class QuorumRpcReaderSpec extends Specification {
                 .verify(Duration.ofSeconds(1))
     }
 
-    def "non-empty-quorum - no result if all failed"() {
+    def "non-empty-quorum - error if all failed"() {
         setup:
+        def api = Mock(Reader) {
+            3 * read(new JsonRpcRequest("eth_test", [])) >>> [
+                    Mono.just(JsonRpcResponse.ok("null")),
+                    Mono.just(JsonRpcResponse.error(1, "test")),
+                    Mono.just(JsonRpcResponse.ok("null"))
+            ]
+        }
         def up = Mock(Upstream) {
             _ * isAvailable() >> true
             _ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
-            _ * getApi() >> Mock(Reader) {
+            _ * getApi() >> api
-                3 * read(new JsonRpcRequest("eth_test", [])) >>> [
-                        Mono.just(JsonRpcResponse.ok("null")),
-                        Mono.just(JsonRpcResponse.error(1, "test")),
-                        Mono.just(JsonRpcResponse.ok("null"))
-                ]
-            }
         }
         def apis = new FilteredApis(
                 [up], Selector.empty
@@ -206,7 +209,35 @@ class QuorumRpcReaderSpec extends Specification {
 
         then:
         StepVerifier.create(act)
-                .expectComplete()
+                .expectError()
+                .verify(Duration.ofSeconds(2))
+    }
+
+    def "Return error is upstream returned it"() {
+        setup:
+        def up = Mock(Upstream) {
+            _ * getLag() >> 0
+            _ * isAvailable() >> true
+            _ * getRole() >> UpstreamsConfig.UpstreamRole.STANDARD
+            _ * getApi() >> Mock(Reader) {
+                _ * read(new JsonRpcRequest("eth_test", [])) >>> [
+                        Mono.just(JsonRpcResponse.error(-3010, "test")),
+                ]
+            }
+        }
+        def apis = new FilteredApis(
+                [up], Selector.empty
+        )
+        def reader = new QuorumRpcReader(apis, new NotLaggingQuorum(1))
+
+        when:
+        def act = reader.read(new JsonRpcRequest("eth_test", []))
+
+        then:
+        StepVerifier.create(act)
+                .expectError()
+        //TODO verify
+        //.expectErrorMatches { t -> t instanceof RpcException && t.code == -3010}
                 .verify(Duration.ofSeconds(1))
     }
 

src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy

@@ -63,7 +63,7 @@ class NativeCallSpec extends Specification {
         when:
         def act = nativeCall.fetch(ctx).block(Duration.ofSeconds(1))
         then:
-        act.payload == "1".bytes
+        act.result == "1".bytes
     }
 
     def "Return error if router denied the requests"() {
@@ -85,12 +85,10 @@ class NativeCallSpec extends Specification {
         def act = nativeCall.fetch(ctx) //.block(Duration.ofSeconds(1))
         then:
         StepVerifier.create(act)
-                .expectErrorMatches { t ->
+                .expectNextMatches { result ->
-                    t instanceof NativeCall.CallFailure &&
+                    result.id == 15 && result.isError()
-                            t.id == 15 &&
-                            t.reason instanceof RpcException &&
-                            t.reason.rpcMessage == "Test message"
                 }
+                .expectComplete()
                 .verify(Duration.ofSeconds(1))
     }
 
@@ -109,7 +107,7 @@ class NativeCallSpec extends Specification {
 
         when:
         def resp = nativeCall.executeOnRemote(call).block(Duration.ofSeconds(1))
-        def act = objectMapper.readValue(resp.payload, Object)
+        def act = objectMapper.readValue(resp.result, Object)
         then:
         act == "foo"
     }
@@ -131,7 +129,10 @@ class NativeCallSpec extends Specification {
         def resp = nativeCall.executeOnRemote(call)
         then:
         StepVerifier.create(resp)
-                .expectErrorMatches({ t -> t instanceof NativeCall.CallFailure && t.id == 1 })
+                .expectNextMatches { result ->
+                    result.isError()
+                }
+                .expectComplete()
                 .verify(Duration.ofSeconds(1))
     }
 
@@ -176,7 +177,7 @@ class NativeCallSpec extends Specification {
 
         when:
         def resp = nativeCall.buildResponse(
-                new NativeCall.CallContext<byte[]>(1561, TestingCommons.multistream(TestingCommons.api()), Selector.empty, new AlwaysQuorum(), objectMapper.writeValueAsBytes(json))
+                new NativeCall.CallResult(1561, objectMapper.writeValueAsBytes(json), null)
         )
         then:
         resp.id == 1561

testing/dshackle/README.adoc

@@ -0,0 +1,7 @@
+= Testing Server
+
+.Run from project root directory:
+[source,bash]
+----
+./gradlew run --args="--configPath=./testing/dshackle/dshackle.yaml"
+----

testing/dshackle/dshackle.yaml

@@ -0,0 +1,28 @@
+version: v1
+port: 12448
+
+tls:
+  enabled: false
+
+cluster:
+  upstreams:
+    - id: test-1
+      chain: ethereum
+      options:
+        disable-validation: true
+      connection:
+        ethereum:
+          rpc:
+            url: "http://localhost:18545"
+
+cache:
+  redis:
+    enabled: false
+
+proxy:
+  port: 18080
+  tls:
+    enabled: false
+  routes:
+    - id: eth
+      blockchain: ethereum

testing/simple-upstream/README.adoc

@@ -0,0 +1,6 @@
+= Simple Testing Upstream
+
+.Run from current directory
+----
+./gradlew run
+----

testing/simple-upstream/build.gradle

@@ -0,0 +1,22 @@
+plugins {
+    id 'java'
+    id 'groovy'
+    id 'idea'
+    id 'application'
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+dependencies {
+    implementation "com.sparkjava:spark-core:2.9.1"
+    implementation "org.codehaus.groovy:groovy:3.0.4"
+    implementation "com.fasterxml.jackson.core:jackson-core:2.9.8"
+    implementation "com.fasterxml.jackson.core:jackson-databind:2.9.8"
+}
+
+application {
+    mainClassName = 'testing.SimpleUpstream'
+}

testing/simple-upstream/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

testing/simple-upstream/gradlew

@@ -0,0 +1,185 @@
+#!/usr/bin/env sh
+
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=`expr $i + 1`
+    done
+    case $i in
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=`save "$@"`
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+exec "$JAVACMD" "$@"

testing/simple-upstream/gradlew.bat

@@ -0,0 +1,104 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

testing/simple-upstream/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'dschackle-testing-simple-upstream'

testing/simple-upstream/src/main/groovy/testing/BlocksHandler.groovy

@@ -0,0 +1,39 @@
+package testing
+
+import com.fasterxml.jackson.databind.ObjectMapper
+
+class BlocksHandler implements CallHandler {
+
+    ObjectMapper objectMapper
+
+    BlocksHandler(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper
+    }
+
+    @Override
+    Result handle(String method, List<Object> params) {
+        if (method == "eth_blockNumber") {
+            return Result.ok("0x100001")
+        }
+        if (method == "eth_getBlockByNumber") {
+            String blockId = params[0]
+            println("get block $blockId")
+            def result = getResource("block-${blockId}.json")
+            return Result.ok(result)
+        }
+        if (method == "eth_getTransactionByHash") {
+            String txId = params[0]
+            def result = getResource("tx-${txId}.json")
+            return Result.ok(result)
+        }
+        return null
+    }
+
+    Object getResource(String name) {
+        String json = BlocksHandler.class.getResourceAsStream("/" + name)?.text
+        if (json == null) {
+            return null
+        }
+        return objectMapper.readValue(json, Map)
+    }
+}

testing/simple-upstream/src/main/groovy/testing/CallHandler.groovy

@@ -0,0 +1,42 @@
+package testing
+
+interface CallHandler {
+
+    Result handle(String method, List<Object> params)
+
+    static class Result {
+        private Object result
+        private Integer errorCode
+        private String errorMessage
+
+        Result(Object result, Integer errorCode, String errorMessage) {
+            this.result = result
+            this.errorCode = errorCode
+            this.errorMessage = errorMessage
+        }
+
+        static Result ok(Object result) {
+            return new Result(result, null, null)
+        }
+
+        static Result error(int errorCode, String errorMessage) {
+            return new Result(null, errorCode, errorMessage)
+        }
+
+        boolean isResult() {
+            return errorCode == null
+        }
+
+        Object getResult() {
+            return result
+        }
+
+        int getErrorCode() {
+            return errorCode
+        }
+
+        String getErrorMessage() {
+            return errorMessage
+        }
+    }
+}

testing/simple-upstream/src/main/groovy/testing/CommonHandlers.groovy

@@ -0,0 +1,13 @@
+package testing
+
+class CommonHandlers implements CallHandler {
+
+
+    @Override
+    Result handle(String method, List<Object> params) {
+        if (method == "eth_syncing") {
+            return Result.ok(false)
+        }
+        return null
+    }
+}

testing/simple-upstream/src/main/groovy/testing/InvalidCallHandler.groovy

@@ -0,0 +1,8 @@
+package testing
+
+class InvalidCallHandler implements CallHandler {
+    @Override
+    Result handle(String method, List<Object> params) {
+        return Result.error(-32000, "Call is not supported")
+    }
+}

testing/simple-upstream/src/main/groovy/testing/SimpleUpstream.groovy

@@ -0,0 +1,62 @@
+package testing
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import spark.Spark
+
+class SimpleUpstream {
+
+    private int port = 18545
+    private ObjectMapper objectMapper
+
+    private List<CallHandler> handlers = []
+
+    void prepare() {
+        objectMapper = new ObjectMapper()
+
+        handlers << new TestcaseHandler()
+        handlers << new CommonHandlers()
+        handlers << new BlocksHandler(objectMapper)
+        handlers << new InvalidCallHandler()
+    }
+
+    void start() {
+        println("Starting upstream on 0.0.0.0:$port")
+        Spark.port(port)
+        Spark.post("/") { req, resp ->
+            println("request")
+            try {
+                Map json = objectMapper.readValue(req.body(), Map)
+                def id = json["id"]
+                String method = json["method"].toString()
+                List<Object> params = json.containsKey("params") ? json["params"] as List<Object> : []
+
+                CallHandler.Result result = handlers.findResult { h ->
+                    return h.handle(method, params)
+                }
+                Map resultJson = [
+                        id     : id,
+                        jsonrpc: "2.0"
+                ]
+                if (result.isResult()) {
+                    resultJson["result"] = result.result
+                } else {
+                    resultJson["error"] = [
+                            code   : result.getErrorCode(),
+                            message: result.getErrorMessage()
+                    ]
+                }
+                resp.status(200)
+                resp.header("content-type", "application/json")
+                return objectMapper.writeValueAsString(resultJson)
+            } catch (Throwable t) {
+                t.printStackTrace()
+            }
+        }
+    }
+
+    public static void main(String[] args) {
+        def server = new SimpleUpstream()
+        server.prepare()
+        server.start()
+    }
+}

testing/simple-upstream/src/main/groovy/testing/TestcaseHandler.groovy

@@ -0,0 +1,14 @@
+package testing
+
+class TestcaseHandler implements CallHandler {
+
+    @Override
+    Result handle(String method, List<Object> params) {
+        // https://github.com/emeraldpay/dshackle/issues/35
+        if (method == "eth_call"
+                && params[0].to?.toLowerCase() == "0x542156d51D10Db5acCB99f9Db7e7C91B74E80a2c".toLowerCase()) {
+            return Result.error(-32015, "VM execution error.")
+        }
+        return null
+    }
+}

testing/simple-upstream/src/main/resources/block-0x100000.json

@@ -0,0 +1,25 @@
+{
+  "difficulty": "0xc6d2fa46fd6",
+  "extraData": "0xd783010400844765746887676f312e352e31856c696e7578",
+  "gasLimit": "0x2fefd8",
+  "gasUsed": "0xa410",
+  "hash": "0x9a834c53bbee9c2665a5a84789a1d1ad73750b2d77b50de44f457f411d02e52e",
+  "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+  "miner": "0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da",
+  "mixHash": "0x72224fd0b7b32d0fea13f2953eba28d2512af43632725ed75583dd30e42dfc64",
+  "nonce": "0xf1a453a44f9de627",
+  "number": "0x100000",
+  "parentHash": "0x2dcaf9a8a3fd329d925eb47c1f22022d4f2d562500e546d878bad4247d013858",
+  "receiptsRoot": "0x178000485de197ed06a4ebada431864e5b564b43f994fb8888521f74b4b3257b",
+  "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+  "size": "0x303",
+  "stateRoot": "0xeaddaa0b6b673ba3f8cb7235ca656340ccf703295cf5252cb04a4202c47252c2",
+  "timestamp": "0x56cc7b73",
+  "totalDifficulty": "0x6bab93cdf810b22a",
+  "transactions": [
+    "0x01c5a8461d06c2c195035c148af0f871c7679841d86ae5bb98676bb2d8e68dfa",
+    "0xb4216e88df6ebfe666bbe43370d1ddfe8d9c1975d2b8375a522700f7f59c3ed0"
+  ],
+  "transactionsRoot": "0x91d9b419b34a790cc2a7d59c6b95e57ccc5ec9bc1034ce9510dfb551d3049c93",
+  "uncles": []
+}

testing/simple-upstream/src/main/resources/block-0x100001.json

@@ -0,0 +1,25 @@
+{
+  "difficulty": "0xc6ba1fe7c49",
+  "extraData": "0xd783010400844765746887676f312e352e31856c696e7578",
+  "gasLimit": "0x2fefd8",
+  "gasUsed": "0xa410",
+  "hash": "0x18c68d9ba58772a4409d65d61891b25db03a105a7769ae08ef2cff697921b446",
+  "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+  "miner": "0x0c729be7c39543c3d549282a40395299d987cec2",
+  "mixHash": "0xf8679fe0f04d5aad4844694cc8fb1a1b97482a5822658379c9bb8633f91daf97",
+  "nonce": "0x347a267c0ec88618",
+  "number": "0x100001",
+  "parentHash": "0x9a834c53bbee9c2665a5a84789a1d1ad73750b2d77b50de44f457f411d02e52e",
+  "receiptsRoot": "0x4f4ed1139baadbd4506d6b0c330335d2108715095fb93fd282bd69aa9edc09eb",
+  "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+  "size": "0x304",
+  "stateRoot": "0xee13382462fcd5748a403ca131c38e871fa7206bca2b962f67b59e800741daa3",
+  "timestamp": "0x56cc7b8c",
+  "totalDifficulty": "0x6baba0399a0f2e73",
+  "transactions": [
+    "0x146b8f4b6300c73bb7476359b9f1c5ee3f686a86b2aa673552cf0f9de9a42e77",
+    "0xe589a39acea3091b584b650158d08b159aa07e97b8e8cddb8f81cb606e13382e"
+  ],
+  "transactionsRoot": "0xc90078e2af52aef81815cb2a71c22ebd781dd658dd953d9df57f7769a0b2fe51",
+  "uncles": []
+}

testing/simple-upstream/src/main/resources/tx-0x01c5a8461d06c2c195035c148af0f871c7679841d86ae5bb98676bb2d8e68dfa.json

@@ -0,0 +1,16 @@
+{
+  "blockHash": "0x9a834c53bbee9c2665a5a84789a1d1ad73750b2d77b50de44f457f411d02e52e",
+  "blockNumber": "0x100000",
+  "from": "0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da",
+  "gas": "0x15f90",
+  "gasPrice": "0xba43b7400",
+  "hash": "0x01c5a8461d06c2c195035c148af0f871c7679841d86ae5bb98676bb2d8e68dfa",
+  "input": "0x",
+  "nonce": "0x2f31",
+  "r": "0x1d263f32de00456cbc3e4099c623a6ca48e3188f01dd16461687512f8c3bb7d6",
+  "s": "0x62629f42a98f7c5fa60f8c3630cf67b5eb4772e0ff65a458d2116c15e22d6451",
+  "to": "0xdae787ec66e65c60ad35203800b97b45fcb0f909",
+  "transactionIndex": "0x0",
+  "v": "0x1b",
+  "value": "0x2b81097c919a9e000"
+}

testing/simple-upstream/src/main/resources/tx-0x146b8f4b6300c73bb7476359b9f1c5ee3f686a86b2aa673552cf0f9de9a42e77.json

@@ -0,0 +1,16 @@
+{
+  "blockHash": "0x18c68d9ba58772a4409d65d61891b25db03a105a7769ae08ef2cff697921b446",
+  "blockNumber": "0x100001",
+  "from": "0x2a65aca4d5fc5b5c859090a6c34d164135398226",
+  "gas": "0x15f90",
+  "gasPrice": "0xba43b7400",
+  "hash": "0x146b8f4b6300c73bb7476359b9f1c5ee3f686a86b2aa673552cf0f9de9a42e77",
+  "input": "0x",
+  "nonce": "0x32c13",
+  "r": "0xe6ca408c2e91163b46c6bb66054a000a25b6c1d8024460499a185ab2e4bb41a0",
+  "s": "0x53bc43662fda46c2e7051fa704b75ec815e9102cca5400817011ffd99e77139d",
+  "to": "0x7388ae5bc873163025fffe5956322f734f9b570d",
+  "transactionIndex": "0x0",
+  "v": "0x1b",
+  "value": "0xe660f79cf5e4800"
+}

testing/simple-upstream/src/main/resources/tx-0xb4216e88df6ebfe666bbe43370d1ddfe8d9c1975d2b8375a522700f7f59c3ed0.json

@@ -0,0 +1,16 @@
+{
+  "blockHash": "0x9a834c53bbee9c2665a5a84789a1d1ad73750b2d77b50de44f457f411d02e52e",
+  "blockNumber": "0x100000",
+  "from": "0x68795c4aa09d6f4ed3e5deddf8c2ad3049a601da",
+  "gas": "0x15f90",
+  "gasPrice": "0xba43b7400",
+  "hash": "0xb4216e88df6ebfe666bbe43370d1ddfe8d9c1975d2b8375a522700f7f59c3ed0",
+  "input": "0x",
+  "nonce": "0x2f32",
+  "r": "0xc25ceef654cd3bd734201fc08589970e0663431121bc47cc4a70335e072b09c0",
+  "s": "0x1c67b037a59a8f5badde3a7eb586d7888dcf6dad64aabb080c4c0db593a91254",
+  "to": "0x04a2a3ee9c9ea82e1f918ece209f8a7e6d7521e7",
+  "transactionIndex": "0x1",
+  "v": "0x1c",
+  "value": "0x377fb2f2cbd73c00"
+}

testing/simple-upstream/src/main/resources/tx-0xe589a39acea3091b584b650158d08b159aa07e97b8e8cddb8f81cb606e13382e.json

@@ -0,0 +1,16 @@
+{
+  "blockHash": "0x18c68d9ba58772a4409d65d61891b25db03a105a7769ae08ef2cff697921b446",
+  "blockNumber": "0x100001",
+  "from": "0x2a65aca4d5fc5b5c859090a6c34d164135398226",
+  "gas": "0x15f90",
+  "gasPrice": "0xba43b7400",
+  "hash": "0xe589a39acea3091b584b650158d08b159aa07e97b8e8cddb8f81cb606e13382e",
+  "input": "0x",
+  "nonce": "0x32c14",
+  "r": "0x77fe7cfb03258f2eed8ffbbdc19f73f4a6095a55a7e06f81ade02f3435f7e249",
+  "s": "0x821ad733ceecfa7567e9c8ffb66e3f8bbc3b2b7b21bacadb2a065e57b2a6992",
+  "to": "0x1c774e876a46f51e305add171ba3bb4589408273",
+  "transactionIndex": "0x1",
+  "v": "0x1b",
+  "value": "0xe5ef66c73b5fc00"
+}