problem: some JSON RPC client cannot use IDs and process response by their order

solution: add option to preserve batch request-response correspondence order
rel: #148

src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfig.kt

@@ -50,6 +50,11 @@ open class ProxyConfig {
      */
     var routes: List<Route> = ArrayList()
 
+    /**
+     * Should proxy preserve request-response correspondence when sending batch request via http
+     */
+    var preserveBatchOrder: Boolean = false
+
     class Route(
         /**
          * URL binding for the route. http://$host:$port/$id

src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt

@@ -61,6 +61,9 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
         getValueAsBool(input, "websocket")?.let {
             config.websocketEnabled = it
         }
+        getValueAsBool(input, "preserve-batch-order")?.let {
+            config.preserveBatchOrder = it
+        }
         val currentRoutes = HashSet<String>()
         getList<MappingNode>(input, "routes")?.let { routes ->
             config.routes = routes.value.map { route ->

src/main/kotlin/io/emeraldpay/dshackle/proxy/BaseHandler.kt

@@ -36,7 +36,12 @@ abstract class BaseHandler(
         private val log = LoggerFactory.getLogger(BaseHandler::class.java)
     }
 
-    fun execute(chain: Chain, call: ProxyCall, handler: AccessHandlerHttp.RequestHandler): Publisher<String> {
+    fun execute(
+        chain: Chain,
+        call: ProxyCall,
+        handler: AccessHandlerHttp.RequestHandler,
+        preserveBatchOrder: Boolean = false
+    ): Publisher<String> {
         // return empty response for empty request
         if (call.items.isEmpty()) {
             return if (call.type == ProxyCall.RpcType.BATCH) {
@@ -46,11 +51,20 @@ abstract class BaseHandler(
             }
         }
         val jsons = execute(chain, call.items, handler)
-            .transform(writeRpcJson.toJsons(call))
+
         return if (call.type == ProxyCall.RpcType.SINGLE) {
-            jsons.next()
+            jsons.transform(writeRpcJson.toJsons(call)).next()
         } else {
-            jsons.transform(writeRpcJson.asArray())
+            jsons
+                .let {
+                    if (preserveBatchOrder) {
+                        it.transform(reorderByRequest(call.items))
+                    } else {
+                        it
+                    }
+                }
+                .transform(writeRpcJson.toJsons(call))
+                .transform(writeRpcJson.asArray())
         }
     }
 
@@ -86,4 +100,28 @@ abstract class BaseHandler(
                 }
             }
     }
+
+    /**
+     * Reorders responses to the original request order.
+     * Note that it's highly inefficient because it requires keeping all the responses in memory until last one is processes, so should be used only if
+     * a client is unable to reference responses by their IDs.
+     */
+    fun reorderByRequest(items: List<BlockchainOuterClass.NativeCallItem>): java.util.function.Function<Flux<NativeCall.CallResult>, Flux<NativeCall.CallResult>> {
+        val order = items.map { it.id }
+        return java.util.function.Function { src ->
+            src.collectList()
+                .map { results ->
+                    order.map { id ->
+                        results.find { it.id == id }
+                            // If Proxy is configured to preserve original order it means that a client expect responses at exact same position
+                            // as requests even if a request completely failed for a some reason. It's very unlikely situation, but still possible
+                            // At this case, if we found a gap in responses, we put a default response with an error
+                            ?: NativeCall.CallResult(id, null, NativeCall.CallError(id, "No response", null))
+                    }
+                }
+                .flatMapMany {
+                    Flux.fromIterable(it)
+                }
+        }
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/proxy/HttpHandler.kt

@@ -37,6 +37,7 @@ import java.util.function.BiFunction
  * Responds to HTTP requests made to the Ethereum Proxy Server
  */
 class HttpHandler(
+    private val config: ProxyConfig,
     private val readRpcJson: ReadRpcJson,
     writeRpcJson: WriteRpcJson,
     nativeCall: NativeCall,
@@ -78,7 +79,7 @@ class HttpHandler(
                 requestMetrics.get(chain, "invalid_method").errorMetric.increment()
             }
             .flatMapMany { call ->
-                execute(chain, call, handler)
+                execute(chain, call, handler, config.preserveBatchOrder)
             }
             .onErrorResume(RpcException::class.java) { err ->
                 val id = err.details?.let {

src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyCall.kt

@@ -36,7 +36,7 @@ class ProxyCall(
     /**
      * Mapping from our internal ids to user provided JSON RPC ids.
      */
-    val ids = HashMap<Int, Any>()
+    val ids = ArrayList<Any>()
 
     /**
      * Content of the request

src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt

@@ -80,7 +80,7 @@ class ProxyServer(
         StandardRequestMetrics()
     }
 
-    private val httpHandler = HttpHandler(readRpcJson, writeRpcJson, nativeCall, accessHandler, requestMetrics)
+    private val httpHandler = HttpHandler(config, readRpcJson, writeRpcJson, nativeCall, accessHandler, requestMetrics)
     private val wsHandler: WebsocketHandler? = if (config.websocketEnabled) {
         WebsocketHandler(readRpcJson, writeRpcJson, nativeCall, nativeSubscribe, accessHandler, requestMetrics)
     } else null

src/main/kotlin/io/emeraldpay/dshackle/proxy/ReadRpcJson.kt

@@ -165,10 +165,9 @@ open class ReadRpcJson : Function<ByteArray, ProxyCall> {
         var seq = seqStart
         return items
             .map { json ->
-                val id = seq++
-                context.ids[id] = json.id
+                context.ids.add(json.id)
                 BlockchainOuterClass.NativeCallItem.newBuilder()
-                    .setId(id)
+                    .setId(context.ids.size - 1)
                     .setMethod(json.method)
                     .setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params)))
                     .build()

src/main/kotlin/io/emeraldpay/dshackle/proxy/WriteRpcJson.kt

@@ -45,7 +45,7 @@ open class WriteRpcJson {
         return Function { flux ->
             flux
                 .flatMap { response ->
-                    if (!call.ids.containsKey(response.id)) {
+                    if (call.ids.size <= response.id) {
                         log.warn("ID wasn't requested: ${response.id}")
                         return@flatMap Flux.empty<String>()
                     }