Chunk huge responses (#292)

src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt

@@ -43,7 +43,7 @@ import java.util.concurrent.TimeUnit
 @Service
 @DependsOn("monitoringSetup")
 class BlockchainRpc(
-    private val nativeCall: NativeCall,
+    private val nativeCallStream: NativeCallStream,
     private val nativeSubscribe: NativeSubscribe,
     private val streamHead: StreamHead,
     private val trackTx: List<TrackTx>,
@@ -78,7 +78,7 @@ class BlockchainRpc(
         var startTime = 0L
         var metrics: RequestMetrics? = null
         val idsMap = mutableMapOf<Int, String>()
-        return nativeCall.nativeCall(
+        return nativeCallStream.nativeCall(
             request
                 .subscribeOn(scheduler)
                 .doOnNext { req ->

src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCallStream.kt

@@ -0,0 +1,69 @@
+package io.emeraldpay.dshackle.rpc
+
+import com.google.protobuf.ByteString
+import io.emeraldpay.api.proto.BlockchainOuterClass.NativeCallReplyItem
+import io.emeraldpay.api.proto.BlockchainOuterClass.NativeCallRequest
+import org.springframework.stereotype.Service
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Mono
+import kotlin.math.min
+
+@Service
+class NativeCallStream(
+    private val nativeCall: NativeCall,
+) {
+
+    fun nativeCall(
+        requestMono: Mono<NativeCallRequest>
+    ): Flux<NativeCallReplyItem> {
+        return requestMono.flatMapMany { req ->
+            nativeCall.nativeCall(Mono.just(req))
+                .map { StreamNativeResult(it, req.chunkSize) }
+                .transform {
+                    if (!req.sorted || req.itemsList.size == 1) {
+                        it
+                    } else {
+                        it.sort { o1, o2 -> o1.response.id - o2.response.id }
+                    }
+                }
+        }.concatMap {
+            val chunkSize = it.chunkSize
+            val response = it.response
+            if (chunkSize == 0 || response.payload.size() <= chunkSize || !response.succeed) {
+                Mono.just(response)
+            } else {
+                Flux.fromIterable(chunks(response, chunkSize))
+            }
+        }
+    }
+
+    private fun chunks(response: NativeCallReplyItem, chunkSize: Int): List<NativeCallReplyItem> {
+        val chunks = mutableListOf<ByteString>()
+        val responseBytes = response.payload
+
+        for (i in 0 until responseBytes.size() step+chunkSize) {
+            chunks.add(responseBytes.substring(i, min(i + chunkSize, responseBytes.size())))
+        }
+
+        return chunks
+            .mapIndexed { index, bytes ->
+                NativeCallReplyItem.newBuilder()
+                    .apply {
+                        id = response.id
+                        payload = bytes
+                        succeed = true
+                        upstreamId = response.upstreamId
+                        chunked = true
+                        finalChunk = index == chunks.size - 1
+                        if (this.finalChunk && response.hasSignature()) {
+                            signature = response.signature
+                        }
+                    }.build()
+            }
+    }
+
+    private data class StreamNativeResult(
+        val response: NativeCallReplyItem,
+        val chunkSize: Int
+    )
+}

src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeNodeStatus.kt

@@ -176,6 +176,6 @@ class SubscribeNodeStatus(
 
     private fun buildStatus(status: UpstreamAvailability, height: Long?): NodeStatus.Builder =
         NodeStatus.newBuilder()
-            .setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(status.grpcId))
+            .setAvailability(Common.AvailabilityEnum.forNumber(status.grpcId))
             .setCurrentHeight(height ?: 0)
 }

src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt

@@ -51,7 +51,7 @@ class SubscribeStatus(
 
     fun chainUnavailable(chain: Chain): BlockchainOuterClass.ChainStatus {
         return BlockchainOuterClass.ChainStatus.newBuilder()
-            .setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE)
+            .setAvailability(Common.AvailabilityEnum.AVAIL_UNAVAILABLE)
             .setChain(Common.ChainRef.forNumber(chain.id))
             .setQuorum(0)
             .build()
@@ -66,7 +66,7 @@ class SubscribeStatus(
             0
         }
         return BlockchainOuterClass.ChainStatus.newBuilder()
-            .setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId))
+            .setAvailability(Common.AvailabilityEnum.forNumber(available.grpcId))
             .setChain(Common.ChainRef.forNumber(chain.id))
             .setQuorum(quorum)
             .build()