problem: doesn't use tx cache when requesting full block by number

src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt

@@ -28,6 +28,12 @@ class TxContainer(
 ) : SourceContainer(json, parsed) {
 
     companion object {
+        @JvmStatic
+        fun from(raw: ByteArray): TxContainer {
+            val tx = Global.objectMapper.readValue(raw, TransactionJson::class.java)
+            return from(tx, raw)
+        }
+
         @JvmStatic
         fun from(tx: TransactionJson): TxContainer {
             return from(tx, Global.objectMapper.writeValueAsBytes(tx))

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt

@@ -15,20 +15,18 @@
  */
 package io.emeraldpay.dshackle.upstream.ethereum
 
-import com.fasterxml.jackson.databind.ObjectMapper
-import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.data.BlockContainer
 import io.emeraldpay.dshackle.data.BlockId
 import io.emeraldpay.dshackle.data.TxContainer
 import io.emeraldpay.dshackle.data.TxId
 import io.emeraldpay.dshackle.reader.Reader
-import io.infinitape.etherjar.rpc.json.BlockJson
-import io.infinitape.etherjar.rpc.json.TransactionJson
-import io.infinitape.etherjar.rpc.json.TransactionRefJson
 import org.slf4j.LoggerFactory
-import org.springframework.beans.BeanUtils
 import reactor.core.publisher.Flux
 import reactor.core.publisher.Mono
+import reactor.util.function.Tuple2
+import reactor.util.function.Tuples
+import java.nio.ByteBuffer
+import java.util.function.BiFunction
 
 /**
  * Reads blocks with full transactions details. Based on data contained in readers for blocks
@@ -46,40 +44,91 @@ class EthereumFullBlocksReader(
         private val log = LoggerFactory.getLogger(EthereumFullBlocksReader::class.java)
     }
 
-    override fun read(key: BlockId): Mono<BlockContainer> {
-        return blocks.read(key).flatMap { block ->
-            val block = Global.objectMapper.readValue(block.json, BlockJson::class.java) as BlockJson<TransactionRefJson>
-            val fullBlock = if (block.transactions == null || block.transactions.isEmpty()) {
-                // in fact it's not necessary to create a copy, made just for code clarity but it may be a performance loss
-                val fullBlock = BlockJson<TransactionJson>()
-                BeanUtils.copyProperties(block, fullBlock)
-                Mono.just(fullBlock)
-            } else {
-                Flux.fromIterable(block.transactions)
-                        .map { TxId.from(it.hash) }
-                        .flatMap { txes.read(it) }
-                        .collectList()
-                        .flatMap { list ->
-                            if (block.transactions.size != list.size) {
-                                Mono.empty<BlockJson<TransactionJson>>()
-                            } else {
-                                val fullBlock = BlockJson<TransactionJson>()
-                                BeanUtils.copyProperties(block, fullBlock)
-                                fullBlock.transactions = list.map {
-                                    Global.objectMapper.readValue(it.json, TransactionJson::class.java)
-                                }
-                                Mono.just(fullBlock)
-                            }
-                        }
-            }
-            fullBlock
-                    .map { block ->
-                        BlockContainer(block.number, BlockId.from(block.hash), block.totalDifficulty, block.timestamp, true,
-                                Global.objectMapper.writeValueAsBytes(block),
-                                block.transactions.map { tx -> TxId.from(tx) }
-                        )
-                    }
+    private val accumulate: BiFunction<ByteBuffer, ByteArray, ByteBuffer> = BiFunction { buf, x ->
+        if (buf.remaining() < x.size) {
+            val resize = ByteBuffer.allocate(buf.capacity() + buf.capacity() / 4 + x.size)
+            resize.put(buf.flip()).put(x)
+        } else {
+            buf.put(x)
         }
     }
 
+    override fun read(key: BlockId): Mono<BlockContainer> {
+        return blocks.read(key).flatMap { block ->
+            if (block.transactions.isEmpty()) {
+                // in fact it's not necessary to create a copy, made just for code clarity but it may be a performance loss
+                val fullBlock = BlockContainer(
+                        block.height, block.hash, block.difficulty, block.timestamp,
+                        true,
+                        block.json,
+                        block.parsed,
+                        block.transactions
+                )
+                return@flatMap Mono.just(fullBlock)
+            }
+
+            val blockSplit = splitByTransactions(block.json!!)
+
+            val transactions = Flux.fromIterable(block.transactions)
+                    .flatMap { txes.read(it) }
+                    .collectList()
+
+            return@flatMap transactions.flatMap { transactionsData ->
+                // make sure that all transaction are loaded, otherwise just return empty because cannot make full block data
+                if (transactionsData.size != block.transactions.size) {
+                    log.warn("No data to fill the block")
+                    Mono.empty()
+                } else {
+                    joinWithTransactions(blockSplit.t1, blockSplit.t2, Flux.fromIterable(transactionsData).map { it.json!! })
+                            .reduce(ByteBuffer.allocate(block.json.size * 4), accumulate)
+                            .map { it.flip().array() }
+                            .map { json ->
+                                BlockContainer(block.height, block.hash, block.difficulty, block.timestamp,
+                                        true,
+                                        json,
+                                        null,
+                                        block.transactions
+                                )
+                            }
+                }
+            }
+        }
+    }
+
+    fun splitByTransactions(json: ByteArray): Tuple2<ByteArray, ByteArray> {
+        //TODO find a lib that implements Knuth-Morris-Pratt Pattern Matching Algorithm for byte arrays
+        // and reimplement without making a string copy from bytes
+
+        val s = String(json)
+        val fieldStart = s.indexOf("\"transactions\"")
+        val arrayStart = s.indexOf("[", fieldStart)
+        val arrayEnd = s.indexOf("]", arrayStart)
+
+        val head = s.substring(0, arrayStart + 1)
+        val tail = s.substring(arrayEnd, s.length)
+
+        return Tuples.of(head.toByteArray(), tail.toByteArray())
+    }
+
+    fun joinWithTransactions(head: ByteArray, tail: ByteArray, transactions: Flux<ByteArray>): Flux<ByteArray> {
+        val separator = Flux.range(0, Integer.MAX_VALUE)
+                .map { it != 0 }
+
+        val transactionsWithSeparator = transactions.zipWith(separator)
+                .flatMap {
+                    val tx = Flux.just(it.t1)
+                    if (it.t2) {
+                        Flux.concat(Flux.just(",".toByteArray()), tx)
+                    } else {
+                        tx
+                    }
+                }
+
+        return Flux.concat(
+                Flux.just(head),
+                transactionsWithSeparator,
+                Flux.just(tail)
+        )
+    }
+
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NativeCallRouter.kt

@@ -158,11 +158,15 @@ class NativeCallRouter(
             throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "[0] must be block number")
         }
         val withTx = params[1].toString().toBoolean()
-        return if (withTx) {
-            log.warn("Block by number is not implemented")
-            null
+        var block = reader.blocksByHeightAsCont()
+                .read(number)
+        block = if (withTx) {
+            block.flatMap {
+                fullBlocksReader.read(it.hash)
+            }
         } else {
-            reader.blocksByHeightAsCont().read(number).map { it.json!! }
+            block
         }
+        return block.map { it.json!! }
     }
 }