StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #81 from p2p-org/optimise-block-by-number

Browse Source 
optimise default getBlockByNumber - skip processing requests of tx bo…
This commit is contained in:
a10zn8
2022-12-13 17:06:16 +04:00
committed by GitHub
parent f6296f63ea 817925ea7d
commit bbf8dc707f
5 changed files with 27 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt vendored
View File

@@ -24,7 +24,6 @@ import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.CompoundReader
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumFullBlocksReader
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
@@ -195,14 +194,6 @@ open class Caches(
return txsByHash
}
fun getFullBlocks(): Reader<BlockId, BlockContainer> {
return EthereumFullBlocksReader(blocksByHash, txsByHash)
}
fun getFullBlocksByHeight(): Reader<Long, BlockContainer> {
return BlockByHeight(blocksByHeight, EthereumFullBlocksReader(blocksByHash, txsByHash))
}
fun getReceipts(): Reader<TxId, ByteArray> {
return receiptByHash
}

145
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt
View File

@@ -1,145 +0,0 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.util.function.Tuple2
import reactor.util.function.Tuples
import java.nio.ByteBuffer
import java.util.function.BiFunction
/**
* Reads blocks with full transactions details. Based on data contained in readers for blocks
* and transactions, i.e. two separate readers that must be provided.
*
* If source block, with just transaction hashes is not available, it returns empty
* If any of the expected block transactions is not available it returns empty
*/
class EthereumFullBlocksReader(
private val blocks: Reader<BlockId, BlockContainer>,
private val txes: Reader<TxId, TxContainer>
) : Reader<BlockId, BlockContainer> {
companion object {
private val log = LoggerFactory.getLogger(EthereumFullBlocksReader::class.java)
}
private val accumulate: BiFunction<ByteBuffer, ByteArray, ByteBuffer> = BiFunction { buf, x ->
if (buf.remaining() < x.size) {
val resize = ByteBuffer.allocate(buf.capacity() + buf.capacity() / 4 + x.size)
resize.put(buf.flip()).put(x)
} else {
buf.put(x)
}
}
override fun read(key: BlockId): Mono<BlockContainer> {
return blocks.read(key).flatMap { block ->
if (block.transactions.isEmpty()) {
// in fact it's not necessary to create a copy, made just for code clarity but it may be a performance loss
val fullBlock = BlockContainer(
block.height, block.hash, block.difficulty, block.timestamp,
true,
block.json,
block.parsed,
block.transactions
)
return@flatMap Mono.just(fullBlock)
}
val blockSplit = splitByTransactions(block.json!!)
val transactions = Flux.fromIterable(block.transactions)
.flatMapSequential { txes.read(it) }
.collectList()
return@flatMap transactions.flatMap { transactionsData ->
// make sure that all transaction are loaded, otherwise just return empty because cannot make full block data
if (transactionsData.size != block.transactions.size) {
log.warn("No data to fill the block")
Mono.empty()
} else {
joinWithTransactions(
blockSplit.t1,
blockSplit.t2,
Flux.fromIterable(transactionsData).map { it.json!! }
)
.reduce(ByteBuffer.allocate(block.json.size * 4), accumulate)
.map(this@EthereumFullBlocksReader::extractContent)
.map { json ->
BlockContainer(
block.height, block.hash, block.difficulty, block.timestamp,
true,
json,
null,
block.transactions
)
}
}
}
}
}
fun extractContent(it: ByteBuffer): ByteArray {
val pos = it.position()
val result = ByteArray(pos)
it.flip().get(result, 0, pos)
return result
}
fun splitByTransactions(json: ByteArray): Tuple2<ByteArray, ByteArray> {
// TODO find a lib that implements Knuth-Morris-Pratt Pattern Matching Algorithm for byte arrays
// and reimplement without making a string copy from bytes
val s = String(json)
val fieldStart = s.indexOf("\"transactions\"")
val arrayStart = s.indexOf("[", fieldStart)
val arrayEnd = s.indexOf("]", arrayStart)
val head = s.substring(0, arrayStart + 1)
val tail = s.substring(arrayEnd, s.length)
return Tuples.of(head.toByteArray(), tail.toByteArray())
}
fun joinWithTransactions(head: ByteArray, tail: ByteArray, transactions: Flux<ByteArray>): Flux<ByteArray> {
val separator = Flux.range(0, Integer.MAX_VALUE)
.map { it != 0 }
val transactionsWithSeparator = transactions.zipWith(separator)
.flatMap {
val tx = Flux.just(it.t1)
if (it.t2) {
Flux.concat(Flux.just(",".toByteArray()), tx)
} else {
tx
}
}
return Flux.concat(
Flux.just(head),
transactionsWithSeparator,
Flux.just(tail)
)
}
}

26
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/LocalCallRouter.kt
View File

@@ -47,11 +47,6 @@ class LocalCallRouter(
private val log = LoggerFactory.getLogger(LocalCallRouter::class.java)
}
private val fullBlocksReader = EthereumFullBlocksReader(
reader.blocksByIdAsCont(),
reader.txByHashAsCont()
)
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> {
if (methods.isHardcoded(key.method)) {
return Mono.just(methods.executeHardcoded(key.method))
@@ -107,7 +102,7 @@ class LocalCallRouter(
}
val withTx = params[1].toString().toBoolean()
if (withTx) {
fullBlocksReader.read(hash).map { it.json!! }
null
} else {
reader.blocksByIdAsCont().read(hash).map { it.json!! }
}
@@ -136,6 +131,11 @@ class LocalCallRouter(
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "Must provide 2 parameters")
}
val number: Long
val withTx = params[1].toString().toBoolean()
if (withTx) {
// with Tx request much more efficient in remote call
return null
}
try {
val blockRef = params[0].toString()
when {
@@ -165,16 +165,8 @@ class LocalCallRouter(
} catch (e: IllegalArgumentException) {
throw RpcException(RpcResponseError.CODE_INVALID_METHOD_PARAMS, "[0] must be a block number")
}
val withTx = params[1].toString().toBoolean()
var block = reader.blocksByHeightAsCont()
.read(number)
block = if (withTx) {
block.flatMap {
fullBlocksReader.read(it.hash)
}
} else {
block
}
return block.map { it.json!! }
return reader.blocksByHeightAsCont()
.read(number).map { it.json!! }
}
}