From f878579c7ecb533a3982f42b639ec83a376b8cb9 Mon Sep 17 00:00:00 2001 From: KirillPamPam Date: Mon, 12 Jun 2023 18:21:08 +0400 Subject: [PATCH] Better performance of tx deserialization (#229) --- .../json/TransactionJsonSnapshot.java | 79 +++++++++++++++++++ .../TransactionJsonSnapshotDeserializer.java | 37 +++++++++ .../emeraldpay/dshackle/data/TxContainer.kt | 11 +++ .../dshackle/rpc/TrackEthereumTx.kt | 6 +- .../ethereum/EthereumCachingReader.kt | 8 +- .../upstream/ethereum/EthereumDirectReader.kt | 4 +- .../upstream/ethereum/EthereumFees.kt | 6 +- .../upstream/ethereum/EthereumLegacyFees.kt | 4 +- .../upstream/ethereum/EthereumPriorityFees.kt | 4 +- .../ethereum/EthereumLegacyFeesSpec.groovy | 6 +- .../ethereum/EthereumPriorityFeesSpec.groovy | 12 +-- 11 files changed, 152 insertions(+), 25 deletions(-) create mode 100644 src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshot.java create mode 100644 src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshotDeserializer.java diff --git a/src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshot.java b/src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshot.java new file mode 100644 index 00000000..2ea55976 --- /dev/null +++ b/src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshot.java @@ -0,0 +1,79 @@ +package io.emeraldpay.dshackle.upstream.ethereum.json; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.emeraldpay.etherjar.domain.BlockHash; +import io.emeraldpay.etherjar.domain.TransactionRef; +import io.emeraldpay.etherjar.domain.Wei; +import io.emeraldpay.etherjar.rpc.json.TransactionRefJson; + +import java.io.Serializable; + +@JsonDeserialize(using = TransactionJsonSnapshotDeserializer.class) +public class TransactionJsonSnapshot extends TransactionRefJson implements TransactionRef, Serializable { + /** + * hash of the block where this transaction was in. null when its pending. + */ + private BlockHash blockHash; + + /** + * block number where this transaction was in. null when its pending. + */ + private Long blockNumber; + + /** + * gas price provided by the sender in Wei. + */ + private Wei gasPrice; + private Wei maxFeePerGas; + private Wei maxPriorityFeePerGas; + + private int type = 0; + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public BlockHash getBlockHash() { + return blockHash; + } + + public void setBlockHash(BlockHash blockHash) { + this.blockHash = blockHash; + } + + public Long getBlockNumber() { + return blockNumber; + } + + public void setBlockNumber(Long blockNumber) { + this.blockNumber = blockNumber; + } + + public Wei getGasPrice() { + return gasPrice; + } + + public void setGasPrice(Wei gasPrice) { + this.gasPrice = gasPrice; + } + + public Wei getMaxFeePerGas() { + return maxFeePerGas; + } + + public void setMaxFeePerGas(Wei maxFeePerGas) { + this.maxFeePerGas = maxFeePerGas; + } + + public Wei getMaxPriorityFeePerGas() { + return maxPriorityFeePerGas; + } + + public void setMaxPriorityFeePerGas(Wei maxPriorityFeePerGas) { + this.maxPriorityFeePerGas = maxPriorityFeePerGas; + } +} diff --git a/src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshotDeserializer.java b/src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshotDeserializer.java new file mode 100644 index 00000000..36add7d1 --- /dev/null +++ b/src/main/java/io/emeraldpay/dshackle/upstream/ethereum/json/TransactionJsonSnapshotDeserializer.java @@ -0,0 +1,37 @@ +package io.emeraldpay.dshackle.upstream.ethereum.json; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import io.emeraldpay.etherjar.rpc.json.EtherJsonDeserializer; + +import java.io.IOException; + +public class TransactionJsonSnapshotDeserializer extends EtherJsonDeserializer { + + @Override + public TransactionJsonSnapshot deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { + JsonNode node = jp.readValueAsTree(); + return deserialize(node); + } + + public TransactionJsonSnapshot deserialize(JsonNode node) { + TransactionJsonSnapshot tx = new TransactionJsonSnapshot(); + tx.setHash(getTxHash(node, "hash")); + tx.setBlockHash(getBlockHash(node, "blockHash")); + Long blockNumber = getLong(node, "blockNumber"); + if (blockNumber != null) { + tx.setBlockNumber(blockNumber); + } + Integer type = getInt(node, "type"); + if (type != null) { + tx.setType(type); + } + tx.setGasPrice(getWei(node, "gasPrice")); + tx.setMaxFeePerGas(getWei(node, "maxFeePerGas")); + tx.setMaxPriorityFeePerGas(getWei(node, "maxPriorityFeePerGas")); + + return tx; + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt index a99ef8d8..d68ddf95 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.data import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.rpc.json.TransactionJson class TxContainer( @@ -48,6 +49,16 @@ class TxContainer( tx ) } + + fun from(tx: TransactionJsonSnapshot, raw: ByteArray): TxContainer { + return TxContainer( + tx.blockNumber, + TxId.from(tx.hash), + tx.blockHash?.let { BlockId.from(it) }, + raw, + tx + ) + } } override fun equals(other: Any?): Boolean { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt index 1fbcdb52..98e236dc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt @@ -26,10 +26,10 @@ import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.rpc.RpcException -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Qualifier @@ -159,7 +159,7 @@ class TrackEthereumTx( .txByHash().read(tx.txid) .onErrorResume(RpcException::class.java) { t -> log.warn("Upstream error, ignoring. {}", t.rpcMessage) - Mono.empty() + Mono.empty() } .flatMap { updateFromBlock(upstream, tx, it) } .doOnError { t -> @@ -212,7 +212,7 @@ class TrackEthereumTx( } } - fun updateFromBlock(upstream: EthereumPosMultiStream, tx: TxDetails, blockTx: TransactionJson): Mono { + fun updateFromBlock(upstream: EthereumPosMultiStream, tx: TxDetails, blockTx: TransactionJsonSnapshot): Mono { return if (blockTx.blockNumber != null && blockTx.blockHash != null && blockTx.blockHash != ZERO_BLOCK) { val updated = tx.withStatus( blockHash = blockTx.blockHash, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt index c0ba9601..4f4a36ab 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt @@ -40,11 +40,11 @@ import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.apache.commons.collections4.Factory import org.slf4j.LoggerFactory @@ -80,8 +80,8 @@ open class EthereumCachingReader( } } - val extractTx = Function { tx -> - tx.getParsed(TransactionJson::class.java) ?: objectMapper.readValue(tx.json, TransactionJson::class.java) + val extractTx = Function { tx -> + tx.getParsed(TransactionJsonSnapshot::class.java) ?: objectMapper.readValue(tx.json, TransactionJsonSnapshot::class.java) } val asRaw = Function { tx -> @@ -141,7 +141,7 @@ open class EthereumCachingReader( ) } - open fun txByHash(): Reader { + open fun txByHash(): Reader { return TransformingReader( CompoundReader( RekeyingReader(txHashToId, caches.getTxByHash()), diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt index 8efcf65b..5b40b7d3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt @@ -16,6 +16,7 @@ import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.domain.BlockHash @@ -24,7 +25,6 @@ import io.emeraldpay.etherjar.domain.Wei import io.emeraldpay.etherjar.hex.HexQuantity import io.emeraldpay.etherjar.rpc.RpcException import io.emeraldpay.etherjar.rpc.RpcResponseError -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.apache.commons.collections4.Factory @@ -79,7 +79,7 @@ class EthereumDirectReader( return readWithQuorum(request) // retries were removed because we use NotNullQuorum which handle errors too .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Tx not read $key"))) .flatMap { txbytes -> - val tx = objectMapper.readValue(txbytes, TransactionJson::class.java) + val tx = objectMapper.readValue(txbytes, TransactionJsonSnapshot::class.java) if (tx == null) { Mono.empty() } else { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt index 40f34242..8c815b12 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt @@ -19,8 +19,8 @@ import io.emeraldpay.dshackle.upstream.AbstractChainFees import io.emeraldpay.dshackle.upstream.ChainFees import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import reactor.core.publisher.Flux @@ -32,7 +32,7 @@ abstract class EthereumFees( upstreams: Multistream, private val reader: EthereumCachingReader, heightLimit: Int, -) : AbstractChainFees, TransactionRefJson, TransactionJson>(heightLimit, upstreams, extractTx), ChainFees { +) : AbstractChainFees, TransactionRefJson, TransactionJsonSnapshot>(heightLimit, upstreams, extractTx), ChainFees { companion object { private val log = LoggerFactory.getLogger(EthereumFees::class.java) @@ -42,7 +42,7 @@ abstract class EthereumFees( } } - abstract fun extractFee(block: BlockJson, tx: TransactionJson): EthereumFee + abstract fun extractFee(block: BlockJson, tx: TransactionJsonSnapshot): EthereumFee override fun readFeesAt(height: Long, selector: TxAt, TransactionRefJson>): Mono { return reader.blocksByHeightParsed().read(height) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt index 93cf69f5..c5861913 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt @@ -17,8 +17,8 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import java.util.function.Function @@ -39,7 +39,7 @@ class EthereumLegacyFees(upstreams: EthereumMultistream, reader: EthereumCaching .build() } - override fun extractFee(block: BlockJson, tx: TransactionJson): EthereumFee { + override fun extractFee(block: BlockJson, tx: TransactionJsonSnapshot): EthereumFee { return EthereumFee(tx.gasPrice, tx.gasPrice, tx.gasPrice, Wei.ZERO) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt index 72aeb0c8..a2b2af8f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt @@ -18,8 +18,8 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import org.slf4j.LoggerFactory import java.util.function.Function @@ -43,7 +43,7 @@ class EthereumPriorityFees(upstreams: Multistream, reader: EthereumCachingReader .build() } - override fun extractFee(block: BlockJson, tx: TransactionJson): EthereumFee { + override fun extractFee(block: BlockJson, tx: TransactionJsonSnapshot): EthereumFee { val baseFee = block.baseFeePerGas ?: Wei.ZERO if (tx.type == 2) { // an EIP-1559 Transaction provides Max and Priority fee diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy index 8c9e8225..a93a66f9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy @@ -15,9 +15,9 @@ */ package io.emeraldpay.dshackle.upstream.ethereum -import io.emeraldpay.etherjar.domain.Wei import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.etherjar.rpc.json.TransactionJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot +import io.emeraldpay.etherjar.domain.Wei import spock.lang.Specification class EthereumLegacyFeesSpec extends Specification { @@ -26,7 +26,7 @@ class EthereumLegacyFeesSpec extends Specification { setup: def block = new BlockJson() // 0x75cc01873a9818bf426a8b23d83450bf18530a822fd4fe9e86a416a5554176a6 - def tx = new TransactionJson().tap { + def tx = new TransactionJsonSnapshot().tap { it.gasPrice = Wei.ofUnits(8, Wei.Unit.GWEI) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy index 517c9813..5deb8f7d 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy @@ -18,10 +18,10 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.ChainFees import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson +import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJsonSnapshot import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.domain.Wei -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import reactor.core.publisher.Flux import reactor.core.publisher.Mono @@ -38,7 +38,7 @@ class EthereumPriorityFeesSpec extends Specification { it.baseFeePerGas = new Wei(104197355513) } // 0x5da50f35a51e56ecd4313417b1c30f9c088222f3f8763701effe14f3dd18b6cc - def tx = new TransactionJson().tap { + def tx = new TransactionJsonSnapshot().tap { it.type = 2 it.maxFeePerGas = Wei.ofUnits(999, Wei.Unit.GWEI) it.maxPriorityFeePerGas = Wei.ofUnits(5.0001, Wei.Unit.GWEI) @@ -60,7 +60,7 @@ class EthereumPriorityFeesSpec extends Specification { it.baseFeePerGas = new Wei(104197355513) } // 0x1f507982bef0f11a8304287d41f228b5f1dda1114a446ee781c3d95ef4a7b891 - def tx = new TransactionJson().tap { + def tx = new TransactionJsonSnapshot().tap { it.type = 0 // 109.564020111 Gwei it.gasPrice = Wei.from("0x198286458f") @@ -111,12 +111,12 @@ class EthereumPriorityFeesSpec extends Specification { new TransactionRefJson(TransactionId.from("0x55555555fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), ] } - def tx1 = new TransactionJson().tap { + def tx1 = new TransactionJsonSnapshot().tap { it.type = 2 it.maxFeePerGas = Wei.ofUnits(150, Wei.Unit.GWEI) it.maxPriorityFeePerGas = Wei.ofUnits(3, Wei.Unit.GWEI) } - def tx2 = new TransactionJson().tap { + def tx2 = new TransactionJsonSnapshot().tap { it.type = 2 it.maxFeePerGas = Wei.ofUnits(200, Wei.Unit.GWEI) it.maxPriorityFeePerGas = Wei.ofUnits(6, Wei.Unit.GWEI)