From 8de75003a8f9773d78a6016e9e618b737ca8b2b4 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Mon, 6 Dec 2021 23:15:29 -0500 Subject: [PATCH] solution: initial code for fee estimation --- .../emeraldpay/dshackle/rpc/BlockchainRpc.kt | 7 +- .../io/emeraldpay/dshackle/rpc/EstimateFee.kt | 38 +++ .../dshackle/upstream/AbstractChainFees.kt | 131 +++++++++ .../emeraldpay/dshackle/upstream/ChainFees.kt | 35 +++ .../dshackle/upstream/Multistream.kt | 2 + .../dshackle/upstream/bitcoin/BitcoinConst.kt | 27 ++ .../dshackle/upstream/bitcoin/BitcoinFees.kt | 146 ++++++++++ .../upstream/bitcoin/BitcoinMultistream.kt | 6 + .../upstream/bitcoin/BitcoinReader.kt | 5 + .../upstream/ethereum/EthereumFees.kt | 86 ++++++ .../upstream/ethereum/EthereumLegacyFees.kt | 50 ++++ .../upstream/ethereum/EthereumMultistream.kt | 11 + .../upstream/ethereum/EthereumPriorityFees.kt | 58 ++++ .../upstream/ethereum/EthereumReader.kt | 9 +- ...eumApiMock.groovy => ApiReaderMock.groovy} | 19 +- .../dshackle/test/TestingCommons.groovy | 8 +- .../upstream/AbstractChainFeesSpec.groovy | 253 ++++++++++++++++++ .../dshackle/upstream/MultistreamSpec.groovy | 5 + .../upstream/bitcoin/BitcoinFeesSpec.groovy | 207 ++++++++++++++ .../upstream/bitcoin/BitcoinReaderSpec.groovy | 47 ++++ .../ethereum/EthereumLegacyFeesSpec.groovy | 41 +++ .../ethereum/EthereumPriorityFeesSpec.groovy | 150 +++++++++++ 22 files changed, 1323 insertions(+), 18 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/rpc/EstimateFee.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinConst.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFees.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt rename src/test/groovy/io/emeraldpay/dshackle/test/{EthereumApiMock.groovy => ApiReaderMock.groovy} (94%) create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFeesSpec.groovy create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReaderSpec.groovy create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt index 6d32a561..0c779e51 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt @@ -43,7 +43,8 @@ class BlockchainRpc( @Autowired private val trackTx: List, @Autowired private val trackAddress: List, @Autowired private val describe: Describe, - @Autowired private val subscribeStatus: SubscribeStatus + @Autowired private val subscribeStatus: SubscribeStatus, + @Autowired private val estimateFee: EstimateFee ) : ReactorBlockchainGrpc.BlockchainImplBase() { private val log = LoggerFactory.getLogger(BlockchainRpc::class.java) @@ -165,6 +166,10 @@ class BlockchainRpc( } } + override fun estimateFee(request: Mono): Mono { + return request.flatMap { estimateFee.estimateFee(it) } + } + override fun describe(request: Mono): Mono { describeMetric.increment() return describe.describe(request) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/EstimateFee.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/EstimateFee.kt new file mode 100644 index 00000000..8328c45e --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/EstimateFee.kt @@ -0,0 +1,38 @@ +package io.emeraldpay.dshackle.rpc + +import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.dshackle.upstream.ChainFees +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.grpc.Chain +import io.grpc.Status +import io.grpc.StatusException +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import reactor.core.publisher.Mono + +@Service +class EstimateFee( + @Autowired private val multistreamHolder: MultistreamHolder +) { + + companion object { + private val log = LoggerFactory.getLogger(EstimateFee::class.java) + } + + fun estimateFee(req: BlockchainOuterClass.EstimateFeeRequest): Mono { + val chain = Chain.byId(req.chainValue) + val up = multistreamHolder.getUpstream(chain) ?: return Mono.error( + StatusException( + Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${req.chainValue}") + ) + ) + val mode = ChainFees.extractMode(req) ?: return Mono.error( + StatusException( + Status.UNAVAILABLE.withDescription("UNSUPPORTED MODE: ${req.mode.number}") + ) + ) + return up.getFeeEstimation() + .estimate(mode, req.blocks) + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt new file mode 100644 index 00000000..11d486e1 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractChainFees.kt @@ -0,0 +1,131 @@ +package io.emeraldpay.dshackle.upstream + +import com.github.benmanes.caffeine.cache.Caffeine +import io.emeraldpay.api.proto.BlockchainOuterClass +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import java.time.Duration +import java.util.EnumMap +import java.util.function.Function + +abstract class AbstractChainFees( + private val heightLimit: Int, + private val upstreams: Multistream, + extractTx: (B) -> List? +) : ChainFees { + + companion object { + private val log = LoggerFactory.getLogger(AbstractChainFees::class.java) + } + + private val txSource = EnumMap>(ChainFees.Mode::class.java) + + init { + txSource[ChainFees.Mode.AVG_TOP] = TxAtTop(extractTx) + txSource[ChainFees.Mode.MIN_ALWAYS] = TxAtBottom(extractTx) + txSource[ChainFees.Mode.AVG_MIDDLE] = TxAtMiddle(extractTx) + txSource[ChainFees.Mode.AVG_LAST] = TxAtBottom(extractTx) + txSource[ChainFees.Mode.AVG_T5] = TxAtPos(extractTx, 5) + txSource[ChainFees.Mode.AVG_T20] = TxAtPos(extractTx, 20) + txSource[ChainFees.Mode.AVG_T50] = TxAtPos(extractTx, 50) + } + + override fun estimate(mode: ChainFees.Mode, blocks: Int): Mono { + return usingBlocks(blocks) + .flatMap { readFeesAt(it, mode) } + .transform(feeAggregation(mode)) + .next() + .map(getResponseBuilder()) + } + + // --- + + private val feeCache = Caffeine.newBuilder() + .expireAfterWrite(Duration.ofMinutes(60)) + .build, F>() + + fun usingBlocks(exp: Int): Flux { + val useBlocks = exp.coerceAtMost(heightLimit).coerceAtLeast(1) + + val height = upstreams.getHead().getCurrentHeight() + ?: return Mono.fromCallable { log.warn("Upstream is not ready. No current height") }.thenMany(Mono.empty()) // TODO or throw an exception to build a gRPC error? + val startBlock: Int = height.toInt() - useBlocks + 1 + if (startBlock < 0) { + log.warn("Blockchain doesn't have enough blocks. Height: $height") + return Flux.empty() + } + + return Flux.range(startBlock, useBlocks).map { it.toLong() } + } + + fun readFeesAt(height: Long, mode: ChainFees.Mode): Mono { + val current = feeCache.getIfPresent(Pair(height, mode)) + if (current != null) { + return Mono.just(current) + } + val txSelector = txSourceFor(mode) + return readFeesAt(height, txSelector).doOnNext { + // TODO it may be EMPTY for some blocks (ex. a no tx block), so nothing gets cached and goes to do the same call each time. so do cache empty values to avoid useless requests + feeCache.put(Pair(height, mode), it!!) + } + } + + open fun txSourceFor(mode: ChainFees.Mode): TxAt { + return txSource[mode] ?: throw IllegalStateException("No TS Source for mode $mode") + } + + abstract fun readFeesAt(height: Long, selector: TxAt): Mono + abstract fun feeAggregation(mode: ChainFees.Mode): Function, Mono> + abstract fun getResponseBuilder(): Function + + abstract class TxAt(private val extractTx: Function?>) { + fun get(block: B): TR? { + val txes = extractTx.apply(block) ?: return null + return get(txes) + } + + abstract fun get(transactions: List): TR? + } + + class TxAtPos(extractTx: Function?>, private val pos: Int) : TxAt(extractTx) { + + override fun get(transactions: List): TR? { + val index = pos.coerceAtMost(transactions.size - 1) + if (index < 0) { + return null + } + return transactions[transactions.size - index - 1] + } + } + + class TxAtTop(extractTx: Function?>) : TxAt(extractTx) { + override fun get(transactions: List): TR? { + if (transactions.isEmpty()) { + return null + } + return transactions[0] + } + } + + class TxAtBottom(extractTx: Function?>) : TxAt(extractTx) { + override fun get(transactions: List): TR? { + if (transactions.isEmpty()) { + return null + } + return transactions.last() + } + } + + class TxAtMiddle(extractTx: Function?>) : TxAt(extractTx) { + override fun get(transactions: List): TR? { + if (transactions.isEmpty()) { + return null + } + if (transactions.size == 1) { + return transactions[0] + } + return transactions[transactions.size / 2] + } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt new file mode 100644 index 00000000..8d6d36bf --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainFees.kt @@ -0,0 +1,35 @@ +package io.emeraldpay.dshackle.upstream + +import io.emeraldpay.api.proto.BlockchainOuterClass +import reactor.core.publisher.Mono + +interface ChainFees { + + companion object { + fun extractMode(req: BlockchainOuterClass.EstimateFeeRequest): Mode? { + return when (req.mode!!) { + BlockchainOuterClass.FeeEstimationMode.INVALID -> null + BlockchainOuterClass.FeeEstimationMode.AVG_LAST -> Mode.AVG_LAST + BlockchainOuterClass.FeeEstimationMode.AVG_T5 -> Mode.AVG_T5 + BlockchainOuterClass.FeeEstimationMode.AVG_T20 -> Mode.AVG_T20 + BlockchainOuterClass.FeeEstimationMode.AVG_T50 -> Mode.AVG_T50 + BlockchainOuterClass.FeeEstimationMode.MIN_ALWAYS -> Mode.MIN_ALWAYS + BlockchainOuterClass.FeeEstimationMode.AVG_MIDDLE -> Mode.AVG_MIDDLE + BlockchainOuterClass.FeeEstimationMode.AVG_TOP -> Mode.AVG_TOP + BlockchainOuterClass.FeeEstimationMode.UNRECOGNIZED -> null + } + } + } + + fun estimate(mode: Mode, blocks: Int): Mono + + enum class Mode { + AVG_LAST, + AVG_T5, + AVG_T20, + AVG_T50, + MIN_ALWAYS, + AVG_MIDDLE, + AVG_TOP + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt index 720a75be..bfb7f637 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt @@ -155,6 +155,8 @@ abstract class Multistream( .switchIfEmpty(Mono.error(Exception("No API available for $chain"))) } + abstract fun getFeeEstimation(): ChainFees + /** * Finds an API that leverages caches and other optimizations/transformations of the request. */ diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinConst.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinConst.kt new file mode 100644 index 00000000..f11a9293 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinConst.kt @@ -0,0 +1,27 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import java.math.BigDecimal +import java.math.BigInteger + +class BitcoinConst { + + companion object { + val COIN: BigInteger = BigInteger.TEN.pow(8) + val COIN_DEC: BigDecimal = COIN.toBigDecimal() + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFees.kt new file mode 100644 index 00000000..74577587 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFees.kt @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.dshackle.upstream.AbstractChainFees +import io.emeraldpay.dshackle.upstream.ChainFees +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import java.math.BigDecimal +import java.util.function.Function + +class BitcoinFees( + upstreams: BitcoinMultistream, + private val reader: BitcoinReader, + heightLimit: Int, +) : AbstractChainFees, String, Map>(heightLimit, upstreams, extractTx), ChainFees { + + companion object { + private val log = LoggerFactory.getLogger(BitcoinFees::class.java) + + private val extractTx = { block: Map -> + block["tx"] + ?.let { it as List } + // drop first tx which is a miner reward + ?.let { if (it.isEmpty()) it else it.drop(1) } + ?: emptyList() + } + } + + fun calculateFee(tx: Map): Mono { + val outAmount = extractVOuts(tx).reduce { acc, l -> (acc ?: 0L) + (l ?: 0L) } ?: 0 + val vinsData = tx.get("vin")?.let { it as List> } ?: return Mono.empty() + return Flux.fromIterable(vinsData) + .flatMap { + val txid = it["txid"] as String? + val vout = (it["vout"] as Number?)?.toInt() + if (txid == null || vout == null) { + Mono.empty() + } else { + getTxOutAmount(txid, vout) + } + } + .reduce { t, u -> t + u } + .map { inAmount -> + (inAmount - outAmount).coerceAtLeast(0) + } + } + + fun extractSize(tx: Map): Int { + val size: Number = if (tx.containsKey("vsize")) { + tx["vsize"] as Number + } else if (tx.containsKey("size")) { + tx["size"] as Number + } else { + 0 + } + return size.toInt() + } + + fun getTxOutAmount(txid: String, vout: Int): Mono { + return reader.getTx(txid) + .switchIfEmpty( + Mono.fromCallable { log.warn("No tx $txid") } + .then(Mono.empty()) + ) + .flatMap { + extractVOuts(it).let { + if (vout < it.size) { + Mono.justOrEmpty(it[vout]) + } else { + Mono.empty() + } + } + } + } + + fun extractVOuts(tx: Map): List { + val voutsData = tx.get("vout")?.let { it as List> } ?: return emptyList() + return voutsData.map { + val amount = it["value"] ?: return@map null + BigDecimal(amount.toString()) + .multiply(BitcoinConst.COIN_DEC) + .longValueExact() + } + } + + override fun readFeesAt(height: Long, selector: TxAt, String>): Mono { + return reader.getBlock(height) + .flatMap { block -> + Mono.justOrEmpty(selector.get(block)) + .flatMap { txid -> reader.getTx(txid!!) } + .flatMap { tx -> + calculateFee(tx) + .map { fee -> + TxFee(1, fee * 1024 / extractSize(tx)) + } + } + } + } + + override fun feeAggregation(mode: ChainFees.Mode): Function, Mono> { + if (mode == ChainFees.Mode.MIN_ALWAYS) { + return Function { src -> + src.reduce { a, b -> + if (a.fee > b.fee) a else b + } + } + } + return Function { src -> + src.reduce { a, b -> + TxFee(a.count + b.count, a.fee + b.fee) + } + } + } + + override fun getResponseBuilder(): Function { + return Function { + val fee = (it.fee / it.count).coerceAtLeast(1) + BlockchainOuterClass.EstimateFeeResponse.newBuilder() + .setBitcoinStd( + BlockchainOuterClass.BitcoinStdFees.newBuilder() + .setSatPerKb(fee.toString()) + ) + .build() + } + } + + // ------- + + data class TxFee(val count: Int, val fee: Long) +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt index adba1e65..c7740f1e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt @@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.ChainFees import io.emeraldpay.dshackle.upstream.EmptyHead import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.MergedHead @@ -49,6 +50,7 @@ open class BitcoinMultistream( private var reader = BitcoinReader(this, head, esplora) private var addressActiveCheck: AddressActiveCheck? = null private var xpubAddresses: XpubAddresses? = null + private val feeEstimation = BitcoinFees(this, reader, 6) override fun init() { if (upstreams.size > 0) { @@ -57,6 +59,10 @@ open class BitcoinMultistream( super.init() } + override fun getFeeEstimation(): ChainFees { + return feeEstimation + } + open fun getXpubAddresses(): XpubAddresses? { return xpubAddresses } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt index e1c43c2c..d72148f6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt @@ -55,6 +55,11 @@ open class BitcoinReader( return castedRead(JsonRpcRequest("getblock", listOf(hash)), Map::class.java).cast() } + open fun getBlock(height: Long): Mono> { + return castedRead(JsonRpcRequest("getblockhash", listOf(height)), String::class.java) + .flatMap(this@BitcoinReader::getBlock) + } + open fun getTx(txid: String): Mono> { return castedRead(JsonRpcRequest("getrawtransaction", listOf(txid, true)), Map::class.java).cast() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt new file mode 100644 index 00000000..3e798ecf --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.dshackle.upstream.AbstractChainFees +import io.emeraldpay.dshackle.upstream.ChainFees +import io.emeraldpay.etherjar.domain.Wei +import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson +import io.emeraldpay.etherjar.rpc.json.TransactionRefJson +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import reactor.util.function.Tuples +import java.util.function.Function + +abstract class EthereumFees( + upstreams: EthereumMultistream, + private val reader: EthereumReader, + heightLimit: Int, +) : AbstractChainFees, TransactionRefJson, TransactionJson>(heightLimit, upstreams, extractTx), ChainFees { + + companion object { + private val log = LoggerFactory.getLogger(EthereumFees::class.java) + + private val extractTx = { block: BlockJson -> + block.transactions + } + } + + abstract fun extractFee(block: BlockJson, tx: TransactionJson): EthereumFee + + override fun readFeesAt(height: Long, selector: TxAt, TransactionRefJson>): Mono { + return reader.blocksByHeightParsed().read(height) + .flatMap { block -> + Mono.justOrEmpty(selector.get(block)) + .cast(TransactionRefJson::class.java) + .flatMap { reader.txByHash().read(it.hash) } + .map { tx -> extractFee(block, tx) } + } + } + + override fun feeAggregation(mode: ChainFees.Mode): Function, Mono> { + if (mode == ChainFees.Mode.MIN_ALWAYS) { + return Function { src -> + src.reduce { a, b -> + EthereumFee( + a.max.coerceAtLeast(b.max), + a.priority.coerceAtLeast(b.priority), + a.paid.coerceAtLeast(b.paid), + Wei.ZERO + ) + } + } + } + return Function { src -> + src.map { Tuples.of(1, it) } + .reduce { a, b -> + Tuples.of(a.t1 + b.t1, a.t2.plus(b.t2)) + }.map { + EthereumFee(it.t2.max / it.t1, it.t2.priority / it.t1, it.t2.paid / it.t1, it.t2.base / it.t1) + } + } + } + + // --- + + data class EthereumFee(val max: Wei, val priority: Wei, val paid: Wei, val base: Wei) { + fun plus(o: EthereumFee): EthereumFee { + return EthereumFee(max + o.max, priority + o.priority, paid + o.paid, base + o.base) + } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt new file mode 100644 index 00000000..5407496c --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFees.kt @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.etherjar.domain.Wei +import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson +import io.emeraldpay.etherjar.rpc.json.TransactionRefJson +import org.slf4j.LoggerFactory +import java.util.function.Function + +class EthereumLegacyFees(upstreams: EthereumMultistream, reader: EthereumReader, heightLimit: Int) : + EthereumFees(upstreams, reader, heightLimit) { + + companion object { + private val log = LoggerFactory.getLogger(EthereumLegacyFees::class.java) + } + + private val toGrpc: Function = Function { + BlockchainOuterClass.EstimateFeeResponse.newBuilder() + .setEthereumExtended( + BlockchainOuterClass.EthereumExtFees.newBuilder() + .setMax(it.paid.amount.toString()) + .setPriority(it.priority.amount.toString()) + ) + .build() + } + + override fun extractFee(block: BlockJson, tx: TransactionJson): EthereumFee { + return EthereumFee(tx.gasPrice, tx.gasPrice, tx.gasPrice, Wei.ZERO) + } + + override fun getResponseBuilder(): Function { + return toGrpc + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt index dcb94c20..0e4fcdfe 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt @@ -19,6 +19,7 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.ChainFees import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.MergedHead import io.emeraldpay.dshackle.upstream.Multistream @@ -46,6 +47,12 @@ open class EthereumMultistream( private val reader: EthereumReader = EthereumReader(this, this.caches, getMethodsFactory()) private val subscribe = EthereumSubscribe(this) + private val supportsEIP1559 = when (chain) { + Chain.ETHEREUM, Chain.TESTNET_ROPSTEN, Chain.TESTNET_GOERLI, Chain.TESTNET_RINKEBY -> true + else -> false + } + private val feeEstimation = if (supportsEIP1559) EthereumPriorityFees(this, reader, 256) + else EthereumLegacyFees(this, reader, 256) init { this.init() @@ -133,4 +140,8 @@ open class EthereumMultistream( open fun getSubscribe(): EthereumSubscribe { return subscribe } + + override fun getFeeEstimation(): ChainFees { + return feeEstimation + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt new file mode 100644 index 00000000..f3138799 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.api.proto.BlockchainOuterClass +import io.emeraldpay.etherjar.domain.Wei +import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson +import io.emeraldpay.etherjar.rpc.json.TransactionRefJson +import org.slf4j.LoggerFactory +import java.util.function.Function + +class EthereumPriorityFees(upstreams: EthereumMultistream, reader: EthereumReader, heightLimit: Int) : + EthereumFees(upstreams, reader, heightLimit) { + + companion object { + private val log = LoggerFactory.getLogger(EthereumPriorityFees::class.java) + } + + private val toGrpc: Function = + Function { + BlockchainOuterClass.EstimateFeeResponse.newBuilder() + .setEthereumExtended( + BlockchainOuterClass.EthereumExtFees.newBuilder() + .setMax(it.max.amount.toString()) + .setPriority(it.priority.amount.toString()) + .setExpect(it.paid.amount.toString()) + ) + .build() + } + + override fun extractFee(block: BlockJson, tx: TransactionJson): EthereumFee { + val baseFee = block.baseFeePerGas ?: Wei.ZERO + if (tx.type == 2) { + // an EIP-1559 Transaction provides Max and Priority fee + val paid = (baseFee + tx.maxPriorityFeePerGas).coerceAtMost(tx.maxFeePerGas) + return EthereumFee(tx.maxFeePerGas, tx.maxPriorityFeePerGas, paid, baseFee) + } + return EthereumFee(tx.gasPrice, (tx.gasPrice - baseFee).coerceAtLeast(Wei.ZERO), tx.gasPrice, baseFee) + } + + override fun getResponseBuilder(): Function { + return toGrpc + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt index 607582f2..8501a99b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt @@ -125,7 +125,14 @@ open class EthereumReader( ) } - fun txByHash(): Reader { + open fun blocksByHeightParsed(): Reader> { + return TransformingReader( + blocksByHeightAsCont(), + extractBlock + ) + } + + open fun txByHash(): Reader { return TransformingReader( CompoundReader( RekeyingReader(txHashToId, caches.getTxByHash()), diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy similarity index 94% rename from src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiMock.groovy rename to src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy index f9901f55..4bdaa89c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/EthereumApiMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy @@ -26,7 +26,6 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.grpc.stub.StreamObserver import io.emeraldpay.etherjar.rpc.RpcResponseError -import io.emeraldpay.etherjar.rpc.json.ResponseJson import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBufAllocator import io.netty.buffer.ByteBufInputStream @@ -57,7 +56,7 @@ import java.util.function.BiFunction import java.util.function.Consumer import java.util.function.Predicate -class EthereumApiMock implements Reader { +class ApiReaderMock implements Reader { private static final Logger log = LoggerFactory.getLogger(this) List predefined = [] @@ -66,15 +65,15 @@ class EthereumApiMock implements Reader { String id = "default" AtomicInteger calls = new AtomicInteger(0) - EthereumApiMock() { + ApiReaderMock() { } - EthereumApiMock answerOnce(@NotNull String method, List params, Object result) { + ApiReaderMock answerOnce(@NotNull String method, List params, Object result) { return answer(method, params, result, 1) } - EthereumApiMock answer(@NotNull String method, List params, Object result, - Integer limit = null, Throwable exception = null) { + ApiReaderMock answer(@NotNull String method, List params, Object result, + Integer limit = null, Throwable exception = null) { predefined << new PredefinedResponse(method: method, params: params, result: result, limit: limit, exception: exception) return this } @@ -169,7 +168,7 @@ class EthereumApiMock implements Reader { } class WebsocketApi { - private final EthereumApiMock api + private final ApiReaderMock api private Sinks.Many responses = Sinks .many() @@ -182,7 +181,7 @@ class EthereumApiMock implements Reader { private WebsocketOutboundMock outbound private WebsocketInboundMock inbound - WebsocketApi(EthereumApiMock api) { + WebsocketApi(ApiReaderMock api) { this.api = api outbound = new WebsocketOutboundMock(api, responses) inbound = new WebsocketInboundMock(responses.asFlux(), jsonResponses.asFlux()) @@ -260,10 +259,10 @@ class EthereumApiMock implements Reader { class WebsocketOutboundMock implements WebsocketOutbound { - private final EthereumApiMock api + private final ApiReaderMock api private final Sinks.Many responses - WebsocketOutboundMock(EthereumApiMock api, Sinks.Many responses) { + WebsocketOutboundMock(ApiReaderMock api, Sinks.Many responses) { this.api = api this.responses = responses } diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy index ff8a480f..a9203234 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy @@ -26,7 +26,6 @@ import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Multistream -import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream @@ -39,15 +38,12 @@ import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.logging.LoggingMeterRegistry import org.apache.commons.lang3.StringUtils -import java.time.Duration import java.time.Instant -import java.time.temporal.ChronoUnit -import java.time.temporal.TemporalUnit class TestingCommons { - static EthereumApiMock api() { - return new EthereumApiMock() + static ApiReaderMock api() { + return new ApiReaderMock() } static EthereumUpstreamMock upstream() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy new file mode 100644 index 00000000..37db9b97 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractChainFeesSpec.groovy @@ -0,0 +1,253 @@ +package io.emeraldpay.dshackle.upstream + +import kotlin.jvm.functions.Function1 +import org.jetbrains.annotations.NotNull +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import spock.lang.Specification + +import java.time.Duration +import java.util.function.Function + +class AbstractChainFeesSpec extends Specification { + + Function, List> extractTx = { it } + + def "Iterates N last blocks"() { + setup: + def ups = Mock(Multistream) { + 1 * getHead() >> Mock(Head) { + 1 * it.getCurrentHeight() >> 234 + } + } + def fees = new TestChainFees(5, ups, Stub(Function1)) + when: + def act = fees.usingBlocks(3).collectList().block(Duration.ofSeconds(1)).toSorted() + then: + act == [232L, 233L, 234L] + } + + def "Limits iteration to configured"() { + setup: + def ups = Mock(Multistream) { + 1 * getHead() >> Mock(Head) { + 1 * it.getCurrentHeight() >> 234 + } + } + def fees = new TestChainFees(3, ups, Stub(Function1)) + when: + def act = fees.usingBlocks(5).collectList().block(Duration.ofSeconds(1)).toSorted() + then: + act == [232L, 233L, 234L] + } + + def "TxAtPos 0 for empty list"() { + setup: + def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 0) + + when: + def act = txAt.get([]) + + then: + act == null + } + + def "TxAtPos 0 for single item list"() { + setup: + def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 0) + + when: + def act = txAt.get(["t_1"]) + + then: + act == "t_1" + } + + def "TxAtPos 0 for many item list"() { + setup: + def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 0) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_3" + } + + def "TxAtPos 1 for many item list"() { + setup: + def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 1) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_2" + } + + def "TxAtPos 2 for many item list"() { + setup: + def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 2) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_1" + } + + def "TxAtPos 5 for 3 item list"() { + setup: + def txAt = new AbstractChainFees.TxAtPos, String>(extractTx, 5) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_1" + } + + def "TxAtBottom for empty list"() { + setup: + def txAt = new AbstractChainFees.TxAtBottom, String>(extractTx) + + when: + def act = txAt.get([]) + + then: + act == null + } + + def "TxAtBottom for single item list"() { + setup: + def txAt = new AbstractChainFees.TxAtBottom, String>(extractTx) + + when: + def act = txAt.get(["t_1"]) + + then: + act == "t_1" + } + + def "TxAtBottom for multi item list"() { + setup: + def txAt = new AbstractChainFees.TxAtBottom, String>(extractTx) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_3" + } + + def "TxAtTop for empty list"() { + setup: + def txAt = new AbstractChainFees.TxAtTop, String>(extractTx) + + when: + def act = txAt.get([]) + + then: + act == null + } + + def "TxAtTop for single item list"() { + setup: + def txAt = new AbstractChainFees.TxAtTop, String>(extractTx) + + when: + def act = txAt.get(["t_1"]) + + then: + act == "t_1" + } + + def "TxAtTop for multi item list"() { + setup: + def txAt = new AbstractChainFees.TxAtTop, String>(extractTx) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_1" + } + + def "TxAtMiddle for empty list"() { + setup: + def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) + + when: + def act = txAt.get([]) + + then: + act == null + } + + def "TxAtMiddle for single item list"() { + setup: + def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) + + when: + def act = txAt.get(["t_1"]) + + then: + act == "t_1" + } + + def "TxAtMiddle for 3 item list"() { + setup: + def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) + + when: + def act = txAt.get(["t_1", "t_2", "t_3"]) + + then: + act == "t_2" + } + + def "TxAtMiddle for 4 item list"() { + setup: + def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) + + when: + def act = txAt.get(["t_1", "t_2", "t_3", "t_4"]) + + then: + act == "t_2" || act == "t_3" + } + + def "TxAtMiddle for 5 item list"() { + setup: + def txAt = new AbstractChainFees.TxAtMiddle, String>(extractTx) + + when: + def act = txAt.get(["t_1", "t_2", "t_3", "t_4", "t_5"]) + + then: + act == "t_3" + } + + class TestChainFees extends AbstractChainFees { + + TestChainFees(int heightLimit, @NotNull Multistream upstreams, @NotNull Function1 extractTx) { + super(heightLimit, upstreams, extractTx) + } + + @Override + Mono readFeesAt(long height, @NotNull TxAt selector) { + return null + } + + @Override + Function feeAggregation(@NotNull Mode mode) { + return null + } + + @Override + Function getResponseBuilder() { + return null + } + } + +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy index 78a047d5..60331c22 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy @@ -225,5 +225,10 @@ class MultistreamSpec extends Specification { public T cast(Class selfType) { return this } + + @Override + ChainFees getFeeEstimation() { + return null + } } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFeesSpec.groovy new file mode 100644 index 00000000..7f3b0a38 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinFeesSpec.groovy @@ -0,0 +1,207 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.upstream.ChainFees +import io.emeraldpay.dshackle.upstream.Head +import reactor.core.publisher.Mono +import spock.lang.Specification + +import java.time.Duration + +class BitcoinFeesSpec extends Specification { + + def block3 = [ + hash: "3300000000033327aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506", + tx : [ + "d5a67d8f99ad05ece282bc8714da55cbd2266d9d7bae98fd5dda3b55d1696fb8", + "e17eca6b595da9c1c7b883fe505184d7ff586315d5b8a0c476198f0d49a75ad3", + "1735dfef80bcfdc443943dcbc8d26a4133da87c5c81d9e2a1e0a0e5f59d1ef96", + "c83c99e50946a6d30bc344a24401ad0ff0598425d73a5f74d6545e15ae588321", + "0d4f85f840f09f317d0b202725acee868b2e969078bb6a6b89fc3e0bc1216c6a", + "4d56b1a0992a3fa3132083ce29e203165ef4768f028d98a738da79dd26ff91e2", + ] + ] + def block2 = [ + hash: "2200000000022227aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506", + tx : [ + "87cc7d37851af41da60a5d14e40983d0a6750ffcdf1f2ae1291bfc1e712f0712", + "3bde80635c62e81ba86d68a1c78ac47b1a02d8ef334fdf6a7b6e8acc0a475f20", + "2f4f413c07ac072918e12fcac4afd8d30a76dfe49ac68d46d582c3a130772734", + "f29ecfa5b692cfc46b1eae29416a35782657f0bde2ae44acde882e477e8624d8", + ] + ] + + def tx4D5 = [ + hash : "4d56b1a0992a3fa3132083ce29e203165ef4768f028d98a738da79dd26ff91e2", + size : 223, + vsize: 142, + vin : [ + [txid: "3bde80635c62e81ba86d68a1c78ac47b1a02d8ef334fdf6a7b6e8acc0a475f20", vout: 1] + ], + vout : [ + ["value": 0.00029163, "n": 0], + ["value": 0.00144221, "n": 1] + ] + ] + def txF29 = [ + hash : "f29ecfa5b692cfc46b1eae29416a35782657f0bde2ae44acde882e477e8624d8", + size : 2900, + vsize: 1366, + vin : [ + [txid: "37e19fad6c3e5fafc431ee731428452dc01f56b0bc4d9fe876dcee58384934ec", "vout": 0], + [txid: "3da079b10451ba1cde42d9a0f485e79151e5d4216930561227be5471921af17f", "vout": 0] + ], + vout : [ + ["value": 1.20100000, "n": 0], + ["value": 0.00210984, "n": 1] + ] + ] + + def tx3BD = [ + hash: "3bde80635c62e81ba86d68a1c78ac47b1a02d8ef334fdf6a7b6e8acc0a475f20", + size: 292, + vin : [ + ], + vout: [ + ["value": 0.00400000, "n": 0], + ["value": 0.00200000, "n": 1], + ] + ] + def tx37E = [ + hash : "37e19fad6c3e5fafc431ee731428452dc01f56b0bc4d9fe876dcee58384934ec", + size : 292, + vsize: 200, + vin : [ + ], + vout : [ + ["value": 1.00000000, "n": 0], + ] + ] + def tx3DA = [ + hash : "3da079b10451ba1cde42d9a0f485e79151e5d4216930561227be5471921af17f", + size : 297, + vsize: 201, + vin : [ + ], + vout : [ + ["value": 0.21000000, "n": 0], + ] + ] + + + def "fetch tx amount"() { + setup: + def reader = Mock(BitcoinReader) { + 1 * it.getTx("4d56b1a0992a3fa3132083ce29e203165ef4768f028d98a738da79dd26ff91e2") >> Mono.just(tx4D5) + } + def fees = new BitcoinFees(Stub(BitcoinMultistream), reader, 3) + + when: + def amount = fees.getTxOutAmount("4d56b1a0992a3fa3132083ce29e203165ef4768f028d98a738da79dd26ff91e2", 0) + .block(Duration.ofSeconds(1)) + then: + amount == 29163 + } + + def "extract amounts"() { + setup: + def fees = new BitcoinFees(Stub(BitcoinMultistream), Stub(BitcoinReader), 3) + + when: + def act = fees.extractVOuts(txF29) + then: + act == [120100000L, 210984L] + } + + def "extract vsize when available"() { + setup: + def fees = new BitcoinFees(Stub(BitcoinMultistream), Stub(BitcoinReader), 3) + + when: + def act = fees.extractSize(tx4D5) + then: + act == 142 + } + + def "extract size when vsize unavailable"() { + setup: + def fees = new BitcoinFees(Stub(BitcoinMultistream), Stub(BitcoinReader), 3) + + when: + def act = fees.extractSize(tx3BD) + then: + act == 292 + } + + def "calculates fee"() { + setup: + def reader = Mock(BitcoinReader) { + 1 * it.getTx("3bde80635c62e81ba86d68a1c78ac47b1a02d8ef334fdf6a7b6e8acc0a475f20") >> Mono.just(tx3BD) + } + def fees = new BitcoinFees(Stub(BitcoinMultistream), reader, 3) + when: + def act = fees.calculateFee(tx4D5).block(Duration.ofSeconds(1)) + then: + act == 200000 - (29163 + 144221) + } + + def "calculates fee with multiple inputs"() { + setup: + def reader = Mock(BitcoinReader) { + 1 * it.getTx("37e19fad6c3e5fafc431ee731428452dc01f56b0bc4d9fe876dcee58384934ec") >> Mono.just(tx37E) + 1 * it.getTx("3da079b10451ba1cde42d9a0f485e79151e5d4216930561227be5471921af17f") >> Mono.just(tx3DA) + } + def fees = new BitcoinFees(Stub(BitcoinMultistream), reader, 3) + when: + def act = fees.calculateFee(txF29).block(Duration.ofSeconds(1)) + then: + act == (100000000 + 21000000) - (120100000 + 210984) + } + + def "get average bottom fee"() { + setup: + def ups = Mock(BitcoinMultistream) { + _ * getHead() >> Mock(Head) { + _ * getCurrentHeight() >> 100 + } + } + def reader = Mock(BitcoinReader) { + 1 * it.getBlock(100) >> Mono.just(block3) + 1 * it.getBlock(99) >> Mono.just(block2) + // last txes on those blocks + 1 * it.getTx("4d56b1a0992a3fa3132083ce29e203165ef4768f028d98a738da79dd26ff91e2") >> Mono.just(tx4D5) + 1 * it.getTx("f29ecfa5b692cfc46b1eae29416a35782657f0bde2ae44acde882e477e8624d8") >> Mono.just(txF29) + // their inputs + 1 * it.getTx("3bde80635c62e81ba86d68a1c78ac47b1a02d8ef334fdf6a7b6e8acc0a475f20") >> Mono.just(tx3BD) + 1 * it.getTx("37e19fad6c3e5fafc431ee731428452dc01f56b0bc4d9fe876dcee58384934ec") >> Mono.just(tx37E) + 1 * it.getTx("3da079b10451ba1cde42d9a0f485e79151e5d4216930561227be5471921af17f") >> Mono.just(tx3DA) + } + def fees = new BitcoinFees(ups, reader, 3) + + when: + def act = fees.estimate(ChainFees.Mode.AVG_LAST, 2).block(Duration.ofSeconds(1)) + + then: + act.hasBitcoinStd() + // first tx fee: 187.43661971830985915493 + // second tx fee: 504.40409956076134699854 + // average is 345 + // but if we calculate original fees per KB it's 354222 + act.bitcoinStd.satPerKb == "354222" + } + +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReaderSpec.groovy new file mode 100644 index 00000000..5e65cd5f --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReaderSpec.groovy @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.test.ApiReaderMock +import io.emeraldpay.dshackle.upstream.Head +import reactor.core.publisher.Mono +import spock.lang.Specification + +import java.time.Duration + +class BitcoinReaderSpec extends Specification { + + def "gets block by height"() { + setup: + def block = [ + hash: "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506", + tx : [] + ] + def api = new ApiReaderMock() + api.answerOnce("getblockhash", [100000], "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506") + api.answerOnce("getblock", ["000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506"], block) + def ups = Mock(BitcoinMultistream) { + _ * it.getDirectApi(_) >> Mono.just(api) + } + def reader = new BitcoinReader(ups, Stub(Head), null) + + when: + def act = reader.getBlock(100000).block(Duration.ofSeconds(1)) + + then: + act == block + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy new file mode 100644 index 00000000..e8695a35 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLegacyFeesSpec.groovy @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.etherjar.domain.Wei +import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson +import spock.lang.Specification + +class EthereumLegacyFeesSpec extends Specification { + + def "Extract fee from"() { + setup: + def block = new BlockJson() + // 0x75cc01873a9818bf426a8b23d83450bf18530a822fd4fe9e86a416a5554176a6 + def tx = new TransactionJson().tap { + it.gasPrice = Wei.ofUnits(8, Wei.Unit.GWEI) + } + + def fees = new EthereumLegacyFees(Stub(EthereumMultistream), Stub(EthereumReader), 10) + when: + def act = fees.extractFee(block, tx) + then: + act.priority == Wei.ofUnits(8, Wei.Unit.GWEI) + act.max == Wei.ofUnits(8, Wei.Unit.GWEI) + act.base == Wei.ZERO + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy new file mode 100644 index 00000000..b1638242 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFeesSpec.groovy @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.ChainFees +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.etherjar.domain.TransactionId +import io.emeraldpay.etherjar.domain.Wei +import io.emeraldpay.etherjar.rpc.json.BlockJson +import io.emeraldpay.etherjar.rpc.json.TransactionJson +import io.emeraldpay.etherjar.rpc.json.TransactionRefJson +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import spock.lang.Specification + +import java.time.Duration + +class EthereumPriorityFeesSpec extends Specification { + + def "Extract fee from EIP1559 tx"() { + setup: + // 13756007 + def block = new BlockJson().tap { + it.baseFeePerGas = new Wei(104197355513) + } + // 0x5da50f35a51e56ecd4313417b1c30f9c088222f3f8763701effe14f3dd18b6cc + def tx = new TransactionJson().tap { + it.type = 2 + it.maxFeePerGas = Wei.ofUnits(999, Wei.Unit.GWEI) + it.maxPriorityFeePerGas = Wei.ofUnits(5.0001, Wei.Unit.GWEI) + } + + def fees = new EthereumPriorityFees(Stub(EthereumMultistream), Stub(EthereumReader), 10) + when: + def act = fees.extractFee(block, tx) + then: + act.priority == Wei.ofUnits(5.0001, Wei.Unit.GWEI) + act.max == Wei.ofUnits(999, Wei.Unit.GWEI) + act.base == new Wei(104197355513) + } + + def "Extract fee from legacy tx"() { + setup: + // 13756007 + def block = new BlockJson().tap { + it.baseFeePerGas = new Wei(104197355513) + } + // 0x1f507982bef0f11a8304287d41f228b5f1dda1114a446ee781c3d95ef4a7b891 + def tx = new TransactionJson().tap { + it.type = 0 + // 109.564020111 Gwei + it.gasPrice = Wei.from("0x198286458f") + } + + def fees = new EthereumPriorityFees(Stub(EthereumMultistream), Stub(EthereumReader), 10) + when: + def act = fees.extractFee(block, tx) + then: + // as difference between base minimum and actually paid + act.priority == Wei.ofUnits(5.366664598, Wei.Unit.GWEI) + act.max == Wei.from("0x198286458f") + act.base == new Wei(104197355513) + } + + def "Calculates average fee"() { + setup: + def inputs = [ + new EthereumFees.EthereumFee(Wei.ofEthers(1), Wei.ofEthers(0.5), Wei.ofEthers(0.75), Wei.ofEthers(0.5)), + new EthereumFees.EthereumFee(Wei.ofEthers(0.75), Wei.ofEthers(0.5), Wei.ofEthers(0.75), Wei.ofEthers(0.5)), + new EthereumFees.EthereumFee(Wei.ofEthers(0.6), Wei.ofEthers(0.2), Wei.ofEthers(0.6), Wei.ofEthers(0.5)), + ] + def fees = new EthereumPriorityFees(Stub(EthereumMultistream), Stub(EthereumReader), 10) + when: + def act = Flux.fromIterable(inputs) + .transform(fees.feeAggregation(ChainFees.Mode.AVG_LAST)) + .next().block(Duration.ofSeconds(1)) + then: + act.priority == Wei.ofEthers(0.4) // 0.5 + 0.5 + 0.2 + act.paid == Wei.ofEthers(0.7) // 0.75 + 0.75 + 0.6 + } + + def "Estimate"() { + setup: + def block1 = new BlockJson().tap { + it.baseFeePerGas = new Wei(92633661632) + it.transactions = [ + new TransactionRefJson(TransactionId.from("0x00000000fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), + new TransactionRefJson(TransactionId.from("0x11111111fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), + new TransactionRefJson(TransactionId.from("0x22222222fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), + ] + } + def block2 = new BlockJson().tap { + it.baseFeePerGas = new Wei(104197355513) + it.transactions = [ + new TransactionRefJson(TransactionId.from("0x33333333fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), + new TransactionRefJson(TransactionId.from("0x44444444fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), + new TransactionRefJson(TransactionId.from("0x55555555fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")), + ] + } + def tx1 = new TransactionJson().tap { + it.type = 2 + it.maxFeePerGas = Wei.ofUnits(150, Wei.Unit.GWEI) + it.maxPriorityFeePerGas = Wei.ofUnits(3, Wei.Unit.GWEI) + } + def tx2 = new TransactionJson().tap { + it.type = 2 + it.maxFeePerGas = Wei.ofUnits(200, Wei.Unit.GWEI) + it.maxPriorityFeePerGas = Wei.ofUnits(6, Wei.Unit.GWEI) + } + + def ups = Mock(EthereumMultistream) { + 1 * getHead() >> Mock(Head) { + 1 * getCurrentHeight() >> 13756007 + } + } + def reader = Mock(EthereumReader) { + _ * it.blocksByHeightParsed() >> Mock(Reader) { + 1 * it.read(13756006) >> Mono.just(block1) + 1 * it.read(13756007) >> Mono.just(block2) + } + _ * it.txByHash() >> Mock(Reader) { + 1 * it.read(TransactionId.from("0x22222222fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")) >> Mono.just(tx1) + 1 * it.read(TransactionId.from("0x55555555fad596cad644b785a8a74f6580ceec9ae13c8aa174f819c0223b8c77")) >> Mono.just(tx2) + } + } + def fees = new EthereumPriorityFees(ups, reader, 10) + when: + def act = fees.estimate(ChainFees.Mode.AVG_LAST, 2).block(Duration.ofSeconds(1)) + + then: + act.hasEthereumExtended() + act.ethereumExtended.priority == "4500000000" + act.ethereumExtended.max == "175000000000" + act.ethereumExtended.expect == "102915508572" + } +}