From 56318d800e792bb84093d339bd5d163e718af687 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Mon, 14 Sep 2020 23:08:13 -0400 Subject: [PATCH 1/4] solution: esplora upstream connection for bitcoin --- .../kotlin/io/emeraldpay/dshackle/Global.kt | 9 +- .../dshackle/config/UpstreamsConfig.kt | 1 + .../dshackle/config/UpstreamsConfigReader.kt | 8 + .../dshackle/rpc/TrackBitcoinAddress.kt | 72 ++++---- .../dshackle/startup/ConfiguredUpstreams.kt | 12 +- .../dshackle/upstream/Capability.kt | 6 + .../dshackle/upstream/Multistream.kt | 8 +- .../upstream/bitcoin/BitcoinMultistream.kt | 18 +- .../upstream/bitcoin/BitcoinReader.kt | 15 +- .../upstream/bitcoin/BitcoinRpcUpstream.kt | 5 +- .../upstream/bitcoin/BitcoinUpstream.kt | 3 +- .../upstream/bitcoin/EsploraClient.kt | 104 ++++++++++++ .../upstream/bitcoin/EsploraUnspentReader.kt | 53 ++++++ .../upstream/bitcoin/RpcUnspentReader.kt | 61 +++++++ .../upstream/bitcoin/UnspentReader.kt | 23 +++ .../upstream/bitcoin/data/EsploraUnspent.kt | 26 +++ .../data/EsploraUnspentDeserializer.kt | 36 ++++ .../upstream/bitcoin/data/RpcUnspent.kt | 24 +++ .../bitcoin/data/RpcUnspentDeserializer.kt | 37 +++++ .../upstream/bitcoin/data/SimpleUnspent.kt | 23 +++ .../config/UpstreamsConfigReaderSpec.groovy | 55 ++++++ .../rpc/TrackBitcoinAddressSpec.groovy | 97 +++-------- .../upstream/bitcoin/EsploraClientSpec.groovy | 83 ++++++++++ .../bitcoin/RpcUnspentReaderSpec.groovy | 135 +++++++++++++++ .../rpcclient/JsonRpcHttpClientSpec.groovy | 9 +- .../resources/bitcoin/esplora-utxo-1.json | 156 ++++++++++++++++++ .../resources/upstreams-bitcoin-esplora.yaml | 17 ++ src/test/resources/upstreams-bitcoin.yaml | 2 +- 28 files changed, 962 insertions(+), 136 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy create mode 100644 src/test/resources/bitcoin/esplora-utxo-1.json create mode 100644 src/test/resources/upstreams-bitcoin-esplora.yaml diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index ed34ecad..89b12488 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -19,9 +19,11 @@ import com.fasterxml.jackson.core.Version import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.module.SimpleModule +import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent +import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspentDeserializer +import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent +import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.infinitape.etherjar.rpc.json.TransactionReceiptJson -import io.infinitape.etherjar.rpc.json.TransactionReceiptJsonDeserializer import java.text.SimpleDateFormat import java.util.* @@ -36,6 +38,9 @@ class Global { val module = SimpleModule("EmeraldDshackle", Version(1, 0, 0, null, null, null)) module.addSerializer(JsonRpcResponse::class.java, JsonRpcResponse.ResponseJsonSerializer()) + module.addDeserializer(EsploraUnspent::class.java, EsploraUnspentDeserializer()) + module.addDeserializer(RpcUnspent::class.java, RpcUnspentDeserializer()) + val objectMapper = ObjectMapper() objectMapper.registerModule(module) objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt index da6f74d8..70881287 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt @@ -107,6 +107,7 @@ class UpstreamsConfig { } class BitcoinConnection : RpcConnection() { + var esplora: HttpEndpoint? = null } class HttpEndpoint(val url: URI) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 145d9bca..8ded3edb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -132,6 +132,14 @@ class UpstreamsConfigReader( http.tls = authConfigReader.readClientTls(node) } } + getMapping(connConfigNode, "esplora")?.let { node -> + getValueAsString(node, "url")?.let { url -> + val http = UpstreamsConfig.HttpEndpoint(URI(url)) + http.basicAuth = authConfigReader.readClientBasicAuth(node) + http.tls = authConfigReader.readClientTls(node) + connection.esplora = http + } + } } else { log.error("Upstream at #0 has invalid configuration") } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt index 3ac705b5..268f5fe5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt @@ -21,15 +21,16 @@ import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream +import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.grpc.Chain +import org.bitcoinj.params.MainNetParams +import org.bitcoinj.params.TestNet3Params import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Flux -import reactor.util.function.Tuples -import java.math.BigDecimal +import reactor.core.publisher.Mono import java.math.BigInteger -import java.util.* import kotlin.collections.HashMap @Service @@ -64,10 +65,18 @@ class TrackBitcoinAddress( } fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: List): Flux { - return api.getReader().listUnspent() - .flatMapMany { unspents -> - val result = getTotal(chain, addresses, unspents) - Flux.fromIterable(result) + return Flux.fromIterable(addresses) + .map { Address(chain, it) } + .flatMap { address -> + api.getReader() + .listUnspent(address.bitcoinAddress) + .map { unspents -> + getTotal(address, unspents) + } + .onErrorResume { t -> + log.error("Failed to get unspent", t) + Mono.empty() + } } } @@ -83,37 +92,13 @@ class TrackBitcoinAddress( .map(this@TrackBitcoinAddress::buildResponse) } - fun getTotal(chain: Chain, addresses: List, unspents: List<*>): List { - return unspents.asSequence() - .filterIsInstance>() - .filter { unspent -> - unspent.containsKey("address") - && Collections.binarySearch(addresses, unspent["address"] as String) >= 0 - && unspent.containsKey("amount") - } - .map { - Tuples.of(it["address"] as String, it["amount"] as Number) - } - .map { - AddressBalance(chain, it.t1, - //use toString because toDecimal makes rounding - BigDecimal(it.t2.toString()).multiply(BigDecimal.TEN.pow(8)).toBigInteger() - ) - } - .plus( - //add default ZERO value - addresses.map { - AddressBalance(chain, it, BigInteger.ZERO) - } - ) - .groupBy { - it.address - } - .map { - it.value.reduceRight { x, acc -> - x.plus(acc) - } - }.toList() + fun getTotal(address: Address, unspents: List): AddressBalance { + val total = if (unspents.isEmpty()) { + 0L + } else { + unspents.map { it.value }.reduce(Long::plus) + } + return AddressBalance(address, BigInteger.valueOf(total)) } @@ -158,5 +143,14 @@ class TrackBitcoinAddress( fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance) } - data class Address(val chain: Chain, val address: String) + class Address(val chain: Chain, val address: String) { + val network = if (chain == Chain.BITCOIN) { + MainNetParams() + } else { + TestNet3Params() + } + val bitcoinAddress = org.bitcoinj.core.Address.fromString( + network, address + ) + } } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index 0ce260cd..e3f54418 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.CurrentMultistreamHolder import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream +import io.emeraldpay.dshackle.upstream.bitcoin.EsploraClient import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream @@ -142,12 +143,21 @@ open class ConfiguredUpstreams( return } + val esplora = conn.esplora?.let { endpoint -> + val tls = endpoint.tls?.let { tls -> + tls.ca?.let { ca -> + fileResolver.resolve(ca).readBytes() + } + } + EsploraClient(endpoint.url, endpoint.basicAuth, tls) + } + val methods = buildMethods(config, chain) val upstream = BitcoinRpcUpstream(config.id ?: "bitcoin-${seq.getAndIncrement()}", chain, directApi, options, config.role, QuorumForLabels.QuorumItem(1, config.labels), - methods) + methods, esplora) upstream.start() currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED)) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt new file mode 100644 index 00000000..9805b4b2 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt @@ -0,0 +1,6 @@ +package io.emeraldpay.dshackle.upstream + +enum class Capability { + RPC, + BALANCE +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt index 9bc0e389..96b03e8e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt @@ -78,14 +78,14 @@ abstract class Multistream( */ fun addUpstream(upstream: Upstream) { upstreams.add(upstream) - setHead(updateHead()) onUpstreamsUpdated() + setHead(updateHead()) } fun removeUpstream(id: String) { if (upstreams.removeIf { it.getId() == id }) { - setHead(updateHead()) onUpstreamsUpdated() + setHead(updateHead()) } } @@ -103,7 +103,7 @@ abstract class Multistream( /** * Finds an API that executed directly on a remote. */ - fun getDirectApi(matcher: Selector.Matcher): Mono> { + open fun getDirectApi(matcher: Selector.Matcher): Mono> { val apis = getApiSource(matcher) apis.request(1) return Mono.from(apis) @@ -120,7 +120,7 @@ abstract class Multistream( throw NotImplementedError("Immediate direct API is not implemented for Aggregated Upstream") } - fun onUpstreamsUpdated() { + open fun onUpstreamsUpdated() { reconfigLock.withLock { getAll().map { it.getMethods() }.let { //TODO made list of uniq instances, and then if only one, just use it directly diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt index 8f647089..b5e1eace 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt @@ -15,7 +15,6 @@ */ package io.emeraldpay.dshackle.upstream.bitcoin -import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.EmptyReader @@ -38,8 +37,9 @@ open class BitcoinMultistream( private val log = LoggerFactory.getLogger(BitcoinMultistream::class.java) } - private var head: Head? = null - private var reader = BitcoinReader(this, EmptyHead()) + private var head: Head = EmptyHead() + private var esplora = upstreams.find { it.esploraClient != null }?.esploraClient + private var reader = BitcoinReader(this, head, esplora) override fun init() { if (upstreams.size > 0) { @@ -49,7 +49,7 @@ open class BitcoinMultistream( } override fun updateHead(): Head { - head?.let { + head.let { if (it is Lifecycle) { it.stop() } @@ -81,13 +81,19 @@ open class BitcoinMultistream( return reader } + override fun onUpstreamsUpdated() { + super.onUpstreamsUpdated() + esplora = upstreams.find { it.esploraClient != null }?.esploraClient + reader = BitcoinReader(this, this.head, esplora) + } + override fun setHead(head: Head) { this.head = head - reader = BitcoinReader(this, head) + reader = BitcoinReader(this, head, esplora) } override fun getHead(): Head { - return head!! + return head } override fun getLabels(): Collection { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt index e0b58aeb..c272fe9b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt @@ -19,8 +19,10 @@ import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import org.bitcoinj.core.Address import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import reactor.core.publisher.Mono @@ -28,7 +30,8 @@ import reactor.kotlin.core.publisher.cast open class BitcoinReader( private val upstreams: BitcoinMultistream, - head: Head + head: Head, + esploraClient: EsploraClient? ) : Lifecycle { companion object { @@ -38,6 +41,12 @@ open class BitcoinReader( private val objectMapper: ObjectMapper = Global.objectMapper private val mempool = CachingMempoolData(upstreams, head) + private val unspentReader: UnspentReader = if (esploraClient != null) { + EsploraUnspentReader(esploraClient, head) + } else { + RpcUnspentReader(upstreams) + } + open fun getMempool(): CachingMempoolData { return mempool } @@ -50,8 +59,8 @@ open class BitcoinReader( return castedRead(JsonRpcRequest("getrawtransaction", listOf(txid, true)), Map::class.java).cast() } - open fun listUnspent(): Mono> { - return castedRead(JsonRpcRequest("listunspent", emptyList()), List::class.java).cast() + open fun listUnspent(address: Address): Mono> { + return unspentReader.read(address) } override fun isRunning(): Boolean { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt index 19e69226..d875970e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt @@ -34,8 +34,9 @@ open class BitcoinRpcUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, node: QuorumForLabels.QuorumItem, - callMethods: CallMethods -) : BitcoinUpstream(id, chain, options, role, callMethods, node), Lifecycle { + callMethods: CallMethods, + esploraClient: EsploraClient? = null +) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient), Lifecycle { companion object { private val log = LoggerFactory.getLogger(BitcoinRpcUpstream::class.java) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt index 22929246..9325b37f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt @@ -29,7 +29,8 @@ abstract class BitcoinUpstream( options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, callMethods: CallMethods, - node: QuorumForLabels.QuorumItem + node: QuorumForLabels.QuorumItem, + val esploraClient: EsploraClient? = null ) : DefaultUpstream(id, options, role, callMethods, node) { constructor(id: String, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt new file mode 100644 index 00000000..74ab7bfb --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.AuthConfig +import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaders +import io.netty.handler.ssl.SslContextBuilder +import org.bitcoinj.core.Address +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono +import reactor.netty.http.client.HttpClient +import java.io.ByteArrayInputStream +import java.net.URI +import java.security.KeyStore +import java.security.cert.CertificateFactory +import java.security.cert.X509Certificate +import java.util.* +import java.util.function.Consumer + +class EsploraClient( + private val url: URI, + basicAuth: AuthConfig.ClientBasicAuth? = null, + tlsCAAuth: ByteArray? = null +) { + + companion object { + private val log = LoggerFactory.getLogger(EsploraClient::class.java) + } + + private val httpClient: HttpClient + + init { + var build = HttpClient.create() + + build = build.headers { h -> + h.add(HttpHeaderNames.CONTENT_TYPE, "application/json") + } + + basicAuth?.let { auth -> + val authString: String = auth.username + ":" + auth.password + val authBase64 = Base64.getEncoder().encodeToString(authString.toByteArray()) + val encodedAuth = "Basic $authBase64" + val headers = Consumer { h: HttpHeaders -> h.add(HttpHeaderNames.AUTHORIZATION, encodedAuth) } + build = build.headers(headers) + } + + tlsCAAuth?.let { auth -> + val cf = CertificateFactory.getInstance("X.509") + val cert = cf.generateCertificate(ByteArrayInputStream(auth)) as X509Certificate + val ks = KeyStore.getInstance(KeyStore.getDefaultType()) + ks.load(null, "".toCharArray()) + ks.setCertificateEntry("server", cert) + val sslContext = SslContextBuilder.forClient().trustManager(cert).build() + + build.secure { spec -> + spec.sslContext(sslContext) + } + } + + this.httpClient = build + } + + fun getUtxo(address: Address): Mono> { + val response = httpClient + .get() + .uri("$url/address/$address/utxo") + + val json = response + .response { header, bytes -> + if (header.status().code() != 200) { + Mono.error(EsploraException("HTTP Code: ${header.status().code()}")) + } else { + bytes.aggregate().asByteArray() + } + } + .single() + .map { bytes -> String(bytes) } + + return json.map { + val parsed = Global.objectMapper.readerFor(EsploraUnspent::class.java) + .readValues(it); + parsed.readAll() + } + } + + class EsploraException(msg: String) : Exception(msg) + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt new file mode 100644 index 00000000..ba5e54f6 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent +import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent +import org.bitcoinj.core.Address +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono +import java.util.function.Function + +class EsploraUnspentReader( + private val esploraClient: EsploraClient, + private val head: Head +) : UnspentReader { + + companion object { + private val log = LoggerFactory.getLogger(EsploraUnspentReader::class.java) + } + + private val convert: (T: EsploraUnspent) -> SimpleUnspent = { base -> + SimpleUnspent( + base.txid, + base.vout, + base.value, + head.getCurrentHeight()?.let { base.height - it } ?: 0 + ) + } + + private val convertAll: Function, List> = Function { base -> + base.map(convert) + } + + override fun read(key: Address): Mono> { + return esploraClient.getUtxo(key) + .map(convertAll) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt new file mode 100644 index 00000000..92db7f86 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent +import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import org.bitcoinj.core.Address +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono + +class RpcUnspentReader( + private val upstreams: BitcoinMultistream +) : UnspentReader { + + companion object { + private val log = LoggerFactory.getLogger(RpcUnspentReader::class.java) + } + + private val convert: (T: RpcUnspent) -> SimpleUnspent = { base -> + SimpleUnspent( + base.txid, + base.vout, + base.amount, + base.confirmations + ) + } + + override fun read(key: Address): Mono> { + val address = key.toString() + return upstreams.getDirectApi(Selector.empty).flatMap { api -> + api.read(JsonRpcRequest("listunspent", emptyList())) + .flatMap(JsonRpcResponse::requireResult) + .map { + Global.objectMapper.readerFor(RpcUnspent::class.java).readValues(it).readAll() + } + .map { + it.filter { + it.address == address + }.map(convert) + } + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt new file mode 100644 index 00000000..42f4fbde --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent +import org.bitcoinj.core.Address + +interface UnspentReader : Reader> { +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt new file mode 100644 index 00000000..7e91cb39 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin.data + +import java.time.Instant + +data class EsploraUnspent( + val txid: String, + val vout: Int, + val value: Long, + val timestamp: Instant, + val height: Long +) \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt new file mode 100644 index 00000000..5e3abe5c --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin.data + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonNode +import java.time.Instant + +class EsploraUnspentDeserializer : JsonDeserializer() { + override fun deserialize(jp: JsonParser, ctxt: DeserializationContext): EsploraUnspent { + val node: JsonNode = jp.readValueAsTree() + val status = node.get("status") + return EsploraUnspent( + node.get("txid").asText(), + node.get("vout").asInt(), + node.get("value").asLong(), + Instant.ofEpochSecond(status.get("block_time").asLong()), + status.get("block_height").asLong() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt new file mode 100644 index 00000000..63700ef6 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin.data + +data class RpcUnspent( + val txid: String, + val vout: Int, + val address: String, + val amount: Long, + val confirmations: Long +) \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt new file mode 100644 index 00000000..cd63b6ed --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin.data + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonNode +import java.math.BigDecimal + +class RpcUnspentDeserializer : JsonDeserializer() { + + override fun deserialize(jp: JsonParser, ctxt: DeserializationContext): RpcUnspent { + val node: JsonNode = jp.readValueAsTree() + return RpcUnspent( + node.get("txid").asText(), + node.get("vout").asInt(), + node.get("address").asText(), + BigDecimal(node.get("amount").asText()).multiply(BigDecimal.TEN.pow(8)).longValueExact(), + node.get("confirmations").asLong() + ) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt new file mode 100644 index 00000000..afc9e9db --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin.data + +data class SimpleUnspent( + val txid: String, + val vout: Int, + val value: Long, + val confirmations: Long +) \ No newline at end of file diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy index f2ed56ee..cc06cc8c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy @@ -74,6 +74,61 @@ class UpstreamsConfigReaderSpec extends Specification { } } + def "Parse bitcoin upstreams"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("upstreams-bitcoin.yaml") + when: + def act = reader.read(config) + then: + act != null + with(act.defaultOptions) { + size() == 1 + with(get(0)) { + chains == ["bitcoin"] + options.minPeers == 3 + } + } + act.upstreams.size() == 1 + with(act.upstreams.get(0)) { + id == "local" + chain == "bitcoin" + connection instanceof UpstreamsConfig.BitcoinConnection + with((UpstreamsConfig.BitcoinConnection) connection) { + rpc != null + rpc.url == new URI("http://localhost:8545") + esplora == null + } + } + } + + def "Parse bitcoin upstreams with esplora"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("upstreams-bitcoin-esplora.yaml") + when: + def act = reader.read(config) + then: + act != null + with(act.defaultOptions) { + size() == 1 + with(get(0)) { + chains == ["bitcoin"] + options.minPeers == 3 + } + } + act.upstreams.size() == 1 + with(act.upstreams.get(0)) { + id == "local" + chain == "bitcoin" + connection instanceof UpstreamsConfig.BitcoinConnection + with((UpstreamsConfig.BitcoinConnection) connection) { + rpc != null + rpc.url == new URI("http://localhost:8545") + esplora != null + esplora.url == new URI("http://localhost:3001") + } + } + } + def "Parse ds config"() { setup: def config = this.class.getClassLoader().getResourceAsStream("upstreams-ds.yaml") diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy index 842a9145..14513dd9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy @@ -27,6 +27,7 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinReader +import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.grpc.Chain import reactor.core.publisher.Flux import reactor.core.publisher.Mono @@ -42,89 +43,37 @@ class TrackBitcoinAddressSpec extends Specification { String hash1 = "0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22" ObjectMapper objectMapper = Global.objectMapper - def "Correct sum from multiple"() { + def "Correct sum for single"() { setup: - def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-one-addr.json") - def unspents = objectMapper.readValue(json, List) + def unspents = [ + new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 100L, 123L) + ] TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) + def address = new TrackBitcoinAddress.Address( + Chain.BITCOIN, "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" + ) when: - def total = track.getTotal(Chain.BITCOIN, ["1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK"], unspents) + def total = track.getTotal(address, unspents) then: - total.size() == 1 - total[0].address.chain == Chain.BITCOIN - total[0].address.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" - total[0].balance.toString() == "32928461" + total.balance == 100 } - def "Correct sum when other addresses"() { + def "Correct sum for few"() { setup: - def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-two-addr.json") - def unspents = objectMapper.readValue(json, List) + def unspents = [ + new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 100L, 123L), + new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 123L, 123L), + ] TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) + def address = new TrackBitcoinAddress.Address( + Chain.BITCOIN, "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" + ) when: - def total = track.getTotal(Chain.BITCOIN, ["1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK"], unspents) + def total = track.getTotal(address, unspents) then: - total.size() == 1 - total[0].address.chain == Chain.BITCOIN - total[0].address.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" - total[0].balance.toString() == "32928461" - } - - def "Sum for two addresses"() { - setup: - def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-two-addr.json") - def unspents = objectMapper.readValue(json, List) - TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) - when: - def total = track.getTotal(Chain.BITCOIN, ["1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK", "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP"], unspents).sort { it.address.address } - - then: - total.size() == 2 - with(total[0]) { - address.chain == Chain.BITCOIN - address.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" - balance.toString() == "32928461" - } - with(total[1]) { - address.chain == Chain.BITCOIN - address.address == "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP" - balance.toString() == "25550215615737" - } - } - - def "Zero for empty unspents"() { - setup: - TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) - when: - def total = track.getTotal(Chain.BITCOIN, ["1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK"], []) - - then: - total.size() == 1 - total[0].address.chain == Chain.BITCOIN - total[0].address.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" - total[0].balance.toString() == "0" - } - - def "Zero for unknown address"() { - setup: - def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-two-addr.json") - def unspents = objectMapper.readValue(json, List) - TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) - when: - def total = track.getTotal(Chain.BITCOIN, ["16rCmCmbuWDhPjWTrpQGaU3EPdZF7MTdUk", "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK"], unspents).sort { it.address.address } - - then: - total.size() == 2 - with(total[0]) { - address.address == "16rCmCmbuWDhPjWTrpQGaU3EPdZF7MTdUk" - balance.toString() == "0" - } - with(total[1]) { - address.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" - balance.toString() == "32928461" - } + total.balance == 223 } def "One address for single provided"() { @@ -226,9 +175,11 @@ class TrackBitcoinAddressSpec extends Specification { def upstream = null upstream = Mock(BitcoinMultistream) { _ * getReader() >> Mock(BitcoinReader) { - 2 * listUnspent() >>> [ + 2 * listUnspent(_) >>> [ Mono.just([]), - Mono.just([[address: "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK", amount: 0.0123]]) + Mono.just([ + new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 1230000L, 123L) + ]) ] } _ * getHead() >> head diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy new file mode 100644 index 00000000..5b046841 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy @@ -0,0 +1,83 @@ +package io.emeraldpay.dshackle.upstream.bitcoin + +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent +import org.bitcoinj.core.Address +import org.bitcoinj.params.MainNetParams +import org.mockserver.integration.ClientAndServer +import org.mockserver.model.HttpRequest +import org.mockserver.model.HttpResponse +import reactor.test.StepVerifier +import spock.lang.Specification + +import java.time.Duration +import java.time.Instant + +class EsploraClientSpec extends Specification { + + ClientAndServer mockServer + + def setup() { + mockServer = ClientAndServer.startClientAndServer(23001); + } + + def cleanup() { + mockServer.stop() + } + + def "get utxo"() { + setup: + def responseJson = this.class.getClassLoader().getResourceAsStream("bitcoin/esplora-utxo-1.json").text + mockServer.when( + HttpRequest.request() + .withMethod("GET") + .withPath("/address/35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY/utxo") + ).respond( + HttpResponse.response(responseJson) + ) + def client = new EsploraClient(new URI("http://localhost:23001"), null, null) + when: + def act = client.getUtxo(Address.fromString(new MainNetParams(), "35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY")) + + then: + StepVerifier.create(act) + .expectNextMatches { list -> + def ok = list.size() == 14 + if (!ok) println("invalid size") + ok = ok && list[0] == new EsploraUnspent("002eba7d9e8081afc687e1c3fa7b6a6451713b75bc91432d441bb1e7e1511c5c", 0, 105524, Instant.ofEpochSecond(1599808442), 647721) + if (!ok) println("invalid 0") + ok = ok && list[1] == new EsploraUnspent("1ee3cb3833b1e499e7f8babb1100c4aa0b4273f655036561d84dd72a5b197258", 0, 5248, Instant.ofEpochSecond(1600114632), 648312) + if (!ok) println("invalid 1") + ok = ok && list[12] == new EsploraUnspent("311d0d1d4eea6ee37d57954cd1002898bef64885d8e438afbbd7fe4fdc6e08de", 2250, 22978, Instant.ofEpochSecond(1599018027), 646382) + if (!ok) println("invalid 12") + ok = ok && list[13] == new EsploraUnspent("230317b8fdf1ae85f5fbdc49ca90851b1728c2d9432b2738d3fe1c6f68f046e4", 11, 8642, Instant.ofEpochSecond(1599273442), 646777) + if (!ok) println("invalid 13") + ok + } + .expectComplete() + .verify(Duration.ofSeconds(3)) + + when: + def actTotal = client.getUtxo(Address.fromString(new MainNetParams(), "35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY")) + .block() + .sum { it.value } + + then: + actTotal == 309841L + } + +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy new file mode 100644 index 00000000..f65bc832 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy @@ -0,0 +1,135 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import org.bitcoinj.core.Address +import org.bitcoinj.params.MainNetParams +import reactor.core.publisher.Mono +import spock.lang.Specification + +class RpcUnspentReaderSpec extends Specification { + + def "all if single address"() { + setup: + def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-one-addr.json").bytes + def rpcReader = Mock(Reader) { + 1 * read(new JsonRpcRequest("listunspent", [])) >> Mono.just(JsonRpcResponse.ok(json)) + } + def upstreams = Mock(BitcoinMultistream) { + 1 * getDirectApi(_) >> Mono.just(rpcReader) + } + def reader = new RpcUnspentReader(upstreams) + + when: + def act = reader.read(Address.fromString(new MainNetParams(), "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")).block() + + then: + // cat src/test/resources/bitcoin/unspent-one-addr.json | jq '. | length' + act.size() == 36 + with(act[0]) { + txid == "e0f946c8f971b25cdffa64eed71d886019e437c0bf6a1b280584c0be5d1b5409" + vout == 29 + confirmations == 2010 + value == 1230030 + } + with(act[1]) { + txid == "66e1e4d14ed6f454d2fda036f35cba423274ecdf5d46deb93f172c412a0f650d" + vout == 83 + confirmations == 4963 + value == 756339 + } + with(act[35]) { + txid == "f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef" + vout == 58 + confirmations == 2139 + value == 1105047 + } + } + + def "select if two addresses - 35hK"() { + setup: + def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-two-addr.json").bytes + def rpcReader = Mock(Reader) { + 1 * read(new JsonRpcRequest("listunspent", [])) >> Mono.just(JsonRpcResponse.ok(json)) + } + def upstreams = Mock(BitcoinMultistream) { + 1 * getDirectApi(_) >> Mono.just(rpcReader) + } + def reader = new RpcUnspentReader(upstreams) + + when: + def act = reader.read(Address.fromString(new MainNetParams(), "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP")).block() + + then: + // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP")] | length' + act.size() == 340 + with(act[0]) { + txid == "8ad0d954a01eeb4f2c62d58d291699af847f9c8df43b775c27ffe8a5f76eba00" + vout == 1 + value == 216465 + confirmations == 2583 + } + with(act[11]) { + txid == "777671a46b30b068052a73387e036bc8515cd3ba6adf9be4c70dfc0699f67c09" + vout == 0 + confirmations == 13705 + value == 307906 + } + // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP")] | .[211]' + with(act[211]) { + txid == "f20727393b0a586a3062a615fb71f43ec21c24258c3c6ec546fee5cbc1fa2ba7" + vout == 0 + confirmations == 21890 + value == 50000000000 + } + } + + def "select if two addresses - 1K7x"() { + setup: + def json = this.class.getClassLoader().getResourceAsStream("bitcoin/unspent-two-addr.json").bytes + def rpcReader = Mock(Reader) { + 1 * read(new JsonRpcRequest("listunspent", [])) >> Mono.just(JsonRpcResponse.ok(json)) + } + def upstreams = Mock(BitcoinMultistream) { + 1 * getDirectApi(_) >> Mono.just(rpcReader) + } + def reader = new RpcUnspentReader(upstreams) + + when: + def act = reader.read(Address.fromString(new MainNetParams(), "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")).block() + + then: + // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")] | length' + act.size() == 36 + // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")] | .[0]' + with(act[0]) { + txid == "e0f946c8f971b25cdffa64eed71d886019e437c0bf6a1b280584c0be5d1b5409" + vout == 29 + value == 1230030 + confirmations == 2030 + } + // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")] | .[35]' + with(act[35]) { + txid == "f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef" + vout == 58 + confirmations == 2159 + value == 1105047 + } + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClientSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClientSpec.groovy index 6b8126b6..48acdc87 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClientSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClientSpec.groovy @@ -31,9 +31,10 @@ import java.time.Duration class JsonRpcHttpClientSpec extends Specification { ClientAndServer mockServer + int port = 19332 def setup() { - mockServer = ClientAndServer.startClientAndServer(18332); + mockServer = ClientAndServer.startClientAndServer(port); } def cleanup() { @@ -42,7 +43,7 @@ class JsonRpcHttpClientSpec extends Specification { def "Make a request"() { setup: - JsonRpcHttpClient client = new JsonRpcHttpClient("localhost:18332", null, null) + JsonRpcHttpClient client = new JsonRpcHttpClient("localhost:${port}", null, null) def resp = '{' + ' "jsonrpc": "2.0",' + ' "result": "0x98de45",' + @@ -64,7 +65,7 @@ class JsonRpcHttpClientSpec extends Specification { def "Make request with basic auth"() { setup: def auth = new AuthConfig.ClientBasicAuth("user", "passwd") - def client = new JsonRpcHttpClient("localhost:18332", auth, null) + def client = new JsonRpcHttpClient("localhost:${port}", auth, null) mockServer.when( HttpRequest.request() @@ -92,7 +93,7 @@ class JsonRpcHttpClientSpec extends Specification { def "Produces RPC Exception on error status code"() { setup: - def client = new JsonRpcHttpClient("localhost:18332", null, null) + def client = new JsonRpcHttpClient("localhost:${port}", null, null) mockServer.when( HttpRequest.request() diff --git a/src/test/resources/bitcoin/esplora-utxo-1.json b/src/test/resources/bitcoin/esplora-utxo-1.json new file mode 100644 index 00000000..8d3218f9 --- /dev/null +++ b/src/test/resources/bitcoin/esplora-utxo-1.json @@ -0,0 +1,156 @@ +[ + { + "txid": "002eba7d9e8081afc687e1c3fa7b6a6451713b75bc91432d441bb1e7e1511c5c", + "vout": 0, + "status": { + "confirmed": true, + "block_height": 647721, + "block_hash": "00000000000000000000dfcec4f7a9bfb8e866a1ce84c65413f823a30d58be65", + "block_time": 1599808442 + }, + "value": 105524 + }, + { + "txid": "1ee3cb3833b1e499e7f8babb1100c4aa0b4273f655036561d84dd72a5b197258", + "vout": 0, + "status": { + "confirmed": true, + "block_height": 648312, + "block_hash": "000000000000000000013ae68460e47a6336e936c39ce509c5eb962d7982b3af", + "block_time": 1600114632 + }, + "value": 5248 + }, + { + "txid": "e21f15131c6fedc165d2daca98043430e6c526f0e3b3d6821f6dc27e450f3eb2", + "vout": 724, + "status": { + "confirmed": true, + "block_height": 645601, + "block_hash": "000000000000000000042e6455477ef88859b630ccb740e675326e7679485fb3", + "block_time": 1598566638 + }, + "value": 59539 + }, + { + "txid": "c14d13fcabfd701a86c6fdac7cd9dc9abd849538308e6526539791ad6befafec", + "vout": 429, + "status": { + "confirmed": true, + "block_height": 647600, + "block_hash": "00000000000000000002864bf9affc84973de0b0099659ad06ae463ac9e3db04", + "block_time": 1599736973 + }, + "value": 10548 + }, + { + "txid": "7c9a77c8a6d4a02afca1dad358098cad6d7b0990079fdb0d6d6289a0538974fd", + "vout": 165, + "status": { + "confirmed": true, + "block_height": 646920, + "block_hash": "0000000000000000000435313e49202f818d6bf9bea6a2864b6da4d8457b7156", + "block_time": 1599363124 + }, + "value": 17050 + }, + { + "txid": "9dd94f926ba50850819fd7a5f07b93fb0d4bbdebf9c80cd5ccd1efd1af65a43d", + "vout": 707, + "status": { + "confirmed": true, + "block_height": 647228, + "block_hash": "0000000000000000000ea8779c8c4d5f973c49fa7c7e7e6acca789240b349582", + "block_time": 1599522964 + }, + "value": 7383 + }, + { + "txid": "10d2f54b99e20abec3b84aaffdbe6214d73bb7c6200350fb1562fd5ef4143d69", + "vout": 812, + "status": { + "confirmed": true, + "block_height": 647013, + "block_hash": "0000000000000000001005dcd53f484fb34bd114eb676d2fb681255ce5877071", + "block_time": 1599408073 + }, + "value": 18941 + }, + { + "txid": "78449f63c1d99deeb038afd49acd46a62e570cdbcfff18c29a57112798c19557", + "vout": 454, + "status": { + "confirmed": true, + "block_height": 647371, + "block_hash": "00000000000000000000d9efae5d8b23187fef9352791576b7355ceeff14f46a", + "block_time": 1599602375 + }, + "value": 9735 + }, + { + "txid": "36a77970d6c0355d3734c6b41dc7f799c7e48c7e508737486bac89848604dbb3", + "vout": 378, + "status": { + "confirmed": true, + "block_height": 647678, + "block_hash": "0000000000000000000b4f64c1e59a81bc6241faa4bf777b0cd84fe2080d290d", + "block_time": 1599776128 + }, + "value": 27053 + }, + { + "txid": "b1b301adb1b46e309bf62d8f4aee4dbf481d7d9008c5d1e840feed78f8fea658", + "vout": 1429, + "status": { + "confirmed": true, + "block_height": 647852, + "block_hash": "0000000000000000000da3a1f8d23df1f9a7e89ab6e348c37cdb056b659f8e71", + "block_time": 1599880147 + }, + "value": 7244 + }, + { + "txid": "6733d19b07b0fa0b22ea6fe7b23371ecfe9f6f1b8d5884a3fe9a1db786bef3df", + "vout": 446, + "status": { + "confirmed": true, + "block_height": 648090, + "block_hash": "00000000000000000008b2f35ae277da04d5b1e7b55d7a20fb4190b21c1cfb74", + "block_time": 1600001410 + }, + "value": 4073 + }, + { + "txid": "01588730a39a90b30321d19b451333b1f24302458da0f5e851ce13a912caa142", + "vout": 303, + "status": { + "confirmed": true, + "block_height": 646625, + "block_hash": "0000000000000000000de3b0f43dcd0870be269a1326f3cbb4b692490cfa78aa", + "block_time": 1599175143 + }, + "value": 5883 + }, + { + "txid": "311d0d1d4eea6ee37d57954cd1002898bef64885d8e438afbbd7fe4fdc6e08de", + "vout": 2250, + "status": { + "confirmed": true, + "block_height": 646382, + "block_hash": "0000000000000000000393bc7709a85b572b2ea239ed07002b230e20b5f47699", + "block_time": 1599018027 + }, + "value": 22978 + }, + { + "txid": "230317b8fdf1ae85f5fbdc49ca90851b1728c2d9432b2738d3fe1c6f68f046e4", + "vout": 11, + "status": { + "confirmed": true, + "block_height": 646777, + "block_hash": "0000000000000000000dab9a4fab9d612f402f491dad9aad6ea94b57995e45a1", + "block_time": 1599273442 + }, + "value": 8642 + } +] \ No newline at end of file diff --git a/src/test/resources/upstreams-bitcoin-esplora.yaml b/src/test/resources/upstreams-bitcoin-esplora.yaml new file mode 100644 index 00000000..165158ec --- /dev/null +++ b/src/test/resources/upstreams-bitcoin-esplora.yaml @@ -0,0 +1,17 @@ +version: v1 + +defaults: + - chains: + - bitcoin + options: + min-peers: 3 + +upstreams: + - id: local + chain: bitcoin + connection: + bitcoin: + rpc: + url: "http://localhost:8545" + esplora: + url: "http://localhost:3001" \ No newline at end of file diff --git a/src/test/resources/upstreams-bitcoin.yaml b/src/test/resources/upstreams-bitcoin.yaml index 13587264..d0bd7ff6 100644 --- a/src/test/resources/upstreams-bitcoin.yaml +++ b/src/test/resources/upstreams-bitcoin.yaml @@ -1,6 +1,6 @@ version: v1 -defaultOptions: +defaults: - chains: - bitcoin options: From 77d7ec1101457f440a42891036cf3ba6166e91c3 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Tue, 15 Sep 2020 19:17:24 -0400 Subject: [PATCH 2/4] solution: optionally provide utxo for bitcoin balance request --- build.gradle | 2 +- .../dshackle/rpc/TrackBitcoinAddress.kt | 69 ++++++++++++------- .../rpc/TrackBitcoinAddressSpec.groovy | 28 +++++++- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/build.gradle b/build.gradle index 7ec58e6e..d811fe76 100644 --- a/build.gradle +++ b/build.gradle @@ -57,7 +57,7 @@ configurations { } dependencies { - implementation "io.emeraldpay:emerald-api:0.7.1" + implementation "io.emeraldpay:emerald-api:0.8.0" implementation "io.grpc:grpc-protobuf:${grpcVersion}" implementation "io.grpc:grpc-stub:${grpcVersion}" diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt index 268f5fe5..717cc081 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt @@ -64,22 +64,41 @@ class TrackBitcoinAddress( } } - fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: List): Flux { + fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: List, includeUtxo: Boolean): Flux { return Flux.fromIterable(addresses) .map { Address(chain, it) } .flatMap { address -> - api.getReader() - .listUnspent(address.bitcoinAddress) - .map { unspents -> - getTotal(address, unspents) - } - .onErrorResume { t -> - log.error("Failed to get unspent", t) - Mono.empty() - } + balanceForAddress(api, address, includeUtxo) } } + fun balanceForAddress(api: BitcoinMultistream, address: Address, includeUtxo: Boolean): Mono { + return api.getReader() + .listUnspent(address.bitcoinAddress) + .map { unspent -> + totalUnspent(address, includeUtxo, unspent) + } + .onErrorResume { t -> + log.error("Failed to get unspent", t) + Mono.empty() + } + } + + fun totalUnspent(address: Address, includeUtxo: Boolean, unspent: List): AddressBalance { + return if (unspent.isEmpty()) { + AddressBalance(address, BigInteger.ZERO) + } else { + unspent.map { + AddressBalance( + address, + BigInteger.valueOf(it.value), + if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value)) + else emptyList() + ) + }.reduce { a, b -> a.plus(b) } + } + } + override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux { val chain = Chain.byId(request.asset.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) @@ -88,19 +107,10 @@ class TrackBitcoinAddress( if (addresses.isEmpty()) { return Flux.empty() } - return requestBalances(chain, upstream, addresses) + return requestBalances(chain, upstream, addresses, request.includeUtxo) .map(this@TrackBitcoinAddress::buildResponse) } - fun getTotal(address: Address, unspents: List): AddressBalance { - val total = if (unspents.isEmpty()) { - 0L - } else { - unspents.map { it.value }.reduce(Long::plus) - } - return AddressBalance(address, BigInteger.valueOf(total)) - } - override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux { val chain = Chain.byId(request.asset.chainValue) @@ -110,10 +120,10 @@ class TrackBitcoinAddress( if (addresses.isEmpty()) { return Flux.empty() } - val initial = requestBalances(chain, upstream, addresses) + val initial = requestBalances(chain, upstream, addresses, request.includeUtxo) val following = upstream.getHead().getFlux() .flatMap { block -> - requestBalances(chain, upstream, addresses) + requestBalances(chain, upstream, addresses, request.includeUtxo) } val last = HashMap() val result = Flux.merge(initial, following) @@ -134,15 +144,26 @@ class TrackBitcoinAddress( .setChainValue(address.address.chain.id) .setCode("BTC")) .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address)) + .addAllUtxo( + address.utxo.map { utxo -> + BlockchainOuterClass.Utxo.newBuilder() + .setBalance(utxo.value.toString()) + .setIndex(utxo.vout.toLong()) + .setTxId(utxo.txid) + .build() + } + ) .build() } - open class AddressBalance(val address: Address, var balance: BigInteger = BigInteger.ZERO) { + open class AddressBalance(val address: Address, var balance: BigInteger = BigInteger.ZERO, var utxo: List = emptyList()) { constructor(chain: Chain, address: String, balance: BigInteger) : this(Address(chain, address), balance) - fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance) + fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance, utxo.plus(other.utxo)) } + open class BalanceUtxo(val txid: String, val vout: Int, val value: Long) + class Address(val chain: Chain, val address: String) { val network = if (chain == Chain.BITCOIN) { MainNetParams() diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy index 14513dd9..8fe856a9 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy @@ -53,27 +53,49 @@ class TrackBitcoinAddressSpec extends Specification { Chain.BITCOIN, "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" ) when: - def total = track.getTotal(address, unspents) + def total = track.totalUnspent(address, false, unspents) then: total.balance == 100 + total.utxo.isEmpty() } def "Correct sum for few"() { setup: def unspents = [ new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 100L, 123L), - new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 123L, 123L), + new SimpleUnspent("17d1c4adf14b222e652c58d11435fa9ee2ddea000c6f5e20e6b715eb940fc28f", 0, 123L, 123L), ] TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) def address = new TrackBitcoinAddress.Address( Chain.BITCOIN, "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" ) when: - def total = track.getTotal(address, unspents) + def total = track.totalUnspent(address, false, unspents) then: total.balance == 223 + total.utxo.isEmpty() + } + + def "Correct sum for few with utxo"() { + setup: + def unspents = [ + new SimpleUnspent("f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef", 0, 100L, 123L), + new SimpleUnspent("17d1c4adf14b222e652c58d11435fa9ee2ddea000c6f5e20e6b715eb940fc28f", 0, 123L, 123L), + ] + TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) + def address = new TrackBitcoinAddress.Address( + Chain.BITCOIN, "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK" + ) + when: + def total = track.totalUnspent(address, true, unspents) + + then: + total.balance == 223 + total.utxo.size() == 2 + total.utxo[0].txid == "f14b222e652c58d11435fa9172ddea000c6f5e20e6b715eb940fc28d1c4adeef" + total.utxo[1].txid == "17d1c4adf14b222e652c58d11435fa9ee2ddea000c6f5e20e6b715eb940fc28f" } def "One address for single provided"() { From 96a6f22a5bd5ecb5ef3a5640a004627603268650 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Tue, 15 Sep 2020 22:55:58 -0400 Subject: [PATCH 3/4] solution: support xpub addresses for bitcoin --- README.adoc | 1 - .../dshackle/rpc/TrackBitcoinAddress.kt | 47 +-- .../upstream/bitcoin/AddressActiveCheck.kt | 35 +++ .../upstream/bitcoin/BitcoinMultistream.kt | 8 + .../upstream/bitcoin/EsploraClient.kt | 38 ++- .../upstream/bitcoin/XpubAddresses.kt | 103 +++++++ .../rpc/TrackBitcoinAddressSpec.groovy | 53 +++- .../upstream/bitcoin/EsploraClientSpec.groovy | 51 ++++ .../upstream/bitcoin/XpubAddressesSpec.groovy | 175 +++++++++++ src/test/resources/bitcoin/esplora-txs-1.json | 287 ++++++++++++++++++ 10 files changed, 764 insertions(+), 34 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt create mode 100644 src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy create mode 100644 src/test/resources/bitcoin/esplora-txs-1.json diff --git a/README.adoc b/README.adoc index 6846a1d2..b9662393 100644 --- a/README.adoc +++ b/README.adoc @@ -45,7 +45,6 @@ image::dshackle-intro.png[alt="",width=80%,align="center"] - [ ] External logging - [ ] Prometheus monitoring - [ ] Subscription to bitcoind notification over gRPC (instead of ZeroMQ) -- [ ] BIP-32 Pubkey - [ ] Lightweight sidecar node connector - [ ] Configurable upstream roles diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt index 717cc081..4b914022 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt @@ -47,25 +47,43 @@ class TrackBitcoinAddress( && BlockchainType.fromBlockchain(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain) } - fun allAddresses(request: BlockchainOuterClass.BalanceRequest): List? { + fun allAddresses(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux { if (!request.hasAddress()) { - return null + return Flux.empty() } return when { + request.address.hasAddressXpub() -> { + val xpubAddresses = api.getXpubAddresses() + ?: return Flux.error(IllegalStateException("Xpub verification is not available")) + + val addressXpub = request.address.addressXpub + if (!addressXpub.xpub.isValidUtf8) { + return Flux.error(IllegalArgumentException("Invalid xpub string")) + } + val xpub = addressXpub.xpub.toStringUtf8() + val start = Math.max(0, addressXpub.start).toInt() + val limit = Math.min(100, Math.max(1, addressXpub.limit)).toInt() + xpubAddresses.activeAddresses(xpub, start, limit) + .map { it.toString() } + .doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") } + } request.address.hasAddressSingle() -> { - listOf(request.address.addressSingle.address) + Flux.just(request.address.addressSingle.address) } request.address.hasAddressMulti() -> { - request.address.addressMulti.addressesList - .map { addr -> addr.address } - .sorted() + Flux.fromIterable( + request.address.addressMulti.addressesList + .map { addr -> addr.address } + //TODO why sorted? + .sorted() + ) } - else -> null + else -> Flux.error(IllegalArgumentException("Unsupported address type")) } } - fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: List, includeUtxo: Boolean): Flux { - return Flux.fromIterable(addresses) + fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: Flux, includeUtxo: Boolean): Flux { + return addresses .map { Address(chain, it) } .flatMap { address -> balanceForAddress(api, address, includeUtxo) @@ -103,10 +121,7 @@ class TrackBitcoinAddress( val chain = Chain.byId(request.asset.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) - val addresses = allAddresses(request) ?: return Flux.error(SilentException("Unsupported address")) - if (addresses.isEmpty()) { - return Flux.empty() - } + val addresses = allAddresses(upstream, request) ?: return Flux.error(SilentException("Unsupported address")) return requestBalances(chain, upstream, addresses, request.includeUtxo) .map(this@TrackBitcoinAddress::buildResponse) } @@ -116,10 +131,7 @@ class TrackBitcoinAddress( val chain = Chain.byId(request.asset.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) - val addresses = allAddresses(request) ?: return Flux.error(SilentException("Unsupported address")) - if (addresses.isEmpty()) { - return Flux.empty() - } + val addresses = allAddresses(upstream, request) ?: return Flux.error(SilentException("Unsupported address")) val initial = requestBalances(chain, upstream, addresses, request.includeUtxo) val following = upstream.getHead().getFlux() .flatMap { block -> @@ -164,6 +176,7 @@ class TrackBitcoinAddress( open class BalanceUtxo(val txid: String, val vout: Int, val value: Long) + //TODO use bitcoin class for address class Address(val chain: Chain, val address: String) { val network = if (chain == Chain.BITCOIN) { MainNetParams() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt new file mode 100644 index 00000000..66426ca8 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import org.bitcoinj.core.Address +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono + +open class AddressActiveCheck( + private val esploraClient: EsploraClient +) { + + companion object { + private val log = LoggerFactory.getLogger(AddressActiveCheck::class.java) + } + + open fun isActive(address: Address): Mono { + //TODO cache with bloom filter + return esploraClient.getTransactions(address).map { it.isNotEmpty() } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt index b5e1eace..f845729f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt @@ -40,6 +40,8 @@ open class BitcoinMultistream( private var head: Head = EmptyHead() private var esplora = upstreams.find { it.esploraClient != null }?.esploraClient private var reader = BitcoinReader(this, head, esplora) + private var addressActiveCheck: AddressActiveCheck? = null + private var xpubAddresses: XpubAddresses? = null override fun init() { if (upstreams.size > 0) { @@ -48,6 +50,10 @@ open class BitcoinMultistream( super.init() } + open fun getXpubAddresses(): XpubAddresses? { + return xpubAddresses + } + override fun updateHead(): Head { head.let { if (it is Lifecycle) { @@ -85,6 +91,8 @@ open class BitcoinMultistream( super.onUpstreamsUpdated() esplora = upstreams.find { it.esploraClient != null }?.esploraClient reader = BitcoinReader(this, this.head, esplora) + addressActiveCheck = esplora?.let { AddressActiveCheck(it) } + xpubAddresses = addressActiveCheck?.let { XpubAddresses(it) } } override fun setHead(head: Head) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt index 74ab7bfb..3ecf533f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt @@ -80,17 +80,7 @@ class EsploraClient( val response = httpClient .get() .uri("$url/address/$address/utxo") - - val json = response - .response { header, bytes -> - if (header.status().code() != 200) { - Mono.error(EsploraException("HTTP Code: ${header.status().code()}")) - } else { - bytes.aggregate().asByteArray() - } - } - .single() - .map { bytes -> String(bytes) } + val json = parseResponse(response) return json.map { val parsed = Global.objectMapper.readerFor(EsploraUnspent::class.java) @@ -99,6 +89,32 @@ class EsploraClient( } } + fun getTransactions(address: Address): Mono>> { + val response = httpClient + .get() + .uri("$url/address/$address/txs") + val json = parseResponse(response) + + return json.map { + val parsed = Global.objectMapper.readerFor(Map::class.java) + .readValues>(it); + parsed.readAll() + } + } + + private fun parseResponse(response: HttpClient.ResponseReceiver<*>): Mono { + return response + .response { header, bytes -> + if (header.status().code() != 200) { + Mono.error(EsploraException("HTTP Code: ${header.status().code()} for ${header.fullPath()}")) + } else { + bytes.aggregate().asByteArray() + } + } + .single() + .map { bytes -> String(bytes) } + } + class EsploraException(msg: String) : Exception(msg) } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt new file mode 100644 index 00000000..e91012c6 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import org.bitcoinj.core.Address +import org.bitcoinj.core.ECKey +import org.bitcoinj.core.NetworkParameters +import org.bitcoinj.crypto.ChildNumber +import org.bitcoinj.crypto.DeterministicKey +import org.bitcoinj.crypto.HDKeyDerivation +import org.bitcoinj.params.MainNetParams +import org.bitcoinj.params.TestNet3Params +import org.bitcoinj.script.Script +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.util.function.Tuples +import java.util.concurrent.atomic.AtomicInteger + +open class XpubAddresses( + private val addressActiveCheck: AddressActiveCheck +) { + + companion object { + private val log = LoggerFactory.getLogger(XpubAddresses::class.java) + private val MAINNET = MainNetParams() + private val TESTNET = TestNet3Params() + private val INACTIVE_LIMIT = 20 + } + + open fun allAddresses(xpub: String, start: Int, limit: Int): Flux
{ + // versions: + // https://electrum.readthedocs.io/en/latest/xpub_version_bytes.html + // TODO doesn't support SH keys right now. should? + val prefix = xpub.substring(0, 4) + val type: Script.ScriptType + val network: NetworkParameters + + when (prefix) { + "xpub" -> { + type = Script.ScriptType.P2PKH + network = MAINNET + } + "zpub" -> { + type = Script.ScriptType.P2WPKH + network = MAINNET + } + "tpub" -> { + type = Script.ScriptType.P2PKH + network = TESTNET + } + "vpub" -> { + type = Script.ScriptType.P2WPKH + network = TESTNET + } + else -> return Flux.error(IllegalArgumentException("Unsupported type: $prefix")) + } + + val key: DeterministicKey + try { + key = DeterministicKey.deserializeB58(xpub, network) + } catch (t: Throwable) { + return Flux.error(t) + } + + return Flux.range(start, limit) + .map { HDKeyDerivation.deriveChildKey(key, ChildNumber(it, false)) } + .map { Address.fromKey(network, ECKey.fromPublicOnly(it.pubKeyPoint), type) } + } + + open fun activeAddresses(xpub: String, start: Int, limit: Int): Flux
{ + val lastActive = AtomicInteger(0) + return this.allAddresses(xpub, start, limit) + .zipWith(Flux.range(0, limit)) + .takeUntil { + it.t2 - lastActive.get() >= INACTIVE_LIMIT + } + .concatMap { toCheck -> + addressActiveCheck.isActive(toCheck.t1) + .doOnNext { active -> if (active) lastActive.set(toCheck.t2) } + .map { Tuples.of(toCheck.t1, it) } + } + .filter { + it.t2 + } + .map { + it.t1 + } + } + +} \ No newline at end of file diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy index 8fe856a9..34062d3e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.rpc import com.fasterxml.jackson.databind.ObjectMapper +import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.Global @@ -27,8 +28,12 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinReader +import io.emeraldpay.dshackle.upstream.bitcoin.XpubAddresses import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.grpc.Chain +import org.bitcoinj.core.Address +import org.bitcoinj.params.MainNetParams +import org.bitcoinj.params.TestNet3Params import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.core.publisher.TopicProcessor @@ -111,7 +116,7 @@ class TrackBitcoinAddressSpec extends Specification { ) .build() when: - def act = track.allAddresses(req) + def act = track.allAddresses(Stub(BitcoinMultistream), req).collectList().block() then: act == ["16rCmCmbuWDhPjWTrpQGaU3EPdZF7MTdUk"] } @@ -132,20 +137,58 @@ class TrackBitcoinAddressSpec extends Specification { ) .build() when: - def act = track.allAddresses(req) + def act = track.allAddresses(Stub(BitcoinMultistream), req).collectList().block() then: act == ["16rCmCmbuWDhPjWTrpQGaU3EPdZF7MTdUk", "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK", "3BMqADKWoWHPASsUdHvnUL6E1jpZkMnLZz", "bc1qdthqvt6cllzej7uhdddrltdfsmnt7d0gl5ue5n"] } - def "Null for no address provided"() { + def "Empty for no address provided"() { setup: TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) def req = BlockchainOuterClass.BalanceRequest.newBuilder() .build() when: - def act = track.allAddresses(req) + def act = track.allAddresses(Stub(BitcoinMultistream), req).collectList().block() then: - act == null + act == [] + } + + def "Use active for xpub"() { + setup: + TrackBitcoinAddress track = new TrackBitcoinAddress(Stub(MultistreamHolder)) + def req = BlockchainOuterClass.BalanceRequest.newBuilder() + .setAddress( + Common.AnyAddress.newBuilder() + .setAddressXpub( + // seed: chimney battle code relief era plug finish video patch dream pumpkin govern destroy fresh color + Common.XpubAddress.newBuilder() + .setXpub(ByteString.copyFromUtf8("zpub6tz4F49K5B4m7r7EyBKYM9K44eGECaQ2AfrCybq1w7ALFatz9856vrXxAPSrteDA4d5sjUPW3ACNq8wB2V3ugXVJxvAPAYPAYHsVm3VAncL")) + .setLimit(25) + ) + ) + .build() + def xpubAddresses = Mock(XpubAddresses) { + 1 * activeAddresses( + "zpub6tz4F49K5B4m7r7EyBKYM9K44eGECaQ2AfrCybq1w7ALFatz9856vrXxAPSrteDA4d5sjUPW3ACNq8wB2V3ugXVJxvAPAYPAYHsVm3VAncL", + 0, + 25 + ) >> Flux.fromIterable([ + "bc1q25590fu8djhw9lvxxqz8ufjyfwup9h54u8fl6t", + "bc1q3k6e6vawd5l5syu9nlxn2xsch9afgunl8dnz94", + "bc1qu7hd6wycy686kakfps9c093szufjpwnh6rjs9s" + ]).map { Address.fromString(MainNetParams.get(), it) } + } + def multistream = Mock(BitcoinMultistream) { + 1 * getXpubAddresses() >> xpubAddresses + } + when: + def act = track.allAddresses(multistream, req).collectList().block() + then: + act == [ + "bc1q25590fu8djhw9lvxxqz8ufjyfwup9h54u8fl6t", + "bc1q3k6e6vawd5l5syu9nlxn2xsch9afgunl8dnz94", + "bc1qu7hd6wycy686kakfps9c093szufjpwnh6rjs9s" + ] } def "Build proto for common balance"() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy index 5b046841..59f4240c 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy @@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent import org.bitcoinj.core.Address import org.bitcoinj.params.MainNetParams +import org.bitcoinj.params.TestNet3Params import org.mockserver.integration.ClientAndServer import org.mockserver.model.HttpRequest import org.mockserver.model.HttpResponse @@ -80,4 +81,54 @@ class EsploraClientSpec extends Specification { actTotal == 309841L } + def "get txs"() { + setup: + def responseJson = this.class.getClassLoader().getResourceAsStream("bitcoin/esplora-txs-1.json").text + mockServer.when( + HttpRequest.request() + .withMethod("GET") + .withPath("/address/tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx/txs") + ).respond( + HttpResponse.response(responseJson) + ) + def client = new EsploraClient(new URI("http://localhost:23001"), null, null) + when: + def act = client.getTransactions(Address.fromString(TestNet3Params.get(), "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx")) + + then: + StepVerifier.create(act) + .expectNextMatches { list -> + def ok = list.size() == 6 + if (!ok) println("invalid size") + ok = ok && list[0].txid == "a738f4bd63f785d58acd2e83e8c8c5e84e68dabacac3a1142ee47cb188103aab" + ok = ok && list[1].txid == "4a7066adb2cd8e37f1578d6991f14c12e677e8bc0556992a73a91a26d263f89a" + ok = ok && list[2].txid == "16aa2d98e37e50c4c007a815a3cb8c20026a3df467781a7e97206a730cf4ef01" + ok = ok && list[3].txid == "caaea0ca92343a2d7115c44a0f58cb6574b9349905799854e1b5a6c3b3f33587" + ok = ok && list[4].txid == "a386377a406465423275f51d6dc71a2c245acc55c356a7e851a03b508827bc1e" + ok = ok && list[5].txid == "7944cfcd3d04c58a81aaa7067616f8ac368581a847e03447d5d2917cc75b67d2" + ok + } + .expectComplete() + .verify(Duration.ofSeconds(3)) + } + + def "get txs when empty"() { + setup: + mockServer.when( + HttpRequest.request() + .withMethod("GET") + .withPath("/address/tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx/txs") + ).respond( + HttpResponse.response("[]") + ) + def client = new EsploraClient(new URI("http://localhost:23001"), null, null) + when: + def act = client.getTransactions(Address.fromString(TestNet3Params.get(), "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx")) + + then: + StepVerifier.create(act) + .expectNext([]) + .expectComplete() + .verify(Duration.ofSeconds(3)) + } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy new file mode 100644 index 00000000..f9be8eda --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy @@ -0,0 +1,175 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.bitcoin + +import org.bitcoinj.core.Address +import org.bitcoinj.params.MainNetParams +import reactor.core.publisher.Mono +import reactor.test.StepVerifier +import spock.lang.Specification + +import java.time.Duration + +class XpubAddressesSpec extends Specification { + + + def "list all for mainnet"() { + setup: + XpubAddresses addresses = new XpubAddresses(Stub(AddressActiveCheck)) + + when: + // seed sustain pyramid victory drum primary silver safe wrestle section leg caught praise spring prepare input + // account 0 + def act = addresses.allAddresses( + "zpub6tsHyQGj1UmDcw9UavB31HSChH9rqMi8Qje5KPbnczzhFSZykmm3afivpVytxWubaabDih6AbGGhioiTCg5PssYXuVGiZ1vTVaMYvvQvvmz", + 0, + 4 + ).collectList().block() + + then: + act.size() == 4 + act[0].toString() == "bc1qd6p79j20w2zy5zagptf5ksjkdmhx4d5sykrz0e" + act[1].toString() == "bc1q54um5vzdjuwt6qvzvk2y5em72jedpktrvr24a3" + act[2].toString() == "bc1q3yl03ugy6l5k3te5x340m70hyeqjzmmvksnuc8" + act[3].toString() == "bc1qp0cgpkxl52r82ev9k2wpdzzk4m4nflwurj5yr8" + } + + def "list all for testnet"() { + setup: + XpubAddresses addresses = new XpubAddresses(Stub(AddressActiveCheck)) + + when: + // seed sustain pyramid victory drum primary silver safe wrestle section leg caught praise spring prepare input + // account 1 + def act = addresses.allAddresses( + "vpub5aEsY5bNGvjkHPjdMiz8FbdoiT81kmF3JwuJ1LRdoFb7DxNh1qAHQCmMeDnr6RG2EcCwzGohem3oESxZGa6YWLPW79ryCyMrdYj54uUzNNq", + 0, + 4 + ).collectList().block() + + then: + act.size() == 4 + act[0].toString() == "tb1q2phmcxl4tflgcrnm00jvrkh4a8n876vkjcv97m" + act[1].toString() == "tb1qnhn0psv0uqtmg64tm258kqu0thurjvutcc80xs" + act[2].toString() == "tb1qg62jwv77ul329pe4wcchr83zz02egwfh6syqsr" + act[3].toString() == "tb1q8yg5qknrl5wmc98lw8dyfgrzamzaq8c9l0g44d" + } + + def "list starting from some"() { + setup: + XpubAddresses addresses = new XpubAddresses(Stub(AddressActiveCheck)) + + when: + // seed sustain pyramid victory drum primary silver safe wrestle section leg caught praise spring prepare input + // account 2 + def act = addresses.allAddresses( + "zpub6tyKtxsQL16XpzxwFJSrVTxpKF4z8GL2WTqvdErC2iQBSEAzLqJNc8h6hg4EHMSPEavFDneR3JQjWjbZL7vtQrXMwN9bD4c9K9s1Z55aZfB", + 10, + 4 + ).collectList().block() + + then: + act.size() == 4 + act[0].toString() == "bc1q3dzm92mnqvkyadwl3x5cqdvxaqa5h9c37qdlgs" + act[1].toString() == "bc1q3kqug4cx95a02yhwn6geelftmw3zklrgmhjll8" + act[2].toString() == "bc1q6znmcw8zzjv3asdylgkmt0k0xvhzl9q5awh0sq" + act[3].toString() == "bc1q3fzlxvw803r8j44culrt7jz7q5y7zakushujmq" + } + + def "list when only fist is active"() { + setup: + AddressActiveCheck check = Mock(AddressActiveCheck) { + 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qaexx257l7sgm62szw2ulj6n2v99t5ph9ekkul3")) >> Mono.just(true) + 20 * isActive(_) >> Mono.just(false) + } + XpubAddresses addresses = new XpubAddresses(check) + + // seed sustain pyramid victory drum primary silver safe wrestle section leg caught praise spring prepare input + // account 2 + when: + def act = addresses.activeAddresses( + "zpub6tyKtxsQL16XpzxwFJSrVTxpKF4z8GL2WTqvdErC2iQBSEAzLqJNc8h6hg4EHMSPEavFDneR3JQjWjbZL7vtQrXMwN9bD4c9K9s1Z55aZfB", + 0, + 50 + ).map { it.toString() } + + then: + StepVerifier.create(act) + .expectNext("bc1qaexx257l7sgm62szw2ulj6n2v99t5ph9ekkul3") + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + + def "list when only 3rd is active"() { + setup: + AddressActiveCheck check = Mock(AddressActiveCheck) { + 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz")) >> Mono.just(true) + // 2 times before, 20 times after + 22 * isActive(_) >> Mono.just(false) + } + XpubAddresses addresses = new XpubAddresses(check) + + // seed sustain pyramid victory drum primary silver safe wrestle section leg caught praise spring prepare input + // account 2 + when: + def act = addresses.activeAddresses( + "zpub6tyKtxsQL16XpzxwFJSrVTxpKF4z8GL2WTqvdErC2iQBSEAzLqJNc8h6hg4EHMSPEavFDneR3JQjWjbZL7vtQrXMwN9bD4c9K9s1Z55aZfB", + 0, + 50 + ).map { it.toString() } + + then: + StepVerifier.create(act) + .expectNext("bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz") + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + + def "list when only 22 is active"() { + setup: + AddressActiveCheck check = Mock(AddressActiveCheck) { + // 2 + 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz")) >> Mono.just(true) + // 11 + 1 * isActive(Address.fromString(MainNetParams.get(), "bc1q3kqug4cx95a02yhwn6geelftmw3zklrgmhjll8")) >> Mono.just(true) + // 22 + 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qf7m2rrmrksj34vhgxmm04y43dlj7c5f58f8ku6")) >> Mono.just(true) + // 0..1 = 2 + // + 3..10 = 8 + // + 12..21 = 10 + // + 20 + 40 * isActive(_) >> Mono.just(false) + } + XpubAddresses addresses = new XpubAddresses(check) + + // seed sustain pyramid victory drum primary silver safe wrestle section leg caught praise spring prepare input + // account 2 + when: + def act = addresses.activeAddresses( + "zpub6tyKtxsQL16XpzxwFJSrVTxpKF4z8GL2WTqvdErC2iQBSEAzLqJNc8h6hg4EHMSPEavFDneR3JQjWjbZL7vtQrXMwN9bD4c9K9s1Z55aZfB", + 0, + 100 + ).map { it.toString() } + + then: + StepVerifier.create(act) + .expectNext("bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz") + .expectNext("bc1q3kqug4cx95a02yhwn6geelftmw3zklrgmhjll8") + .expectNext("bc1qf7m2rrmrksj34vhgxmm04y43dlj7c5f58f8ku6") + .expectComplete() + .verify(Duration.ofSeconds(1)) + } +} diff --git a/src/test/resources/bitcoin/esplora-txs-1.json b/src/test/resources/bitcoin/esplora-txs-1.json new file mode 100644 index 00000000..252a96a2 --- /dev/null +++ b/src/test/resources/bitcoin/esplora-txs-1.json @@ -0,0 +1,287 @@ +[ + { + "txid": "a738f4bd63f785d58acd2e83e8c8c5e84e68dabacac3a1142ee47cb188103aab", + "version": 1, + "locktime": 0, + "vin": [ + { + "txid": "16aa2d98e37e50c4c007a815a3cb8c20026a3df467781a7e97206a730cf4ef01", + "vout": 1, + "prevout": { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 120000 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304502210084f12093ae826635c6bce4915e927dd7a2285b37d0b51148c6d6177ce72e459d02201bf0b3d7cde090d2dbc4766b7a8b3d947e1bdb02dcf03ce894ff1a105278eb8901", + "03a3aa29d96671671b065b35de511c03ea5592eafb5de7a07542633af2d42f49ea" + ], + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 119568 + } + ], + "size": 192, + "weight": 438, + "fee": 432, + "status": { + "confirmed": true, + "block_height": 1831865, + "block_hash": "000000000000014e643687acc0a975f9aa7319f1e9bb428e0f1d618ed6aba370", + "block_time": 1599447368 + } + }, + { + "txid": "4a7066adb2cd8e37f1578d6991f14c12e677e8bc0556992a73a91a26d263f89a", + "version": 1, + "locktime": 0, + "vin": [ + { + "txid": "a386377a406465423275f51d6dc71a2c245acc55c356a7e851a03b508827bc1e", + "vout": 1, + "prevout": { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 120000 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3045022100c5af6d33efc9a564c0e955160e0b8def089cc6c933ddf08967eac25d4ddfc09a02205ed59dcc068251dee669ca19425c0c83208d1b1c1f5b7147f4f0b7cb2d15a87201", + "03a3aa29d96671671b065b35de511c03ea5592eafb5de7a07542633af2d42f49ea" + ], + "is_coinbase": false, + "sequence": 4294967295 + } + ], + "vout": [ + { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 119568 + } + ], + "size": 192, + "weight": 438, + "fee": 432, + "status": { + "confirmed": true, + "block_height": 1831846, + "block_hash": "0000000000000104a5b1241385c2cfb728128ce692ab8bd8970a0372761f8887", + "block_time": 1599439188 + } + }, + { + "txid": "16aa2d98e37e50c4c007a815a3cb8c20026a3df467781a7e97206a730cf4ef01", + "version": 2, + "locktime": 1831845, + "vin": [ + { + "txid": "a386377a406465423275f51d6dc71a2c245acc55c356a7e851a03b508827bc1e", + "vout": 0, + "prevout": { + "scriptpubkey": "00140fc773ca476039acb48162e26190b3b93cda6762", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 0fc773ca476039acb48162e26190b3b93cda6762", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qplrh8jj8vqu6edypvt3xry9nhy7d5emzyj3a3h", + "value": 1619316 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022067011a2a7b6823146ff8291eed673e2e52703e07b8b99cf78b0fbb968e5ca980022018244b0584f4851ec9d09296007898c00f05afdcc6c2d8eafb48e95923be609801", + "031117740bfada16dbe932219d5d5ddbb0a829d83ccbfe5eb7734d8646f9a63479" + ], + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "0014a871aa972c4ef9255a610b3b849598991ba9df1c", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a871aa972c4ef9255a610b3b849598991ba9df1c", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1q4pc649evfmuj2knppvacf9vcnyd6nhculm5lsk", + "value": 1499175 + }, + { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 120000 + } + ], + "size": 222, + "weight": 561, + "fee": 141, + "status": { + "confirmed": true, + "block_height": 1831846, + "block_hash": "0000000000000104a5b1241385c2cfb728128ce692ab8bd8970a0372761f8887", + "block_time": 1599439188 + } + }, + { + "txid": "caaea0ca92343a2d7115c44a0f58cb6574b9349905799854e1b5a6c3b3f33587", + "version": 1, + "locktime": 0, + "vin": [ + { + "txid": "7944cfcd3d04c58a81aaa7067616f8ac368581a847e03447d5d2917cc75b67d2", + "vout": 1, + "prevout": { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 120000 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402200fe43b74f1326e51d5c718b5c0a9319c0241de9329e7d39e613218f66bf72f1202205918d5b29b7d6887a4f61fe3f7c330beb306e195b49e7149f5de1554f8aaa10501", + "03a3aa29d96671671b065b35de511c03ea5592eafb5de7a07542633af2d42f49ea" + ], + "is_coinbase": false, + "sequence": 4294967294 + } + ], + "vout": [ + { + "scriptpubkey": "00146d2112c30c7ef1bf66a4ff2e1bfa0304c3b8f118", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 6d2112c30c7ef1bf66a4ff2e1bfa0304c3b8f118", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qd5s39scv0mcm7e4yluhph7srqnpm3ugcx2mc5d", + "value": 119568 + } + ], + "size": 191, + "weight": 437, + "fee": 432, + "status": { + "confirmed": true, + "block_height": 1831820, + "block_hash": "0000000009f11ea4cd3a3393d1cc355a63ea37a5a2cbf983e34cbb7f37590eb2", + "block_time": 1599426488 + } + }, + { + "txid": "a386377a406465423275f51d6dc71a2c245acc55c356a7e851a03b508827bc1e", + "version": 2, + "locktime": 1831819, + "vin": [ + { + "txid": "7944cfcd3d04c58a81aaa7067616f8ac368581a847e03447d5d2917cc75b67d2", + "vout": 0, + "prevout": { + "scriptpubkey": "0014da239eb3f9727ccbfe73823e427fb7289e7c192a", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 da239eb3f9727ccbfe73823e427fb7289e7c192a", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qmg3eavlewf7vhlnnsglyylah9z08cxf2a0g2nv", + "value": 1739457 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022039020c3e01db872751e8c55f584aea2820b4e7c04c5a859ac785983aa2aaf612022012e8b0f6b052cecbcbe452dc0c8b6093f0520b355d206476c934ee879e892cfe01", + "023a49d61f3162fbba038809868a3c943de0549bae2f197d78ea8670452595cece" + ], + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "00140fc773ca476039acb48162e26190b3b93cda6762", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 0fc773ca476039acb48162e26190b3b93cda6762", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qplrh8jj8vqu6edypvt3xry9nhy7d5emzyj3a3h", + "value": 1619316 + }, + { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 120000 + } + ], + "size": 222, + "weight": 561, + "fee": 141, + "status": { + "confirmed": true, + "block_height": 1831820, + "block_hash": "0000000009f11ea4cd3a3393d1cc355a63ea37a5a2cbf983e34cbb7f37590eb2", + "block_time": 1599426488 + } + }, + { + "txid": "7944cfcd3d04c58a81aaa7067616f8ac368581a847e03447d5d2917cc75b67d2", + "version": 2, + "locktime": 1831428, + "vin": [ + { + "txid": "86f3a772c109695ae67fe15e30df8929d2b71a2beef86ac51df406150d991a4b", + "vout": 0, + "prevout": { + "scriptpubkey": "00141d7aa33e35e539b16b11167627e818fd436ea907", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 1d7aa33e35e539b16b11167627e818fd436ea907", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qr4a2x034u5umz6c3zemz06qcl4pka2g8cc2qaf", + "value": 1859598 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022020fa758e75fa4f67665b14cc50f871883a80c1c1f4f45c403e7f8c5631b8734002200ddde3f72dbe88975365777c15da85c44eaa6ba53384194b351709f015f4bf2001", + "02f0db65b4318ee91f58d1f7da51fb16c46e3a01f6f3bd07af1b8be63e525e3e45" + ], + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "0014da239eb3f9727ccbfe73823e427fb7289e7c192a", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 da239eb3f9727ccbfe73823e427fb7289e7c192a", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qmg3eavlewf7vhlnnsglyylah9z08cxf2a0g2nv", + "value": 1739457 + }, + { + "scriptpubkey": "00142757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 2757c732c931d7722a6bdaf99ee9955303116520", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx", + "value": 120000 + } + ], + "size": 222, + "weight": 561, + "fee": 141, + "status": { + "confirmed": true, + "block_height": 1831429, + "block_hash": "00000000000000235978c8a27a20881376bb94b831d2072d80500d2350115be1", + "block_time": 1599254339 + } + } +] \ No newline at end of file From 275a55c4eecfe0361bcc77161bf37073d47e6c9c Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Tue, 15 Sep 2020 23:32:23 -0400 Subject: [PATCH 4/4] problem: balance subscription misses recheck addresses for xpub --- .../dshackle/rpc/TrackBitcoinAddress.kt | 20 ++++++++++--------- .../rpc/TrackBitcoinAddressSpec.groovy | 7 ++++++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt index 4b914022..43167e8b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt @@ -131,19 +131,21 @@ class TrackBitcoinAddress( val chain = Chain.byId(request.asset.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) - val addresses = allAddresses(upstream, request) ?: return Flux.error(SilentException("Unsupported address")) - val initial = requestBalances(chain, upstream, addresses, request.includeUtxo) + val addresses = allAddresses(upstream, request).cache() val following = upstream.getHead().getFlux() .flatMap { block -> - requestBalances(chain, upstream, addresses, request.includeUtxo) + requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo) } - val last = HashMap() - val result = Flux.merge(initial, following) + val last = HashMap() + val result = following .filter { curr -> - val prev = last[curr.address] - val updated = prev == null || curr.balance != prev - last[curr.address] = curr.balance - updated + val prev = last[curr.address.address] + //TODO utxo can change without changing balance + val changed = prev == null || curr.balance != prev + if (changed) { + last[curr.address.address] = curr.balance + } + changed } return result.map(this@TrackBitcoinAddress::buildResponse) diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy index 34062d3e..be866781 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy @@ -235,7 +235,12 @@ class TrackBitcoinAddressSpec extends Specification { def blocks = TopicProcessor.create() Head head = Mock(Head) { - 1 * getFlux() >> Flux.from(blocks) + 1 * getFlux() >> Flux.concat( + Flux.just( + new BlockContainer(0L, BlockId.from(hash1), BigInteger.ZERO, Instant.now(), false, null, null, []) + ), + Flux.from(blocks) + ) } def upstream = null upstream = Mock(BitcoinMultistream) {