StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/emeraldpay/dshackle

Browse Source
This commit is contained in:
terminal
2022-08-01 13:33:15 +04:00
parent 3d44b99346 d65a217535
commit 578c148362
19 changed files with 155 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt
View File

@@ -224,7 +224,11 @@ class UpstreamsConfigReader(
upNode: MappingNode,
upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>
) {
// Dshackle gRPC connection dispatches requests to different upstreams, which may
// be on different blockchains, and each may have different set of labels.
// So the labels and chains assigned to the gRPC connection make no sense.
if (hasAny(upNode, "labels")) {
// Actual labels from underlying upstreams are handled by GrpcUpstreamStatus
log.warn("Labels should be not applied to gRPC upstream")
}
if (hasAny(upNode, "chain")) {

15
src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt
View File

@@ -57,7 +57,7 @@ class TrackBitcoinAddress(
/**
* Keep tracking of the current state of local upstreams. True for a chain that has an upstream with balance data.
*/
private val balanceAvailable: MutableMap<Chain, Boolean> = ConcurrentHashMap()
private val localBalanceAvailable: MutableMap<Chain, Boolean> = ConcurrentHashMap()
/**
* Criteria for a remote grpc upstream that can provide a balance
@@ -72,7 +72,7 @@ class TrackBitcoinAddress(
multistreamHolder.observeChains().subscribe { chain ->
multistreamHolder.getUpstream(chain)?.let { mup ->
val available = mup.getAll().any { up ->
!up.isGrpc() && (up.getOptions().providesBalance ?: false)
!up.isGrpc() && up.getCapabilities().contains(Capability.BALANCE)
}
setBalanceAvailability(chain, available)
}
@@ -80,14 +80,14 @@ class TrackBitcoinAddress(
}
fun setBalanceAvailability(chain: Chain, enabled: Boolean) {
balanceAvailable[chain] = enabled
localBalanceAvailable[chain] = enabled
}
/**
* @return true if the current instance has data sources to provide the balance
*/
fun isBalanceAvailable(chain: Chain): Boolean {
return balanceAvailable[chain] ?: false
return localBalanceAvailable[chain] ?: false
}
fun allAddresses(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux<String> {
@@ -179,10 +179,9 @@ class TrackBitcoinAddress(
}
.timeout(Defaults.timeoutInternal, Mono.empty())
.switchIfEmpty(
Mono.just(0)
.doOnNext {
log.warn("No upstream providing balance for ${api.chain}")
}
Mono.fromCallable {
log.warn("No upstream providing balance for ${api.chain}")
}
.then(Mono.error(SilentException.DataUnavailable("BALANCE")))
)
}

2
src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
View File

@@ -257,7 +257,7 @@ open class ConfiguredUpstreams(
log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}")
ds.start()
.doOnNext {
log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}")
log.info("Chain ${it.chain} ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}. With caps: ${it.upstream.getCapabilities()}")
}
.subscribe(currentUpstreams::update)
}

31
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt
View File

@@ -26,7 +26,7 @@ import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.RequestPostprocessor
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.ethereum.LocalCallRouter
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
@@ -37,28 +37,34 @@ import reactor.core.publisher.Mono
@Suppress("UNCHECKED_CAST")
open class BitcoinMultistream(
chain: Chain,
val upstreams: MutableList<BitcoinUpstream>,
private val sourceUpstreams: MutableList<BitcoinUpstream>,
caches: Caches
) : Multistream(chain, upstreams as MutableList<Upstream>, caches, RequestPostprocessor.Empty()), Lifecycle {
) : Multistream(chain, sourceUpstreams as MutableList<Upstream>, caches, RequestPostprocessor.Empty()), Lifecycle {
companion object {
private val log = LoggerFactory.getLogger(BitcoinMultistream::class.java)
}
private var head: Head = EmptyHead()
private var esplora = upstreams.find { it.esploraClient != null }?.esploraClient
private var esplora = sourceUpstreams.find { it.esploraClient != null }?.esploraClient
private var reader = BitcoinReader(this, head, esplora)
private var addressActiveCheck: AddressActiveCheck? = null
private var xpubAddresses: XpubAddresses? = null
private val feeEstimation = BitcoinFees(this, reader, 6)
private var callRouter: LocalCallRouter = LocalCallRouter(DefaultBitcoinMethods(), reader)
override fun init() {
if (upstreams.size > 0) {
if (sourceUpstreams.size > 0) {
head = updateHead()
}
super.init()
}
open val upstreams: List<BitcoinUpstream>
get() {
return sourceUpstreams
}
override fun getFeeEstimation(): ChainFees {
return feeEstimation
}
@@ -75,8 +81,8 @@ open class BitcoinMultistream(
}
lagObserver?.stop()
lagObserver = null
val head = if (upstreams.size == 1) {
val upstream = upstreams.first()
val head = if (sourceUpstreams.size == 1) {
val upstream = sourceUpstreams.first()
upstream.setLag(0)
upstream.getHead().apply {
if (this is Lifecycle) {
@@ -84,10 +90,10 @@ open class BitcoinMultistream(
}
}
} else {
val newHead = MergedHead(upstreams.map { it.getHead() }).apply {
val newHead = MergedHead(sourceUpstreams.map { it.getHead() }).apply {
this.start()
}
val lagObserver = BitcoinHeadLagObserver(newHead, upstreams)
val lagObserver = BitcoinHeadLagObserver(newHead, sourceUpstreams)
this.lagObserver = lagObserver
lagObserver.start()
newHead
@@ -97,7 +103,7 @@ open class BitcoinMultistream(
}
override fun getRoutedApi(matcher: Selector.Matcher): Mono<Reader<JsonRpcRequest, JsonRpcResponse>> {
return Mono.just(LocalCallRouter(getMethods()))
return Mono.just(callRouter)
}
open fun getReader(): BitcoinReader {
@@ -106,10 +112,11 @@ open class BitcoinMultistream(
override fun onUpstreamsUpdated() {
super.onUpstreamsUpdated()
esplora = upstreams.find { it.esploraClient != null }?.esploraClient
esplora = sourceUpstreams.find { it.esploraClient != null }?.esploraClient
reader = BitcoinReader(this, this.head, esplora)
addressActiveCheck = esplora?.let { AddressActiveCheck(it) }
xpubAddresses = addressActiveCheck?.let { XpubAddresses(it) }
callRouter = LocalCallRouter(getMethods(), reader)
}
override fun setHead(head: Head) {
@@ -122,7 +129,7 @@ open class BitcoinMultistream(
}
override fun getLabels(): Collection<UpstreamsConfig.Labels> {
return upstreams.flatMap { it.getLabels() }
return sourceUpstreams.flatMap { it.getLabels() }
}
@Suppress("UNCHECKED_CAST")

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt
View File

@@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
@@ -43,6 +44,8 @@ open class BitcoinReader(
private val unspentReader: UnspentReader = if (esploraClient != null) {
EsploraUnspentReader(esploraClient, head)
} else if (upstreams.upstreams.any { it.isGrpc() && it.getCapabilities().contains(Capability.BALANCE) }) {
RemoteUnspentReader(upstreams)
} else {
RpcUnspentReader(upstreams)
}

1
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt
View File

@@ -37,7 +37,6 @@ class EsploraUnspentReader(
base.txid,
base.vout,
base.value,
head.getCurrentHeight()?.let { base.height - it } ?: 0
)
}

39
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/LocalCallRouter.kt
View File

@@ -15,12 +15,17 @@
*/
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.RpcResponseError
import org.bitcoinj.core.Address
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
@@ -33,6 +38,7 @@ import reactor.core.publisher.Mono
*/
class LocalCallRouter(
private val methods: CallMethods,
private val reader: BitcoinReader,
) : Reader<JsonRpcRequest, JsonRpcResponse> {
companion object {
@@ -47,6 +53,39 @@ class LocalCallRouter(
if (!methods.isCallable(key.method)) {
return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))
}
if (key.method == "listunspent") {
return processUnspentRequest(key)
}
return Mono.empty()
}
/**
*
*/
fun processUnspentRequest(key: JsonRpcRequest): Mono<JsonRpcResponse> {
if (key.params.size < 3) {
return Mono.error(SilentException("Invalid call to unspent. Address is missing"))
}
val addresses = key.params[2]
if (addresses is List<*> && addresses.size > 0) {
val address = addresses[0].toString().let { Address.fromString(null, it) }
return reader.listUnspent(address).map {
val rpc = it.map(convertUnspent(address))
val json = Global.objectMapper.writeValueAsBytes(rpc)
JsonRpcResponse.ok(json, JsonRpcResponse.NumberId(key.id))
}
}
return Mono.error(SilentException("Invalid call to unspent"))
}
fun convertUnspent(address: Address): (SimpleUnspent) -> RpcUnspent {
return { base ->
RpcUnspent(
base.txid,
base.vout,
address.toString(),
base.value,
)
}
}
}

48
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RemoteUnspentReader.kt Normal file
View File

@@ -0,0 +1,48 @@
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.api.proto.BlockchainOuterClass.BalanceRequest
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream
import org.bitcoinj.core.Address
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
class RemoteUnspentReader(
val upstreams: BitcoinMultistream
) : UnspentReader {
companion object {
private val log = LoggerFactory.getLogger(RemoteUnspentReader::class.java)
}
private val selector = Selector.LocalAndMatcher(
Selector.GrpcMatcher(),
Selector.CapabilityMatcher(Capability.BALANCE)
)
override fun read(key: Address): Mono<List<SimpleUnspent>> {
val apis = upstreams.getApiSource(selector)
apis.request(1)
return Mono.from(apis)
.map { up ->
up.cast(BitcoinGrpcUpstream::class.java).remote
}
.flatMapMany {
val request = BalanceRequest.newBuilder()
.build()
it.getBalance(request)
}
.map { resp ->
resp.utxoList.map { utxo ->
SimpleUnspent(
utxo.txId,
utxo.index.toInt(),
utxo.balance.toLong(),
)
}
}
.reduce(List<SimpleUnspent>::plus)
}
}

13
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt
View File

@@ -16,6 +16,8 @@
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
@@ -38,14 +40,17 @@ class RpcUnspentReader(
base.txid,
base.vout,
base.amount,
base.confirmations
)
}
private val selector = Selector.CapabilityMatcher(Capability.BALANCE)
override fun read(key: Address): Mono<List<SimpleUnspent>> {
// docs: https://developer.bitcoin.org/reference/rpc/listunspent.html
//
val address = key.toString()
return upstreams.getDirectApi(Selector.empty).flatMap { api ->
api.read(JsonRpcRequest("listunspent", emptyList()))
return upstreams.getDirectApi(selector).flatMap { api ->
api.read(JsonRpcRequest("listunspent", listOf(1, 9999999, listOf(address))))
.flatMap(JsonRpcResponse::requireResult)
.map {
Global.objectMapper.readerFor(RpcUnspent::class.java).readValues<RpcUnspent>(it).readAll()
@@ -55,6 +60,6 @@ class RpcUnspentReader(
it.address == address
}.map(convert)
}
}
}.switchIfEmpty(Mono.error(SilentException.DataUnavailable("BALANCE")))
}
}

1
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt
View File

@@ -20,5 +20,4 @@ data class RpcUnspent(
val vout: Int,
val address: String,
val amount: Long,
val confirmations: Long
)

1
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt
View File

@@ -30,7 +30,6 @@ class RpcUnspentDeserializer : JsonDeserializer<RpcUnspent>() {
node.get("vout").asInt(),
node.get("address").asText(),
BigDecimal(node.get("amount").asText()).multiply(BigDecimal.TEN.pow(8)).longValueExact(),
node.get("confirmations").asLong()
)
}
}

1
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt
View File

@@ -19,5 +19,4 @@ data class SimpleUnspent(
val txid: String,
val vout: Int,
val value: Long,
val confirmations: Long
)

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt
View File

@@ -99,6 +99,10 @@ class BitcoinGrpcUpstream(
var timeout = Defaults.timeout
private var capabilities: Set<Capability> = emptySet()
override fun getBlockchainApi(): ReactorBlockchainGrpc.ReactorBlockchainStub {
return remote
}
override fun getHead(): Head {
return grpcHead
}

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt
View File

@@ -103,6 +103,10 @@ open class EthereumGrpcUpstream(
private val defaultReader: Reader<JsonRpcRequest, JsonRpcResponse> = client.forSelector(Selector.empty)
var timeout = Defaults.timeout
override fun getBlockchainApi(): ReactorBlockchainGrpc.ReactorBlockchainStub {
return remote
}
override fun start() {
}

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt
View File

@@ -16,12 +16,16 @@
package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.upstream.Upstream
interface GrpcUpstream {
interface GrpcUpstream : Upstream {
/**
* Update the configuration of the upstream with the new data.
* Called on the first creation, and each time a new state received from upstream
*/
fun update(conf: BlockchainOuterClass.DescribeChain)
fun getBlockchainApi(): ReactorBlockchainGrpc.ReactorBlockchainStub
}

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt
View File

@@ -91,7 +91,7 @@ class GrpcUpstreams(
client.describe(BlockchainOuterClass.DescribeRequest.newBuilder().build())
}.onErrorContinue { t, _ ->
if (ExceptionUtils.indexOfType(t, ConnectException::class.java) >= 0) {
log.warn("gRPC upstream $host:$port is unavailable")
log.warn("gRPC upstream $host:$port is unavailable. (${t.javaClass}: ${t.message})")
known.values.forEach {
it.setStatus(UpstreamAvailability.UNAVAILABLE)
}