From 75c74d987cf2e9293ee6d725f0602f660d54d892 Mon Sep 17 00:00:00 2001 From: KirillPamPam Date: Wed, 30 Oct 2024 19:57:15 +0400 Subject: [PATCH] Support ton v3 (#585) --- .../dshackle/config/UpstreamsConfig.kt | 10 +++ .../dshackle/config/UpstreamsConfigReader.kt | 15 +++- .../configure/RestConnectorFactoryCreator.kt | 10 ++- .../startup/configure/UpstreamCreator.kt | 4 +- .../dshackle/upstream/CallTargetsHolder.kt | 19 +++-- .../dshackle/upstream/HttpReader.kt | 14 ++-- .../upstream/TonCompoundHttpFactory.kt | 18 +++++ .../upstream/calls/DefaultTonHttpMethods.kt | 69 ++++++++++++++++++- .../upstream/restclient/RestHttpReader.kt | 2 +- .../restclient/TonCompoundRestHttpReader.kt | 28 ++++++++ .../upstream/rpcclient/JsonRpcHttpReader.kt | 2 +- 11 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt index c7e697ae..94c02abf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt @@ -73,7 +73,17 @@ data class UpstreamsConfig( var rpc: HttpEndpoint? = null, var ws: WsEndpoint? = null, var connectorMode: String? = null, + var tag: String? = null, ) : UpstreamConnection() { + private val additionalEndpoints = ArrayList() + + fun addEndpoint(newConnection: RpcConnection) { + additionalEndpoints.add(newConnection) + } + + fun getEndpointByTag(tag: String): RpcConnection? { + return additionalEndpoints.find { it.tag == tag } + } fun resolveMode(): ConnectorMode { return if (connectorMode == null) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 61717275..75b17b9a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -195,6 +195,17 @@ class UpstreamsConfigReader( getValueAsString(connConfigNode, "connector-mode")?.let { connection.connectorMode = it } + + getList(connConfigNode, "additional") + ?.value + ?.mapNotNull { + connection.addEndpoint(readRpcConnection(it)) + } + + getValueAsString(connConfigNode, "tag")?.let { + connection.tag = it + } + getMapping(connConfigNode, "ws")?.let { node -> getValueAsString(node, "url")?.let { url -> val ws = UpstreamsConfig.WsEndpoint(URI(url)) @@ -230,14 +241,14 @@ class UpstreamsConfigReader( private fun readUpstream( config: UpstreamsConfig, upNode: MappingNode, - connFactory: () -> T, + connFactory: (String?) -> T, ) { val upstream = UpstreamsConfig.Upstream() readUpstreamCommon(upNode, upstream) readUpstreamStandard(upNode, upstream) if (isValid(upstream)) { config.upstreams.add(upstream) - upstream.connection = connFactory() + upstream.connection = connFactory(upstream.chain) } else { log.error("Upstream at #0 has invalid configuration") } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt index a566bc6f..d040ad70 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt @@ -1,10 +1,12 @@ package io.emeraldpay.dshackle.startup.configure +import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.upstream.BlockValidator +import io.emeraldpay.dshackle.upstream.TonCompoundHttpFactory import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory import io.emeraldpay.dshackle.upstream.generic.connectors.RestConnectorFactory @@ -35,11 +37,17 @@ class RestConnectorFactoryCreator( ): ConnectorFactory? { val urls = ArrayList() val httpFactory = buildHttpFactory(conn.rpc, urls) + val tonV3HttpFactory = buildHttpFactory(conn.getEndpointByTag("ton_v3")?.rpc, urls) + val upstreamHttpFactory = if (httpFactory != null && chain.type == BlockchainType.TON) { + TonCompoundHttpFactory(httpFactory, tonV3HttpFactory) + } else { + httpFactory + } log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}") val connectorFactory = RestConnectorFactory( - httpFactory, + upstreamHttpFactory, forkChoice, blockValidator, headScheduler, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt index 314d848b..4b354f21 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt @@ -83,7 +83,7 @@ abstract class UpstreamCreator( } ManagedCallMethods( - delegate = callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options), + delegate = callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options, config.connection), enabled = config.methods?.enabled?.map { it.name }?.toSet() ?: emptySet(), disabled = config.methods?.disabled?.map { it.name }?.toSet() ?: emptySet(), groupsEnabled = config.methodGroups?.enabled ?: emptySet(), @@ -99,7 +99,7 @@ abstract class UpstreamCreator( } } } else { - callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options) + callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options, config.connection) } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt index dbe0237d..edc06ab3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt @@ -11,6 +11,7 @@ import io.emeraldpay.dshackle.BlockchainType.STARKNET import io.emeraldpay.dshackle.BlockchainType.TON import io.emeraldpay.dshackle.BlockchainType.UNKNOWN import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.DefaultBeaconChainMethods @@ -26,11 +27,21 @@ import org.springframework.stereotype.Component class CallTargetsHolder { private val callTargets = HashMap() - fun getDefaultMethods(chain: Chain, hasLogsOracle: Boolean, options: ChainOptions.Options): CallMethods { - return callTargets[chain] ?: return setupDefaultMethods(chain, hasLogsOracle, options) + fun getDefaultMethods( + chain: Chain, + hasLogsOracle: Boolean, + options: ChainOptions.Options, + connection: UpstreamsConfig.UpstreamConnection?, + ): CallMethods { + return callTargets[chain] ?: return setupDefaultMethods(chain, hasLogsOracle, options, connection) } - private fun setupDefaultMethods(chain: Chain, hasLogsOracle: Boolean, options: ChainOptions.Options): CallMethods { + private fun setupDefaultMethods( + chain: Chain, + hasLogsOracle: Boolean, + options: ChainOptions.Options, + connection: UpstreamsConfig.UpstreamConnection?, + ): CallMethods { val created = when (chain.type) { BITCOIN -> DefaultBitcoinMethods(options.providesBalance == true) ETHEREUM -> DefaultEthereumMethods(chain, hasLogsOracle) @@ -40,7 +51,7 @@ class CallTargetsHolder { NEAR -> DefaultNearMethods() ETHEREUM_BEACON_CHAIN -> DefaultBeaconChainMethods() COSMOS -> DefaultCosmosMethods() - TON -> DefaultTonHttpMethods() + TON -> DefaultTonHttpMethods(connection) UNKNOWN -> throw IllegalArgumentException("unknown chain") } callTargets[chain] = created diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt index 69cfaff9..af8deb96 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt @@ -22,11 +22,13 @@ import java.util.function.Function abstract class HttpReader( protected val target: String, - protected val metrics: RequestMetrics, + protected val metrics: RequestMetrics?, basicAuth: AuthConfig.ClientBasicAuth? = null, tlsCAAuth: ByteArray? = null, ) : ChainReader { + constructor() : this("", null) + protected val httpClient: HttpClient init { @@ -76,9 +78,11 @@ abstract class HttpReader( protected abstract fun internalRead(key: ChainRequest): Mono - fun onStop() { - Metrics.globalRegistry.remove(metrics.timer) - Metrics.globalRegistry.remove(metrics.fails) + open fun onStop() { + if (metrics != null) { + Metrics.globalRegistry.remove(metrics.timer) + Metrics.globalRegistry.remove(metrics.fails) + } } /** @@ -108,7 +112,7 @@ abstract class HttpReader( else -> ChainException(key.id, t.message ?: t.javaClass.name, cause = t) } // here we're measure the internal errors, not upstream errors - metrics.fails.increment() + metrics?.fails?.increment() Mono.error(err) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt new file mode 100644 index 00000000..92fe2151 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt @@ -0,0 +1,18 @@ +package io.emeraldpay.dshackle.upstream + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.upstream.restclient.TonCompoundRestHttpReader + +class TonCompoundHttpFactory( + private val tonHttpFactory: HttpFactory, + private val tonV3HttpFactory: HttpFactory?, +) : HttpFactory { + + override fun create(id: String?, chain: Chain): HttpReader { + val tonReader = tonHttpFactory.create(id, chain) + if (tonV3HttpFactory != null) { + return TonCompoundRestHttpReader(tonReader, tonV3HttpFactory.create(id, chain)) + } + return tonReader + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt index 1cc9c8bb..e47f6932 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt @@ -1,10 +1,13 @@ package io.emeraldpay.dshackle.upstream.calls +import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.quorum.AlwaysQuorum import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException -class DefaultTonHttpMethods : CallMethods { +class DefaultTonHttpMethods( + private val upstreamConnection: UpstreamsConfig.UpstreamConnection?, +) : CallMethods { // HTTP API section private val accountHttpMethods = setOf( @@ -59,13 +62,67 @@ class DefaultTonHttpMethods : CallMethods { postMethod("/jsonRPC"), ) + // indexer v3 methods + + private val indexerAccountsMethods = setOf( + getMethod("/api/v3/accountStates"), + getMethod("/api/v3/addressBook"), + getMethod("/api/v3/walletStates"), + ) + + private val indexerEventsMethods = setOf( + getMethod("/api/v3/actions"), + getMethod("/api/v3/events"), + ) + + private val indexerApiV2Methods = setOf( + getMethod("/api/v3/addressInformation"), + postMethod("/api/v3/estimateFee"), + postMethod("/api/v3/message"), + postMethod("/api/v3/runGetMethod"), + getMethod("/api/v3/walletInformation"), + ) + + private val indexerBlockchainMethods = setOf( + getMethod("/api/v3/adjacentTransactions"), + getMethod("/api/v3/blocks"), + getMethod("/api/v3/masterchainBlockShardState"), + getMethod("/api/v3/masterchainBlockShards"), + getMethod("/api/v3/masterchainInfo"), + getMethod("/api/v3/messages"), + getMethod("/api/v3/transactions"), + getMethod("/api/v3/transactionsByMasterchainBlock"), + getMethod("/api/v3/transactionsByMessage"), + ) + + private val indexerJettonsMethods = setOf( + getMethod("/api/v3/jetton/burns"), + getMethod("/api/v3/jetton/masters"), + getMethod("/api/v3/jetton/transfers"), + getMethod("/api/v3/jetton/wallets"), + ) + + private val indexerNftsMethods = setOf( + getMethod("/api/v3/nft/collections"), + getMethod("/api/v3/nft/items"), + getMethod("/api/v3/nft/transfers"), + ) + + private val indexerStatsMethods = setOf( + getMethod("/api/v3/topAccountsByBalance"), + ) + + private val indexerMethods = indexerAccountsMethods + indexerEventsMethods + indexerApiV2Methods + + indexerBlockchainMethods + indexerJettonsMethods + indexerNftsMethods + indexerStatsMethods + private val allowedHttpMethods: Set = accountHttpMethods + blockHttpMethods + transactionHttpMethods + getConfigHttpMethods + runMethodHttpMethods + sendHttpMethods + - jsonRpcHttpMethods + jsonRpcHttpMethods + + v3Methods() override fun createQuorumFor(method: String): CallQuorum { return AlwaysQuorum() @@ -97,4 +154,12 @@ class DefaultTonHttpMethods : CallMethods { private fun getMethod(method: String) = "GET#$method" private fun postMethod(method: String) = "POST#$method" + + private fun v3Methods(): Set { + return if (upstreamConnection is UpstreamsConfig.RpcConnection && upstreamConnection.getEndpointByTag("ton_v3") != null) { + indexerMethods + } else { + emptySet() + } + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt index 406bb7c9..fd387cc9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt @@ -41,7 +41,7 @@ class RestHttpReader( .flatMap(this::execute) .doOnNext { if (startTime.isStarted) { - metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS) + metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS) } } .handle { it, sink -> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt new file mode 100644 index 00000000..c86b90b8 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt @@ -0,0 +1,28 @@ +package io.emeraldpay.dshackle.upstream.restclient + +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.HttpReader +import reactor.core.publisher.Mono + +class TonCompoundRestHttpReader( + private val restReader: HttpReader, + private val restReaderV3: HttpReader, +) : HttpReader() { + + override fun read(key: ChainRequest): Mono { + if (key.method.contains("v3")) { + return restReaderV3.read(key) + } + return restReader.read(key) + } + + override fun internalRead(key: ChainRequest): Mono { + return Mono.empty() + } + + override fun onStop() { + restReader.onStop() + restReaderV3.onStop() + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt index a9646459..c4e8a710 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt @@ -82,7 +82,7 @@ class JsonRpcHttpReader( .flatMap(this@JsonRpcHttpReader::execute) .doOnNext { if (startTime.isStarted) { - metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS) + metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS) } } .transform(asJsonRpcResponse(key))