Support ton v3 (#585)

2024-10-30 19:57:15 +04:00
parent d628c3340e
commit 75c74d987c
11 changed files with 173 additions and 18 deletions
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt
@@ -73,7 +73,17 @@ data class UpstreamsConfig(
        var rpc: HttpEndpoint? = null,
        var ws: WsEndpoint? = null,
        var connectorMode: String? = null,
+        var tag: String? = null,
    ) : UpstreamConnection() {
+        private val additionalEndpoints = ArrayList<RpcConnection>()
+
+        fun addEndpoint(newConnection: RpcConnection) {
+            additionalEndpoints.add(newConnection)
+        }
+
+        fun getEndpointByTag(tag: String): RpcConnection? {
+            return additionalEndpoints.find { it.tag == tag }
+        }

        fun resolveMode(): ConnectorMode {
            return if (connectorMode == null) {
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt
@@ -195,6 +195,17 @@ class UpstreamsConfigReader(
        getValueAsString(connConfigNode, "connector-mode")?.let {
            connection.connectorMode = it
        }
+
+        getList<MappingNode>(connConfigNode, "additional")
+            ?.value
+            ?.mapNotNull {
+                connection.addEndpoint(readRpcConnection(it))
+            }
+
+        getValueAsString(connConfigNode, "tag")?.let {
+            connection.tag = it
+        }
+
        getMapping(connConfigNode, "ws")?.let { node ->
            getValueAsString(node, "url")?.let { url ->
                val ws = UpstreamsConfig.WsEndpoint(URI(url))
@@ -230,14 +241,14 @@ class UpstreamsConfigReader(
    private fun <T : UpstreamsConfig.UpstreamConnection> readUpstream(
        config: UpstreamsConfig,
        upNode: MappingNode,
-        connFactory: () -> T,
+        connFactory: (String?) -> T,
    ) {
        val upstream = UpstreamsConfig.Upstream<T>()
        readUpstreamCommon(upNode, upstream)
        readUpstreamStandard(upNode, upstream)
        if (isValid(upstream)) {
            config.upstreams.add(upstream)
-            upstream.connection = connFactory()
+            upstream.connection = connFactory(upstream.chain)
        } else {
            log.error("Upstream at #0 has invalid configuration")
        }
--- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt
@@ -1,10 +1,12 @@
 package io.emeraldpay.dshackle.startup.configure

+import io.emeraldpay.dshackle.BlockchainType
 import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.FileResolver
 import io.emeraldpay.dshackle.config.ChainsConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.upstream.BlockValidator
+import io.emeraldpay.dshackle.upstream.TonCompoundHttpFactory
 import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
 import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory
 import io.emeraldpay.dshackle.upstream.generic.connectors.RestConnectorFactory
@@ -35,11 +37,17 @@ class RestConnectorFactoryCreator(
    ): ConnectorFactory? {
        val urls = ArrayList<URI>()
        val httpFactory = buildHttpFactory(conn.rpc, urls)
+        val tonV3HttpFactory = buildHttpFactory(conn.getEndpointByTag("ton_v3")?.rpc, urls)
+        val upstreamHttpFactory = if (httpFactory != null && chain.type == BlockchainType.TON) {
+            TonCompoundHttpFactory(httpFactory, tonV3HttpFactory)
+        } else {
+            httpFactory
+        }
        log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")

        val connectorFactory =
            RestConnectorFactory(
-                httpFactory,
+                upstreamHttpFactory,
                forkChoice,
                blockValidator,
                headScheduler,
--- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt
@@ -83,7 +83,7 @@ abstract class UpstreamCreator(
            }

            ManagedCallMethods(
-                delegate = callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options),
+                delegate = callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options, config.connection),
                enabled = config.methods?.enabled?.map { it.name }?.toSet() ?: emptySet(),
                disabled = config.methods?.disabled?.map { it.name }?.toSet() ?: emptySet(),
                groupsEnabled = config.methodGroups?.enabled ?: emptySet(),
@@ -99,7 +99,7 @@ abstract class UpstreamCreator(
                }
            }
        } else {
-            callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options)
+            callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options, config.connection)
        }
    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt
@@ -11,6 +11,7 @@ import io.emeraldpay.dshackle.BlockchainType.STARKNET
 import io.emeraldpay.dshackle.BlockchainType.TON
 import io.emeraldpay.dshackle.BlockchainType.UNKNOWN
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.foundation.ChainOptions
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.calls.DefaultBeaconChainMethods
@@ -26,11 +27,21 @@ import org.springframework.stereotype.Component
 class CallTargetsHolder {
    private val callTargets = HashMap<Chain, CallMethods>()

-    fun getDefaultMethods(chain: Chain, hasLogsOracle: Boolean, options: ChainOptions.Options): CallMethods {
-        return callTargets[chain] ?: return setupDefaultMethods(chain, hasLogsOracle, options)
+    fun getDefaultMethods(
+        chain: Chain,
+        hasLogsOracle: Boolean,
+        options: ChainOptions.Options,
+        connection: UpstreamsConfig.UpstreamConnection?,
+    ): CallMethods {
+        return callTargets[chain] ?: return setupDefaultMethods(chain, hasLogsOracle, options, connection)
    }

-    private fun setupDefaultMethods(chain: Chain, hasLogsOracle: Boolean, options: ChainOptions.Options): CallMethods {
+    private fun setupDefaultMethods(
+        chain: Chain,
+        hasLogsOracle: Boolean,
+        options: ChainOptions.Options,
+        connection: UpstreamsConfig.UpstreamConnection?,
+    ): CallMethods {
        val created = when (chain.type) {
            BITCOIN -> DefaultBitcoinMethods(options.providesBalance == true)
            ETHEREUM -> DefaultEthereumMethods(chain, hasLogsOracle)
@@ -40,7 +51,7 @@ class CallTargetsHolder {
            NEAR -> DefaultNearMethods()
            ETHEREUM_BEACON_CHAIN -> DefaultBeaconChainMethods()
            COSMOS -> DefaultCosmosMethods()
-            TON -> DefaultTonHttpMethods()
+            TON -> DefaultTonHttpMethods(connection)
            UNKNOWN -> throw IllegalArgumentException("unknown chain")
        }
        callTargets[chain] = created
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt
@@ -22,11 +22,13 @@ import java.util.function.Function

 abstract class HttpReader(
    protected val target: String,
-    protected val metrics: RequestMetrics,
+    protected val metrics: RequestMetrics?,
    basicAuth: AuthConfig.ClientBasicAuth? = null,
    tlsCAAuth: ByteArray? = null,
 ) : ChainReader {

+    constructor() : this("", null)
+
    protected val httpClient: HttpClient

    init {
@@ -76,9 +78,11 @@ abstract class HttpReader(

    protected abstract fun internalRead(key: ChainRequest): Mono<ChainResponse>

-    fun onStop() {
-        Metrics.globalRegistry.remove(metrics.timer)
-        Metrics.globalRegistry.remove(metrics.fails)
+    open fun onStop() {
+        if (metrics != null) {
+            Metrics.globalRegistry.remove(metrics.timer)
+            Metrics.globalRegistry.remove(metrics.fails)
+        }
    }

    /**
@@ -108,7 +112,7 @@ abstract class HttpReader(
                    else -> ChainException(key.id, t.message ?: t.javaClass.name, cause = t)
                }
                // here we're measure the internal errors, not upstream errors
-                metrics.fails.increment()
+                metrics?.fails?.increment()
                Mono.error(err)
            }
        }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt
@@ -0,0 +1,18 @@
+package io.emeraldpay.dshackle.upstream
+
+import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.upstream.restclient.TonCompoundRestHttpReader
+
+class TonCompoundHttpFactory(
+    private val tonHttpFactory: HttpFactory,
+    private val tonV3HttpFactory: HttpFactory?,
+) : HttpFactory {
+
+    override fun create(id: String?, chain: Chain): HttpReader {
+        val tonReader = tonHttpFactory.create(id, chain)
+        if (tonV3HttpFactory != null) {
+            return TonCompoundRestHttpReader(tonReader, tonV3HttpFactory.create(id, chain))
+        }
+        return tonReader
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt
@@ -1,10 +1,13 @@
 package io.emeraldpay.dshackle.upstream.calls

+import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.quorum.AlwaysQuorum
 import io.emeraldpay.dshackle.quorum.CallQuorum
 import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException

-class DefaultTonHttpMethods : CallMethods {
+class DefaultTonHttpMethods(
+    private val upstreamConnection: UpstreamsConfig.UpstreamConnection?,
+) : CallMethods {

    // HTTP API section
    private val accountHttpMethods = setOf(
@@ -59,13 +62,67 @@ class DefaultTonHttpMethods : CallMethods {
        postMethod("/jsonRPC"),
    )

+    // indexer v3 methods
+
+    private val indexerAccountsMethods = setOf(
+        getMethod("/api/v3/accountStates"),
+        getMethod("/api/v3/addressBook"),
+        getMethod("/api/v3/walletStates"),
+    )
+
+    private val indexerEventsMethods = setOf(
+        getMethod("/api/v3/actions"),
+        getMethod("/api/v3/events"),
+    )
+
+    private val indexerApiV2Methods = setOf(
+        getMethod("/api/v3/addressInformation"),
+        postMethod("/api/v3/estimateFee"),
+        postMethod("/api/v3/message"),
+        postMethod("/api/v3/runGetMethod"),
+        getMethod("/api/v3/walletInformation"),
+    )
+
+    private val indexerBlockchainMethods = setOf(
+        getMethod("/api/v3/adjacentTransactions"),
+        getMethod("/api/v3/blocks"),
+        getMethod("/api/v3/masterchainBlockShardState"),
+        getMethod("/api/v3/masterchainBlockShards"),
+        getMethod("/api/v3/masterchainInfo"),
+        getMethod("/api/v3/messages"),
+        getMethod("/api/v3/transactions"),
+        getMethod("/api/v3/transactionsByMasterchainBlock"),
+        getMethod("/api/v3/transactionsByMessage"),
+    )
+
+    private val indexerJettonsMethods = setOf(
+        getMethod("/api/v3/jetton/burns"),
+        getMethod("/api/v3/jetton/masters"),
+        getMethod("/api/v3/jetton/transfers"),
+        getMethod("/api/v3/jetton/wallets"),
+    )
+
+    private val indexerNftsMethods = setOf(
+        getMethod("/api/v3/nft/collections"),
+        getMethod("/api/v3/nft/items"),
+        getMethod("/api/v3/nft/transfers"),
+    )
+
+    private val indexerStatsMethods = setOf(
+        getMethod("/api/v3/topAccountsByBalance"),
+    )
+
+    private val indexerMethods = indexerAccountsMethods + indexerEventsMethods + indexerApiV2Methods +
+        indexerBlockchainMethods + indexerJettonsMethods + indexerNftsMethods + indexerStatsMethods
+
    private val allowedHttpMethods: Set<String> = accountHttpMethods +
        blockHttpMethods +
        transactionHttpMethods +
        getConfigHttpMethods +
        runMethodHttpMethods +
        sendHttpMethods +
-        jsonRpcHttpMethods
+        jsonRpcHttpMethods +
+        v3Methods()

    override fun createQuorumFor(method: String): CallQuorum {
        return AlwaysQuorum()
@@ -97,4 +154,12 @@ class DefaultTonHttpMethods : CallMethods {
    private fun getMethod(method: String) = "GET#$method"

    private fun postMethod(method: String) = "POST#$method"
+
+    private fun v3Methods(): Set<String> {
+        return if (upstreamConnection is UpstreamsConfig.RpcConnection && upstreamConnection.getEndpointByTag("ton_v3") != null) {
+            indexerMethods
+        } else {
+            emptySet()
+        }
+    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt
@@ -41,7 +41,7 @@ class RestHttpReader(
            .flatMap(this::execute)
            .doOnNext {
                if (startTime.isStarted) {
-                    metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
+                    metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
                }
            }
            .handle { it, sink ->
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt
@@ -0,0 +1,28 @@
+package io.emeraldpay.dshackle.upstream.restclient
+
+import io.emeraldpay.dshackle.upstream.ChainRequest
+import io.emeraldpay.dshackle.upstream.ChainResponse
+import io.emeraldpay.dshackle.upstream.HttpReader
+import reactor.core.publisher.Mono
+
+class TonCompoundRestHttpReader(
+    private val restReader: HttpReader,
+    private val restReaderV3: HttpReader,
+) : HttpReader() {
+
+    override fun read(key: ChainRequest): Mono<ChainResponse> {
+        if (key.method.contains("v3")) {
+            return restReaderV3.read(key)
+        }
+        return restReader.read(key)
+    }
+
+    override fun internalRead(key: ChainRequest): Mono<ChainResponse> {
+        return Mono.empty()
+    }
+
+    override fun onStop() {
+        restReader.onStop()
+        restReaderV3.onStop()
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt
@@ -82,7 +82,7 @@ class JsonRpcHttpReader(
            .flatMap(this@JsonRpcHttpReader::execute)
            .doOnNext {
                if (startTime.isStarted) {
-                    metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
+                    metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
                }
            }
            .transform(asJsonRpcResponse(key))