Support ton v3 (#585)

src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt

@@ -73,7 +73,17 @@ data class UpstreamsConfig(
         var rpc: HttpEndpoint? = null,
         var ws: WsEndpoint? = null,
         var connectorMode: String? = null,
+        var tag: String? = null,
     ) : UpstreamConnection() {
+        private val additionalEndpoints = ArrayList<RpcConnection>()
+
+        fun addEndpoint(newConnection: RpcConnection) {
+            additionalEndpoints.add(newConnection)
+        }
+
+        fun getEndpointByTag(tag: String): RpcConnection? {
+            return additionalEndpoints.find { it.tag == tag }
+        }
 
         fun resolveMode(): ConnectorMode {
             return if (connectorMode == null) {

src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt

@@ -195,6 +195,17 @@ class UpstreamsConfigReader(
         getValueAsString(connConfigNode, "connector-mode")?.let {
             connection.connectorMode = it
         }
+
+        getList<MappingNode>(connConfigNode, "additional")
+            ?.value
+            ?.mapNotNull {
+                connection.addEndpoint(readRpcConnection(it))
+            }
+
+        getValueAsString(connConfigNode, "tag")?.let {
+            connection.tag = it
+        }
+
         getMapping(connConfigNode, "ws")?.let { node ->
             getValueAsString(node, "url")?.let { url ->
                 val ws = UpstreamsConfig.WsEndpoint(URI(url))
@@ -230,14 +241,14 @@ class UpstreamsConfigReader(
     private fun <T : UpstreamsConfig.UpstreamConnection> readUpstream(
         config: UpstreamsConfig,
         upNode: MappingNode,
-        connFactory: () -> T,
+        connFactory: (String?) -> T,
     ) {
         val upstream = UpstreamsConfig.Upstream<T>()
         readUpstreamCommon(upNode, upstream)
         readUpstreamStandard(upNode, upstream)
         if (isValid(upstream)) {
             config.upstreams.add(upstream)
-            upstream.connection = connFactory()
+            upstream.connection = connFactory(upstream.chain)
         } else {
             log.error("Upstream at #0 has invalid configuration")
         }

src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt

@@ -1,10 +1,12 @@
 package io.emeraldpay.dshackle.startup.configure
 
+import io.emeraldpay.dshackle.BlockchainType
 import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.FileResolver
 import io.emeraldpay.dshackle.config.ChainsConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.upstream.BlockValidator
+import io.emeraldpay.dshackle.upstream.TonCompoundHttpFactory
 import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
 import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory
 import io.emeraldpay.dshackle.upstream.generic.connectors.RestConnectorFactory
@@ -35,11 +37,17 @@ class RestConnectorFactoryCreator(
     ): ConnectorFactory? {
         val urls = ArrayList<URI>()
         val httpFactory = buildHttpFactory(conn.rpc, urls)
+        val tonV3HttpFactory = buildHttpFactory(conn.getEndpointByTag("ton_v3")?.rpc, urls)
+        val upstreamHttpFactory = if (httpFactory != null && chain.type == BlockchainType.TON) {
+            TonCompoundHttpFactory(httpFactory, tonV3HttpFactory)
+        } else {
+            httpFactory
+        }
         log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
 
         val connectorFactory =
             RestConnectorFactory(
-                httpFactory,
+                upstreamHttpFactory,
                 forkChoice,
                 blockValidator,
                 headScheduler,

src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt

@@ -83,7 +83,7 @@ abstract class UpstreamCreator(
             }
 
             ManagedCallMethods(
-                delegate = callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options),
+                delegate = callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options, config.connection),
                 enabled = config.methods?.enabled?.map { it.name }?.toSet() ?: emptySet(),
                 disabled = config.methods?.disabled?.map { it.name }?.toSet() ?: emptySet(),
                 groupsEnabled = config.methodGroups?.enabled ?: emptySet(),
@@ -99,7 +99,7 @@ abstract class UpstreamCreator(
                 }
             }
         } else {
-            callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options)
+            callTargets.getDefaultMethods(chain, indexConfig.isChainEnabled(chain), options, config.connection)
         }
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt

@@ -11,6 +11,7 @@ import io.emeraldpay.dshackle.BlockchainType.STARKNET
 import io.emeraldpay.dshackle.BlockchainType.TON
 import io.emeraldpay.dshackle.BlockchainType.UNKNOWN
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.foundation.ChainOptions
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.calls.DefaultBeaconChainMethods
@@ -26,11 +27,21 @@ import org.springframework.stereotype.Component
 class CallTargetsHolder {
     private val callTargets = HashMap<Chain, CallMethods>()
 
-    fun getDefaultMethods(chain: Chain, hasLogsOracle: Boolean, options: ChainOptions.Options): CallMethods {
+    fun getDefaultMethods(
-        return callTargets[chain] ?: return setupDefaultMethods(chain, hasLogsOracle, options)
+        chain: Chain,
+        hasLogsOracle: Boolean,
+        options: ChainOptions.Options,
+        connection: UpstreamsConfig.UpstreamConnection?,
+    ): CallMethods {
+        return callTargets[chain] ?: return setupDefaultMethods(chain, hasLogsOracle, options, connection)
     }
 
-    private fun setupDefaultMethods(chain: Chain, hasLogsOracle: Boolean, options: ChainOptions.Options): CallMethods {
+    private fun setupDefaultMethods(
+        chain: Chain,
+        hasLogsOracle: Boolean,
+        options: ChainOptions.Options,
+        connection: UpstreamsConfig.UpstreamConnection?,
+    ): CallMethods {
         val created = when (chain.type) {
             BITCOIN -> DefaultBitcoinMethods(options.providesBalance == true)
             ETHEREUM -> DefaultEthereumMethods(chain, hasLogsOracle)
@@ -40,7 +51,7 @@ class CallTargetsHolder {
             NEAR -> DefaultNearMethods()
             ETHEREUM_BEACON_CHAIN -> DefaultBeaconChainMethods()
             COSMOS -> DefaultCosmosMethods()
-            TON -> DefaultTonHttpMethods()
+            TON -> DefaultTonHttpMethods(connection)
             UNKNOWN -> throw IllegalArgumentException("unknown chain")
         }
         callTargets[chain] = created

src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt

@@ -22,11 +22,13 @@ import java.util.function.Function
 
 abstract class HttpReader(
     protected val target: String,
-    protected val metrics: RequestMetrics,
+    protected val metrics: RequestMetrics?,
     basicAuth: AuthConfig.ClientBasicAuth? = null,
     tlsCAAuth: ByteArray? = null,
 ) : ChainReader {
 
+    constructor() : this("", null)
+
     protected val httpClient: HttpClient
 
     init {
@@ -76,10 +78,12 @@ abstract class HttpReader(
 
     protected abstract fun internalRead(key: ChainRequest): Mono<ChainResponse>
 
-    fun onStop() {
+    open fun onStop() {
+        if (metrics != null) {
             Metrics.globalRegistry.remove(metrics.timer)
             Metrics.globalRegistry.remove(metrics.fails)
         }
+    }
 
     /**
      * The subscribers expect to catch an exception if the response contains JSON RPC Error. Convert it here to JsonRpcException
@@ -108,7 +112,7 @@ abstract class HttpReader(
                     else -> ChainException(key.id, t.message ?: t.javaClass.name, cause = t)
                 }
                 // here we're measure the internal errors, not upstream errors
-                metrics.fails.increment()
+                metrics?.fails?.increment()
                 Mono.error(err)
             }
         }

src/main/kotlin/io/emeraldpay/dshackle/upstream/TonCompoundHttpFactory.kt

@@ -0,0 +1,18 @@
+package io.emeraldpay.dshackle.upstream
+
+import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.upstream.restclient.TonCompoundRestHttpReader
+
+class TonCompoundHttpFactory(
+    private val tonHttpFactory: HttpFactory,
+    private val tonV3HttpFactory: HttpFactory?,
+) : HttpFactory {
+
+    override fun create(id: String?, chain: Chain): HttpReader {
+        val tonReader = tonHttpFactory.create(id, chain)
+        if (tonV3HttpFactory != null) {
+            return TonCompoundRestHttpReader(tonReader, tonV3HttpFactory.create(id, chain))
+        }
+        return tonReader
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultTonHttpMethods.kt

@@ -1,10 +1,13 @@
 package io.emeraldpay.dshackle.upstream.calls
 
+import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.quorum.AlwaysQuorum
 import io.emeraldpay.dshackle.quorum.CallQuorum
 import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
 
-class DefaultTonHttpMethods : CallMethods {
+class DefaultTonHttpMethods(
+    private val upstreamConnection: UpstreamsConfig.UpstreamConnection?,
+) : CallMethods {
 
     // HTTP API section
     private val accountHttpMethods = setOf(
@@ -59,13 +62,67 @@ class DefaultTonHttpMethods : CallMethods {
         postMethod("/jsonRPC"),
     )
 
+    // indexer v3 methods
+
+    private val indexerAccountsMethods = setOf(
+        getMethod("/api/v3/accountStates"),
+        getMethod("/api/v3/addressBook"),
+        getMethod("/api/v3/walletStates"),
+    )
+
+    private val indexerEventsMethods = setOf(
+        getMethod("/api/v3/actions"),
+        getMethod("/api/v3/events"),
+    )
+
+    private val indexerApiV2Methods = setOf(
+        getMethod("/api/v3/addressInformation"),
+        postMethod("/api/v3/estimateFee"),
+        postMethod("/api/v3/message"),
+        postMethod("/api/v3/runGetMethod"),
+        getMethod("/api/v3/walletInformation"),
+    )
+
+    private val indexerBlockchainMethods = setOf(
+        getMethod("/api/v3/adjacentTransactions"),
+        getMethod("/api/v3/blocks"),
+        getMethod("/api/v3/masterchainBlockShardState"),
+        getMethod("/api/v3/masterchainBlockShards"),
+        getMethod("/api/v3/masterchainInfo"),
+        getMethod("/api/v3/messages"),
+        getMethod("/api/v3/transactions"),
+        getMethod("/api/v3/transactionsByMasterchainBlock"),
+        getMethod("/api/v3/transactionsByMessage"),
+    )
+
+    private val indexerJettonsMethods = setOf(
+        getMethod("/api/v3/jetton/burns"),
+        getMethod("/api/v3/jetton/masters"),
+        getMethod("/api/v3/jetton/transfers"),
+        getMethod("/api/v3/jetton/wallets"),
+    )
+
+    private val indexerNftsMethods = setOf(
+        getMethod("/api/v3/nft/collections"),
+        getMethod("/api/v3/nft/items"),
+        getMethod("/api/v3/nft/transfers"),
+    )
+
+    private val indexerStatsMethods = setOf(
+        getMethod("/api/v3/topAccountsByBalance"),
+    )
+
+    private val indexerMethods = indexerAccountsMethods + indexerEventsMethods + indexerApiV2Methods +
+        indexerBlockchainMethods + indexerJettonsMethods + indexerNftsMethods + indexerStatsMethods
+
     private val allowedHttpMethods: Set<String> = accountHttpMethods +
         blockHttpMethods +
         transactionHttpMethods +
         getConfigHttpMethods +
         runMethodHttpMethods +
         sendHttpMethods +
-        jsonRpcHttpMethods
+        jsonRpcHttpMethods +
+        v3Methods()
 
     override fun createQuorumFor(method: String): CallQuorum {
         return AlwaysQuorum()
@@ -97,4 +154,12 @@ class DefaultTonHttpMethods : CallMethods {
     private fun getMethod(method: String) = "GET#$method"
 
     private fun postMethod(method: String) = "POST#$method"
+
+    private fun v3Methods(): Set<String> {
+        return if (upstreamConnection is UpstreamsConfig.RpcConnection && upstreamConnection.getEndpointByTag("ton_v3") != null) {
+            indexerMethods
+        } else {
+            emptySet()
+        }
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt

@@ -41,7 +41,7 @@ class RestHttpReader(
             .flatMap(this::execute)
             .doOnNext {
                 if (startTime.isStarted) {
-                    metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
+                    metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
                 }
             }
             .handle { it, sink ->

src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/TonCompoundRestHttpReader.kt

@@ -0,0 +1,28 @@
+package io.emeraldpay.dshackle.upstream.restclient
+
+import io.emeraldpay.dshackle.upstream.ChainRequest
+import io.emeraldpay.dshackle.upstream.ChainResponse
+import io.emeraldpay.dshackle.upstream.HttpReader
+import reactor.core.publisher.Mono
+
+class TonCompoundRestHttpReader(
+    private val restReader: HttpReader,
+    private val restReaderV3: HttpReader,
+) : HttpReader() {
+
+    override fun read(key: ChainRequest): Mono<ChainResponse> {
+        if (key.method.contains("v3")) {
+            return restReaderV3.read(key)
+        }
+        return restReader.read(key)
+    }
+
+    override fun internalRead(key: ChainRequest): Mono<ChainResponse> {
+        return Mono.empty()
+    }
+
+    override fun onStop() {
+        restReader.onStop()
+        restReaderV3.onStop()
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt

@@ -82,7 +82,7 @@ class JsonRpcHttpReader(
             .flatMap(this@JsonRpcHttpReader::execute)
             .doOnNext {
                 if (startTime.isStarted) {
-                    metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
+                    metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
                 }
             }
             .transform(asJsonRpcResponse(key))