Some tunes (#549)

build.gradle

@@ -190,7 +190,7 @@ jib {
         }
     }
     container {
-        jvmFlags = ['-XX:+UseG1GC', '-XX:+ExitOnOutOfMemoryError', '-Xms1024M', '-XX:+UnlockDiagnosticVMOptions', '-XX:GCLockerRetryAllocationCount=10', '--enable-preview']
+        jvmFlags = ['-XX:+UseG1GC', '-XX:+ExitOnOutOfMemoryError', '-Xms1024M', '-XX:NativeMemoryTracking=summary', '-XX:+UnlockDiagnosticVMOptions', '-XX:GCLockerRetryAllocationCount=10', '--enable-preview']
         mainClass = 'io.emeraldpay.dshackle.StarterKt'
         args = []
         ports = ['2448', '2449', '8545']

src/main/kotlin/io/emeraldpay/dshackle/Defaults.kt

@@ -25,6 +25,7 @@ class Defaults {
         const val maxMetadataSize = 16384
         val timeout: Duration = Duration.ofSeconds(60)
         val timeoutInternal: Duration = timeout.dividedBy(4)
+        val internalCallsTimeout = Duration.ofSeconds(3)
         val retryConnection: Duration = Duration.ofSeconds(10)
         val grpcServerKeepAliveTime: Long = 15 // seconds
         val grpcServerKeepAliveTimeout: Long = 5

src/main/kotlin/io/emeraldpay/dshackle/config/context/SchedulersConfig.kt

@@ -12,7 +12,9 @@ import reactor.core.scheduler.Scheduler
 import reactor.core.scheduler.Schedulers
 import java.util.concurrent.Executor
 import java.util.concurrent.ExecutorService
-import java.util.concurrent.Executors
+import java.util.concurrent.SynchronousQueue
+import java.util.concurrent.ThreadPoolExecutor
+import java.util.concurrent.TimeUnit
 
 @Configuration
 open class SchedulersConfig {
@@ -79,17 +81,24 @@ open class SchedulersConfig {
     }
 
     private fun makePool(name: String, size: Int, monitoringConfig: MonitoringConfig): ExecutorService {
-        val pool = Executors.newFixedThreadPool(size, CustomizableThreadFactory("$name-"))
+        val cachedPool = ThreadPoolExecutor(
+            size,
+            size * threadsMultiplier,
+            60L,
+            TimeUnit.SECONDS,
+            SynchronousQueue(),
+            CustomizableThreadFactory("$name-"),
+        )
 
         return if (monitoringConfig.enableExtended) {
             ExecutorServiceMetrics.monitor(
                 Metrics.globalRegistry,
-                pool,
+                cachedPool,
                 name,
                 Tag.of("reactor_scheduler_id", "_"),
             )
         } else {
-            pool
+            cachedPool
         }
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRequestReader.kt

@@ -178,11 +178,11 @@ class QuorumRequestReader(
             src.onErrorResume { err ->
                 errorHandler.handle(api, key, err.message)
 
-                val msgError = "Error during call upstream ${api.getId()} with method ${key.method}"
+                val msgError = "Error during call upstream ${api.getId()} with method ${key.method}, reason - ${err.message}"
                 if (err is ChainCallUpstreamException) {
-                    log.debug(msgError, err)
+                    log.debug(msgError)
                 } else {
-                    log.warn(msgError, err)
+                    log.warn(msgError)
                 }
 
                 // when the call failed with an error we want to notify the quorum because

src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt

@@ -41,7 +41,7 @@ class CompoundReader<K, D> (
             .flatMap({ rdr ->
                 rdr.read(key)
                     .timeout(Defaults.timeoutInternal, Mono.empty())
-                    .doOnError { t -> log.warn("Failed to read from $rdr", t) }
+                    .doOnError { t -> log.warn("Failed to read from {}, reason - {}", rdr, t.message) }
             }, 1,)
             .next()
     }

src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt

@@ -15,6 +15,7 @@ import java.io.ByteArrayInputStream
 import java.security.KeyStore
 import java.security.cert.CertificateFactory
 import java.security.cert.X509Certificate
+import java.time.Duration
 import java.util.Base64
 import java.util.function.Consumer
 import java.util.function.Function
@@ -31,7 +32,8 @@ abstract class HttpReader(
     init {
         val connectionProvider = ConnectionProvider.builder("dshackleConnectionPool")
             .maxConnections(1500)
-            .pendingAcquireMaxCount(10000)
+            .pendingAcquireMaxCount(1000)
+            .pendingAcquireTimeout(Duration.ofSeconds(10))
             .build()
 
         var build = HttpClient.create(connectionProvider)

src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetector.kt

@@ -3,6 +3,7 @@ package io.emeraldpay.dshackle.upstream
 import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.Defaults.Companion.internalCallsTimeout
 import io.emeraldpay.dshackle.Global
 import org.slf4j.LoggerFactory
 import reactor.core.publisher.Flux
@@ -17,15 +18,25 @@ abstract class UpstreamSettingsDetector(
 ) {
     protected val log = LoggerFactory.getLogger(this::class.java)
 
-    abstract fun detectLabels(): Flux<Pair<String, String>>
+    fun detectLabels(): Flux<Pair<String, String>> {
+        return internalDetectLabels()
+            .timeout(internalCallsTimeout)
+            .onErrorResume {
+                log.warn("Couldn't detect lables of upstream {}, message - {}", upstream.getId(), it.message)
+                Flux.empty()
+            }
+    }
+
+    protected abstract fun internalDetectLabels(): Flux<Pair<String, String>>
 
     fun detectClientVersion(): Mono<String> {
         return upstream.getIngressReader()
             .read(clientVersionRequest())
             .flatMap(ChainResponse::requireResult)
             .map(::parseClientVersion)
+            .timeout(internalCallsTimeout)
             .onErrorResume {
-                log.warn("Can't detect the client version of upstream ${upstream.getId()}, reason - {}", it.message)
+                log.warn("Can't detect the client version of upstream {}, reason - {}", upstream.getId(), it.message)
                 Mono.just(UNKNOWN_CLIENT_VERSION)
             }
     }

src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainUpstreamSettingsDetector.kt

@@ -21,7 +21,7 @@ class BeaconChainUpstreamSettingsDetector(
         )
     }
 
-    override fun detectLabels(): Flux<Pair<String, String>> {
+    override fun internalDetectLabels(): Flux<Pair<String, String>> {
         return Flux.merge(
             detectNodeType(),
         )

src/main/kotlin/io/emeraldpay/dshackle/upstream/cosmos/CosmosLowerBoundService.kt

@@ -1,6 +1,7 @@
 package io.emeraldpay.dshackle.upstream.cosmos
 
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.Upstream
@@ -31,10 +32,13 @@ class CosmosLowerBoundStateDetector(
     }
 
     override fun internalDetectLowerBound(): Flux<LowerBoundData> {
-        return upstream.getIngressReader().read(ChainRequest("status", ListParams())).map {
+        return upstream.getIngressReader()
-            val resp = Global.objectMapper.readValue(it.getResult(), CosmosStatus::class.java)
+            .read(ChainRequest("status", ListParams()))
-            LowerBoundData(resp.syncInfo.earliestBlockHeight.toLong(), STATE)
+            .timeout(Defaults.internalCallsTimeout)
-        }.toFlux()
+            .map {
+                val resp = Global.objectMapper.readValue(it.getResult(), CosmosStatus::class.java)
+                LowerBoundData(resp.syncInfo.earliestBlockHeight.toLong(), STATE)
+            }.toFlux()
     }
 
     override fun types(): Set<LowerBoundType> {

src/main/kotlin/io/emeraldpay/dshackle/upstream/cosmos/CosmosUpstreamSettingsDetector.kt

@@ -14,7 +14,7 @@ import reactor.core.publisher.Flux
 class CosmosUpstreamSettingsDetector(
     upstream: Upstream,
 ) : BasicUpstreamSettingsDetector(upstream) {
-    override fun detectLabels(): Flux<Pair<String, String>> {
+    override fun internalDetectLabels(): Flux<Pair<String, String>> {
         return Flux.merge(
             detectNodeType(),
         )

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFinalizationDetector.kt

@@ -2,6 +2,7 @@ package io.emeraldpay.dshackle.upstream.ethereum
 
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.Upstream
@@ -72,6 +73,7 @@ class EthereumFinalizationDetector : FinalizationDetector {
                         upstream
                             .getIngressReader()
                             .read(req)
+                            .timeout(Defaults.internalCallsTimeout)
                             .flatMap {
                                 it.requireResult().flatMap { result ->
                                     val block = Global.objectMapper.readValue<BlockJson<TransactionRefJson>>(result)

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundBlockDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.ethereum
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.upstream.ChainCallError
 import io.emeraldpay.dshackle.upstream.ChainCallUpstreamException
 import io.emeraldpay.dshackle.upstream.ChainRequest
@@ -45,6 +46,7 @@ class EthereumLowerBoundBlockDetector(
                             ListParams(block.toHex(), false),
                         ),
                     )
+                    .timeout(Defaults.internalCallsTimeout)
                     .doOnNext {
                         if (it.hasResult() && it.getResult().contentEquals("null".toByteArray())) {
                             throw IllegalStateException(NO_BLOCK_DATA)

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundLogsDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.ethereum
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
@@ -39,6 +40,7 @@ class EthereumLowerBoundLogsDetector(
                         ),
                     ),
                 )
+                .timeout(Defaults.internalCallsTimeout)
                 .doOnNext {
                     if (it.hasResult() && (it.getResult().contentEquals("null".toByteArray()) || it.getResult().contentEquals("[]".toByteArray()))) {
                         throw IllegalStateException(NO_LOGS_DATA)

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundStateDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.ethereum
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.ChainResponse
@@ -60,7 +61,7 @@ class EthereumLowerBoundStateDetector(
                         "eth_getBalance",
                         ListParams(ZERO_ADDRESS, block.toHex()),
                     ),
-                )
+                ).timeout(Defaults.internalCallsTimeout)
             }.doOnNext {
                 if (it.hasResult() && it.getResult().contentEquals(Global.nullValue)) {
                     throw IllegalStateException("No state data")

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.ethereum
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.data.BlockContainer
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.Upstream
@@ -32,6 +33,7 @@ class EthereumLowerBoundTxDetector(
                 .read(
                     ChainRequest("eth_getBlockByNumber", ListParams(block.toHex(), false)),
                 )
+                .timeout(Defaults.internalCallsTimeout)
                 .doOnNext {
                     if (it.hasResult() && it.getResult().contentEquals("null".toByteArray())) {
                         throw IllegalStateException(NO_TX_DATA)
@@ -50,6 +52,7 @@ class EthereumLowerBoundTxDetector(
                         .read(
                             ChainRequest("eth_getTransactionByHash", ListParams(tx)),
                         )
+                        .timeout(Defaults.internalCallsTimeout)
                         .doOnNext {
                             if (it.hasResult() && it.getResult().contentEquals("null".toByteArray())) {
                                 throw IllegalStateException(NO_TX_DATA)

src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetector.kt

@@ -11,7 +11,6 @@ import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
 import reactor.core.publisher.Flux
 import reactor.core.publisher.Mono
-import kotlin.text.toBigInteger
 
 const val ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"
 
@@ -21,7 +20,7 @@ class EthereumUpstreamSettingsDetector(
 ) : BasicEthUpstreamSettingsDetector(_upstream) {
     private val blockNumberReader = EthereumArchiveBlockNumberReader(upstream.getIngressReader())
 
-    override fun detectLabels(): Flux<Pair<String, String>> {
+    override fun internalDetectLabels(): Flux<Pair<String, String>> {
         return Flux.merge(
             detectNodeType(),
             detectArchiveNode(),

src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/AggregatedUpstreamValidator.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.generic
 
+import io.emeraldpay.dshackle.Defaults.Companion.internalCallsTimeout
 import io.emeraldpay.dshackle.foundation.ChainOptions
 import io.emeraldpay.dshackle.upstream.SingleValidator
 import io.emeraldpay.dshackle.upstream.Upstream
@@ -21,8 +22,9 @@ class AggregatedUpstreamValidator(
         ) { a -> a.map { it as UpstreamAvailability } }
             .map(::resolve)
             .defaultIfEmpty(UpstreamAvailability.OK) // upstream is OK on case there are no validators
+            .timeout(internalCallsTimeout)
             .onErrorResume {
-                log.error("Error during upstream validation for ${upstream.getId()}", it)
+                log.error("Error during upstream validation for {}, reason - {}", upstream.getId(), it.message)
                 Mono.just(UpstreamAvailability.UNAVAILABLE)
             }
     }
@@ -33,13 +35,16 @@ class AggregatedUpstreamValidator(
         ) { a -> a.map { it as ValidateUpstreamSettingsResult } }
             .map(::resolve)
             .defaultIfEmpty(ValidateUpstreamSettingsResult.UPSTREAM_VALID)
+            .timeout(internalCallsTimeout)
             .onErrorResume {
-                log.error("Error during upstream validation for ${upstream.getId()}", it)
+                log.error("Error during upstream validation for {}, reason - {}", upstream.getId(), it.message)
                 Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR)
             }
     }
 
     override fun validateUpstreamSettingsOnStartup(): ValidateUpstreamSettingsResult {
-        return validateUpstreamSettings().block() ?: ValidateUpstreamSettingsResult.UPSTREAM_VALID
+        return runCatching {
+            validateUpstreamSettings().block(internalCallsTimeout) ?: ValidateUpstreamSettingsResult.UPSTREAM_VALID
+        }.getOrDefault(ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR)
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericRpcHead.kt

@@ -46,12 +46,15 @@ class GenericRpcHead(
     override fun start() {
         super.start()
         refreshSubscription?.dispose()
+
         val base = Flux.interval(interval)
+            .onBackpressureDrop()
             .publishOn(headScheduler)
             .filter { !isSyncing }
-            .flatMap {
+            .concatMap {
                 getLatestBlock(api)
             }
+
         refreshSubscription = super.follow(base)
     }
 

src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt

@@ -1,6 +1,7 @@
 package io.emeraldpay.dshackle.upstream.generic
 
 import io.emeraldpay.dshackle.Chain
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.config.ChainsConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig.Labels
@@ -35,10 +36,13 @@ import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundServiceBuilder
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
 import org.springframework.context.Lifecycle
+import org.springframework.scheduling.concurrent.CustomizableThreadFactory
 import reactor.core.Disposable
 import reactor.core.publisher.Flux
 import reactor.core.publisher.Sinks
+import reactor.core.scheduler.Schedulers
 import java.time.Duration
+import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 import java.util.function.Supplier
@@ -192,6 +196,7 @@ open class GenericUpstream(
                 },
                 headLivenessState.asFlux(),
             )
+                .subscribeOn(upstreamSettingsScheduler)
                 .distinctUntilChanged()
                 .subscribe {
                     when (it) {
@@ -238,29 +243,35 @@ open class GenericUpstream(
                         clientVersion.set(it)
                     },
             )
+                .subscribeOn(settingsScheduler)
         }.subscribe()
     }
 
     private fun detectRpcModules(config: UpstreamsConfig.Upstream<*>, buildMethods: (UpstreamsConfig.Upstream<*>, Chain) -> CallMethods) {
-        val rpcDetector = rpcModulesDetector?.detectRpcModules()?.block() ?: HashMap<String, String>()
+        try {
-        log.info("Upstream rpc detector for  ${getId()} returned  $rpcDetector ")
+            val rpcDetector = rpcModulesDetector?.detectRpcModules()?.block(Defaults.internalCallsTimeout)
-        if (rpcDetector.size != 0) {
+                ?: HashMap<String, String>()
-            var changed = false
+            log.info("Upstream rpc detector for  ${getId()} returned  $rpcDetector ")
-            for ((group, _) in rpcDetector) {
+            if (rpcDetector.size != 0) {
-                if (group == "trace" || group == "debug" || group == "filter") {
+                var changed = false
-                    if (config.methodGroups == null) {
+                for ((group, _) in rpcDetector) {
-                        config.methodGroups = UpstreamsConfig.MethodGroups(setOf("filter"), setOf())
+                    if (group == "trace" || group == "debug" || group == "filter") {
-                    } else {
+                        if (config.methodGroups == null) {
-                        val disabled = config.methodGroups!!.disabled
+                            config.methodGroups = UpstreamsConfig.MethodGroups(setOf("filter"), setOf())
-                        val enabled = config.methodGroups!!.enabled
+                        } else {
-                        if (!disabled.contains(group) && !enabled.contains(group)) {
+                            val disabled = config.methodGroups!!.disabled
-                            config.methodGroups!!.enabled = enabled.plus(group)
+                            val enabled = config.methodGroups!!.enabled
-                            changed = true
+                            if (!disabled.contains(group) && !enabled.contains(group)) {
+                                config.methodGroups!!.enabled = enabled.plus(group)
+                                changed = true
+                            }
                         }
                     }
                 }
+                if (changed) updateMethods(buildMethods(config, getChain()))
             }
-            if (changed) updateMethods(buildMethods(config, getChain()))
+        } catch (e: RuntimeException) {
+            log.error("Couldn't detect rpc modules of upstream {} due to error {}", getId(), e.message)
         }
     }
 
@@ -331,14 +342,17 @@ open class GenericUpstream(
 
     private fun detectFinalization() {
         finalizationDetectorSubscription =
-            finalizationDetector.detectFinalization(this, chainConfig.expectedBlockTime, getChain()).subscribe {
+            finalizationDetector.detectFinalization(this, chainConfig.expectedBlockTime, getChain())
-                sendUpstreamStateEvent(UPDATED)
+                .subscribeOn(finalizationScheduler)
-            }
+                .subscribe {
+                    sendUpstreamStateEvent(UPDATED)
+                }
     }
 
     private fun detectLowerBlock() {
         lowerBlockDetectorSubscription =
             lowerBoundService.detectLowerBounds()
+                .subscribeOn(lowerScheduler)
                 .subscribe {
                     sendUpstreamStateEvent(UPDATED)
                 }
@@ -359,4 +373,15 @@ open class GenericUpstream(
     }
 
     fun isValid(): Boolean = isUpstreamValid.get()
+
+    companion object {
+        private val upstreamSettingsScheduler =
+            Schedulers.fromExecutor(Executors.newFixedThreadPool(4, CustomizableThreadFactory("upstreamSettings-")))
+        private val finalizationScheduler =
+            Schedulers.fromExecutor(Executors.newFixedThreadPool(4, CustomizableThreadFactory("finalization-")))
+        private val settingsScheduler =
+            Schedulers.fromExecutor(Executors.newFixedThreadPool(4, CustomizableThreadFactory("settings-")))
+        private val lowerScheduler =
+            Schedulers.fromExecutor(Executors.newFixedThreadPool(4, CustomizableThreadFactory("lowerBound-")))
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearLowerBoundStateDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.near
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.Upstream
@@ -18,17 +19,20 @@ class NearLowerBoundStateDetector(
     }
 
     override fun internalDetectLowerBound(): Flux<LowerBoundData> {
-        return upstream.getIngressReader().read(ChainRequest("status", ListParams())).map {
+        return upstream.getIngressReader()
-            val resp = Global.objectMapper.readValue(it.getResult(), NearStatus::class.java)
+            .read(ChainRequest("status", ListParams()))
-            resp.syncInfo.earliestHeight
+            .timeout(Defaults.internalCallsTimeout)
-        }.flatMapMany {
+            .map {
-            Flux.fromIterable(
+                val resp = Global.objectMapper.readValue(it.getResult(), NearStatus::class.java)
-                listOf(
+                resp.syncInfo.earliestHeight
-                    LowerBoundData(it, LowerBoundType.STATE),
+            }.flatMapMany {
-                    LowerBoundData(it, LowerBoundType.BLOCK),
+                Flux.fromIterable(
-                ),
+                    listOf(
-            )
+                        LowerBoundData(it, LowerBoundType.STATE),
-        }
+                        LowerBoundData(it, LowerBoundType.BLOCK),
+                    ),
+                )
+            }
     }
 
     override fun types(): Set<LowerBoundType> {

src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearUpstreamSettingsDetector.kt

@@ -16,7 +16,7 @@ import reactor.core.publisher.Flux
 class NearUpstreamSettingsDetector(
     upstream: Upstream,
 ) : BasicUpstreamSettingsDetector(upstream) {
-    override fun detectLabels(): Flux<Pair<String, String>> {
+    override fun internalDetectLabels(): Flux<Pair<String, String>> {
         return Flux.merge(
             detectNodeType(),
         )

src/main/kotlin/io/emeraldpay/dshackle/upstream/polkadot/PolkadotLowerBoundStateDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.polkadot
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.ChainResponse
 import io.emeraldpay.dshackle.upstream.Upstream
@@ -34,6 +35,7 @@ class PolkadotLowerBoundStateDetector(
                     ListParams(block.toHex()), // in polkadot state methods work only with hash
                 ),
             )
+                .timeout(Defaults.internalCallsTimeout)
                 .flatMap(ChainResponse::requireResult)
                 .map {
                     String(it, 1, it.size - 2)
@@ -44,7 +46,7 @@ class PolkadotLowerBoundStateDetector(
                             "state_getMetadata",
                             ListParams(it),
                         ),
-                    )
+                    ).timeout(Defaults.internalCallsTimeout)
                 }
         }
     }

src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundSlotDetector.kt

@@ -1,5 +1,6 @@
 package io.emeraldpay.dshackle.upstream.solana
 
+import io.emeraldpay.dshackle.Defaults
 import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.upstream.ChainRequest
 import io.emeraldpay.dshackle.upstream.ChainResponse
@@ -28,7 +29,7 @@ class SolanaLowerBoundSlotDetector(
             .flatMap {
                 it.read(
                     ChainRequest("getFirstAvailableBlock", ListParams()), // in case of solana we talk about the slot of the lowest confirmed block
-                )
+                ).timeout(Defaults.internalCallsTimeout)
             }
             .flatMap(ChainResponse::requireResult)
             .map {
@@ -53,6 +54,7 @@ class SolanaLowerBoundSlotDetector(
                         ),
                     ),
                 )
+                    .timeout(Defaults.internalCallsTimeout)
                     .flatMap(ChainResponse::requireResult)
                     .flatMapMany { blockData ->
                         val block = Global.objectMapper.readValue(blockData, SolanaBlock::class.java)

src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaUpstreamSettingsDetector.kt

@@ -16,7 +16,7 @@ import reactor.core.publisher.Flux
 class SolanaUpstreamSettingsDetector(
     upstream: Upstream,
 ) : BasicUpstreamSettingsDetector(upstream) {
-    override fun detectLabels(): Flux<Pair<String, String>> {
+    override fun internalDetectLabels(): Flux<Pair<String, String>> {
         return Flux.merge(
             detectNodeType(),
         )

src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetectorSpec.groovy

@@ -40,7 +40,7 @@ class EthereumUpstreamSettingsDetectorSpec extends Specification {
         def detector = new EthereumUpstreamSettingsDetector(up, Chain.ETHEREUM__MAINNET)
 
         when:
-        def act = detector.detectLabels()
+        def act = detector.internalDetectLabels()
         then:
         StepVerifier.create(act)
             .expectNext(
@@ -72,7 +72,7 @@ class EthereumUpstreamSettingsDetectorSpec extends Specification {
         def detector = new EthereumUpstreamSettingsDetector(up, Chain.ETHEREUM__MAINNET)
 
         when:
-        def act = detector.detectLabels()
+        def act = detector.internalDetectLabels()
         then:
         StepVerifier.create(act)
                 .expectNext(
@@ -113,7 +113,7 @@ class EthereumUpstreamSettingsDetectorSpec extends Specification {
         }
         def detector = new EthereumUpstreamSettingsDetector(up, Chain.ETHEREUM__MAINNET)
         when:
-        def act = detector.detectLabels()
+        def act = detector.internalDetectLabels()
         then:
         StepVerifier.create(act)
             .expectNext(new Pair<String, String>("archive", "false"))

src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/GenericWsHeadTest.kt

@@ -444,10 +444,10 @@ class GenericWsHeadTest {
                 StepVerifier.create(wsHead.headLiveness())
                     .expectNext(HeadLivenessState.FATAL_ERROR)
                     .thenCancel()
-                    .verify(Duration.ofSeconds(1))
+                    .verify(Duration.ofSeconds(2))
             }
             .thenCancel()
-            .verify(Duration.ofSeconds(1))
+            .verify(Duration.ofSeconds(3))
 
         verify(connection, times(2)).callRpc(ChainRequest("eth_chainId", ListParams()))
         verify(connection, times(2)).callRpc(ChainRequest("net_version", ListParams()))