New reactive endpoint (#514)

2024-07-01 15:49:35 +04:00
parent a5590f6603
commit eddaf98238
22 changed files with 1277 additions and 233 deletions
--- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt
@@ -52,6 +52,7 @@ class BlockchainRpc(
    @Autowired(required = false)
    private val providerSpanHandler: ProviderSpanHandler?,
    private val tracer: Tracer,
+    private val subscribeChainStatus: SubscribeChainStatus,
 ) : ReactorBlockchainGrpc.BlockchainImplBase() {

    private val log = LoggerFactory.getLogger(BlockchainRpc::class.java)
@@ -164,6 +165,12 @@ class BlockchainRpc(
        }
    }

+    override fun subscribeChainStatus(
+        request: Mono<BlockchainOuterClass.SubscribeChainStatusRequest>,
+    ): Flux<BlockchainOuterClass.SubscribeChainStatusResponse> {
+        return subscribeChainStatus.chainStatuses()
+    }
+
    class RequestMetrics(val chain: Chain) {
        val nativeCallMetric = Counter.builder("request.grpc.request")
            .tag("type", "nativeCall")
--- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/ChainEventMapper.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/ChainEventMapper.kt
@@ -0,0 +1,151 @@
+package io.emeraldpay.dshackle.rpc
+
+import com.google.protobuf.ByteString
+import io.emeraldpay.api.proto.BlockchainOuterClass
+import io.emeraldpay.api.proto.BlockchainOuterClass.SupportedMethodsEvent
+import io.emeraldpay.api.proto.Common
+import io.emeraldpay.dshackle.data.BlockContainer
+import io.emeraldpay.dshackle.startup.QuorumForLabels
+import io.emeraldpay.dshackle.upstream.Capability
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import io.emeraldpay.dshackle.upstream.finalization.FinalizationData
+import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
+import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
+import org.springframework.stereotype.Component
+
+@Component
+class ChainEventMapper {
+
+    fun mapHead(head: BlockContainer): BlockchainOuterClass.ChainEvent {
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setHead(
+                BlockchainOuterClass.HeadEvent.newBuilder()
+                    .setHeight(head.height)
+                    .setSlot(head.slot)
+                    .setTimestamp(head.timestamp.toEpochMilli())
+                    .setWeight(ByteString.copyFrom(head.difficulty.toByteArray()))
+                    .setBlockId(head.hash.toHex())
+                    .setParentBlockId(head.parentHash?.toHex() ?: "")
+                    .build(),
+            )
+            .build()
+    }
+
+    fun mapCapabilities(capabilities: Collection<Capability>): BlockchainOuterClass.ChainEvent {
+        val caps = capabilities.map {
+            when (it) {
+                Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS
+                Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE
+                Capability.WS_HEAD -> BlockchainOuterClass.Capabilities.CAP_WS_HEAD
+            }
+        }
+
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setCapabilitiesEvent(
+                BlockchainOuterClass.CapabilitiesEvent.newBuilder()
+                    .addAllCapabilities(caps)
+                    .build(),
+            )
+            .build()
+    }
+
+    fun mapNodeDetails(nodeDetails: Collection<QuorumForLabels.QuorumItem>): BlockchainOuterClass.ChainEvent {
+        val details = nodeDetails.map {
+            BlockchainOuterClass.NodeDetails.newBuilder()
+                .setQuorum(it.quorum)
+                .addAllLabels(
+                    it.labels.entries.map { label ->
+                        BlockchainOuterClass.Label.newBuilder()
+                            .setName(label.key)
+                            .setValue(label.value)
+                            .build()
+                    },
+                ).build()
+        }
+
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setNodesEvent(
+                BlockchainOuterClass.NodeDetailsEvent.newBuilder()
+                    .addAllNodes(details)
+                    .build(),
+            )
+            .build()
+    }
+
+    fun mapFinalizationData(finalizationData: Collection<FinalizationData>): BlockchainOuterClass.ChainEvent {
+        val data = finalizationData.map {
+            Common.FinalizationData.newBuilder()
+                .setHeight(it.height)
+                .setType(it.type.toProtoFinalizationType())
+                .build()
+        }
+
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setFinalizationDataEvent(
+                BlockchainOuterClass.FinalizationDataEvent.newBuilder()
+                    .addAllFinalizationData(data)
+                    .build(),
+            )
+            .build()
+    }
+
+    fun mapLowerBounds(lowerBounds: Collection<LowerBoundData>): BlockchainOuterClass.ChainEvent {
+        val data = lowerBounds
+            .map {
+                BlockchainOuterClass.LowerBound.newBuilder()
+                    .setLowerBoundTimestamp(it.timestamp)
+                    .setLowerBoundType(mapLowerBoundType(it.type))
+                    .setLowerBoundValue(it.lowerBound)
+                    .build()
+            }
+
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setLowerBoundsEvent(
+                BlockchainOuterClass.LowerBoundEvent.newBuilder()
+                    .addAllLowerBounds(data)
+                    .build(),
+            )
+            .build()
+    }
+
+    fun chainStatus(status: UpstreamAvailability): BlockchainOuterClass.ChainEvent {
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setStatus(
+                BlockchainOuterClass.ChainStatus.newBuilder()
+                    .setAvailability(Common.AvailabilityEnum.forNumber(status.grpcId))
+                    .build(),
+            )
+            .build()
+    }
+
+    fun supportedMethods(methods: Collection<String>): BlockchainOuterClass.ChainEvent {
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setSupportedMethodsEvent(
+                SupportedMethodsEvent.newBuilder()
+                    .addAllMethods(methods)
+                    .build(),
+            )
+            .build()
+    }
+
+    fun supportedSubs(subs: Collection<String>): BlockchainOuterClass.ChainEvent {
+        return BlockchainOuterClass.ChainEvent.newBuilder()
+            .setSupportedSubscriptionsEvent(
+                BlockchainOuterClass.SupportedSubscriptionsEvent.newBuilder()
+                    .addAllSubs(subs)
+                    .build(),
+            )
+            .build()
+    }
+
+    private fun mapLowerBoundType(type: LowerBoundType): BlockchainOuterClass.LowerBoundType {
+        return when (type) {
+            LowerBoundType.SLOT -> BlockchainOuterClass.LowerBoundType.LOWER_BOUND_SLOT
+            LowerBoundType.UNKNOWN -> BlockchainOuterClass.LowerBoundType.LOWER_BOUND_UNSPECIFIED
+            LowerBoundType.STATE -> BlockchainOuterClass.LowerBoundType.LOWER_BOUND_STATE
+            LowerBoundType.BLOCK -> BlockchainOuterClass.LowerBoundType.LOWER_BOUND_BLOCK
+            LowerBoundType.TX -> BlockchainOuterClass.LowerBoundType.LOWER_BOUND_TX
+            LowerBoundType.LOGS -> BlockchainOuterClass.LowerBoundType.LOWER_BOUND_LOGS
+        }
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt
@@ -46,6 +46,7 @@ import io.emeraldpay.dshackle.upstream.MultistreamHolder
 import io.emeraldpay.dshackle.upstream.Selector
 import io.emeraldpay.dshackle.upstream.Upstream
 import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
+import io.emeraldpay.dshackle.upstream.calls.DisabledCallMethods
 import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
 import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
 import io.emeraldpay.dshackle.upstream.finalization.FinalizationData
@@ -110,7 +111,7 @@ open class NativeCall(
                            if (it is ValidCallContext<*>) {
                                if (it.payload is ParsedCallDetails) {
                                    log.error("nativeCallResult method ${it.payload.method} of ${it.upstream.getId()} is not available, disabling")
-                                    val cm = (it.upstream.getMethods() as Multistream.DisabledCallMethods)
+                                    val cm = (it.upstream.getMethods() as DisabledCallMethods)
                                    cm.disableMethodTemporarily(it.payload.method)
                                    it.upstream.updateMethods(cm)
                                }
--- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt
@@ -0,0 +1,128 @@
+package io.emeraldpay.dshackle.rpc
+
+import io.emeraldpay.api.proto.BlockchainOuterClass
+import io.emeraldpay.api.proto.Common
+import io.emeraldpay.dshackle.Global
+import io.emeraldpay.dshackle.data.BlockContainer
+import io.emeraldpay.dshackle.upstream.Multistream
+import io.emeraldpay.dshackle.upstream.MultistreamHolder
+import io.emeraldpay.dshackle.upstream.state.MultistreamStateEvent
+import org.slf4j.LoggerFactory
+import org.springframework.stereotype.Service
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Mono
+import reactor.kotlin.core.publisher.switchIfEmpty
+
+@Service
+class SubscribeChainStatus(
+    private val multistreamHolder: MultistreamHolder,
+    private val chainEventMapper: ChainEventMapper,
+) {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(SubscribeChainStatus::class.java)
+    }
+
+    fun chainStatuses(): Flux<BlockchainOuterClass.SubscribeChainStatusResponse> {
+        return Flux.merge(
+            // we need to track not only multistreams with upstreams but all of them
+            // because upstreams can be added in runtime with hot config reload
+            multistreamHolder.all()
+                .filter { Common.ChainRef.forNumber(it.chain.id) != null }
+                .map { ms ->
+                    Flux.concat(
+                        // the first event must be filled with all fields
+                        firstFullEvent(ms),
+                        Flux.merge(
+                            // head events are separated from others
+                            headEvents(ms),
+                            multistreamEvents(ms),
+                        ),
+                    )
+                },
+        ).doOnError {
+            log.error("Error during sending chain statuses", it)
+        }
+    }
+
+    private fun multistreamEvents(ms: Multistream): Flux<BlockchainOuterClass.SubscribeChainStatusResponse> {
+        return ms.stateEvents()
+            .filter { it.isNotEmpty() }
+            .map { events ->
+                val response = BlockchainOuterClass.SubscribeChainStatusResponse.newBuilder()
+                val chainDescription = BlockchainOuterClass.ChainDescription.newBuilder()
+                    .setChain(Common.ChainRef.forNumber(ms.chain.id))
+
+                events.forEach {
+                    chainDescription.addChainEvent(processMsEvent(it))
+                }
+
+                response.setChainDescription(chainDescription.build())
+                response.build()
+            }
+    }
+
+    private fun processMsEvent(event: MultistreamStateEvent): BlockchainOuterClass.ChainEvent {
+        return when (event) {
+            is MultistreamStateEvent.CapabilitiesEvent -> chainEventMapper.mapCapabilities(event.caps)
+            is MultistreamStateEvent.FinalizationEvent -> chainEventMapper.mapFinalizationData(event.finalizationData)
+            is MultistreamStateEvent.LowerBoundsEvent -> chainEventMapper.mapLowerBounds(event.lowerBounds)
+            is MultistreamStateEvent.MethodsEvent -> chainEventMapper.supportedMethods(event.methods)
+            is MultistreamStateEvent.NodeDetailsEvent -> chainEventMapper.mapNodeDetails(event.details)
+            is MultistreamStateEvent.StatusEvent -> chainEventMapper.chainStatus(event.status)
+            is MultistreamStateEvent.SubsEvent -> chainEventMapper.supportedSubs(event.subs)
+        }
+    }
+
+    private fun firstFullEvent(ms: Multistream): Mono<BlockchainOuterClass.SubscribeChainStatusResponse> {
+        return Mono.justOrEmpty(ms.getHead().getCurrent())
+            .map { toFullResponse(it!!, ms) }
+            .switchIfEmpty {
+                // in case if there is still no head we mush wait until we get it
+                ms.getHead()
+                    .getFlux()
+                    .next()
+                    .map { toFullResponse(it!!, ms) }
+            }
+    }
+
+    private fun headEvents(ms: Multistream): Flux<BlockchainOuterClass.SubscribeChainStatusResponse> {
+        return ms.getHead()
+            .getFlux()
+            .skip(1)
+            .map {
+                BlockchainOuterClass.SubscribeChainStatusResponse.newBuilder()
+                    .setChainDescription(
+                        BlockchainOuterClass.ChainDescription.newBuilder()
+                            .setChain(Common.ChainRef.forNumber(ms.chain.id))
+                            .addChainEvent(chainEventMapper.mapHead(it))
+                            .build(),
+                    )
+                    .build()
+            }
+    }
+
+    private fun toFullResponse(head: BlockContainer, ms: Multistream): BlockchainOuterClass.SubscribeChainStatusResponse {
+        return BlockchainOuterClass.SubscribeChainStatusResponse.newBuilder()
+            .setChainDescription(
+                BlockchainOuterClass.ChainDescription.newBuilder()
+                    .setChain(Common.ChainRef.forNumber(ms.chain.id))
+                    .addChainEvent(chainEventMapper.chainStatus(ms.getStatus()))
+                    .addChainEvent(chainEventMapper.mapHead(head))
+                    .addChainEvent(chainEventMapper.supportedMethods(ms.getMethods().getSupportedMethods()))
+                    .addChainEvent(chainEventMapper.supportedSubs(ms.getEgressSubscription().getAvailableTopics()))
+                    .addChainEvent(chainEventMapper.mapCapabilities(ms.getCapabilities()))
+                    .addChainEvent(chainEventMapper.mapLowerBounds(ms.getLowerBounds()))
+                    .addChainEvent(chainEventMapper.mapFinalizationData(ms.getFinalizations()))
+                    .addChainEvent(chainEventMapper.mapNodeDetails(ms.getQuorumLabels()))
+                    .build(),
+            )
+            .setBuildInfo(
+                BlockchainOuterClass.BuildInfo.newBuilder()
+                    .setVersion(Global.version)
+                    .build(),
+            )
+            .setFullResponse(true)
+            .build()
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/startup/QuorumForLabels.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/QuorumForLabels.kt
@@ -73,7 +73,7 @@ class QuorumForLabels() {
    /**
     * Details for a single element (upstream, node or aggregation)
     */
-    class QuorumItem(val quorum: Int, val labels: UpstreamsConfig.Labels) {
+    data class QuorumItem(val quorum: Int, val labels: UpstreamsConfig.Labels) {
        companion object {
            fun empty(): QuorumItem {
                return QuorumItem(0, UpstreamsConfig.Labels())
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt
@@ -143,7 +143,7 @@ abstract class AbstractHead @JvmOverloads constructor(
        ).onBackpressureLatest()
    }

-    fun getCurrent(): BlockContainer? {
+    override fun getCurrent(): BlockContainer? {
        return forkChoice.getHead()
    }

--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt
@@ -20,6 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
 import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.config.ChainsConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig
+import io.emeraldpay.dshackle.config.UpstreamsConfig.Labels.Companion.fromMap
 import io.emeraldpay.dshackle.foundation.ChainOptions
 import io.emeraldpay.dshackle.startup.QuorumForLabels
 import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
@@ -155,7 +156,7 @@ abstract class DefaultUpstream(
    override fun nodeId(): Byte = hash

    open fun getQuorumByLabel(): QuorumForLabels {
-        return node?.let { QuorumForLabels(it) }
+        return node?.let { QuorumForLabels(it.copy(labels = fromMap(it.labels))) }
            ?: QuorumForLabels(QuorumForLabels.QuorumItem.empty())
    }

--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt
@@ -45,4 +45,8 @@ class EmptyHead : Head {
    }

    override fun headLiveness(): Flux<Boolean> = Flux.empty()
+
+    override fun getCurrent(): BlockContainer? {
+        return null
+    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt
@@ -47,4 +47,6 @@ interface Head {
    fun onSyncingNode(isSyncing: Boolean)

    fun headLiveness(): Flux<Boolean>
+
+    fun getCurrent(): BlockContainer?
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt
@@ -16,26 +16,22 @@
 */
 package io.emeraldpay.dshackle.upstream

-import com.github.benmanes.caffeine.cache.Cache
-import com.github.benmanes.caffeine.cache.Caffeine
 import io.emeraldpay.api.proto.BlockchainOuterClass
 import io.emeraldpay.dshackle.Chain
-import io.emeraldpay.dshackle.Defaults.Companion.multistreamUnavailableMethodDisableDuration
 import io.emeraldpay.dshackle.cache.Caches
 import io.emeraldpay.dshackle.cache.CachesEnabled
 import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.foundation.ChainOptions
-import io.emeraldpay.dshackle.quorum.CallQuorum
 import io.emeraldpay.dshackle.reader.ChainReader
 import io.emeraldpay.dshackle.startup.QuorumForLabels
 import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
-import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods
 import io.emeraldpay.dshackle.upstream.calls.CallMethods
 import io.emeraldpay.dshackle.upstream.calls.CallSelector
 import io.emeraldpay.dshackle.upstream.finalization.FinalizationData
-import io.emeraldpay.dshackle.upstream.finalization.FinalizationType
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
 import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
+import io.emeraldpay.dshackle.upstream.state.MultistreamState
+import io.emeraldpay.dshackle.upstream.state.MultistreamStateEvent
 import io.micrometer.core.instrument.Gauge
 import io.micrometer.core.instrument.Meter
 import io.micrometer.core.instrument.Metrics
@@ -50,7 +46,6 @@ import reactor.core.publisher.Sinks
 import reactor.core.scheduler.Scheduler
 import java.time.Duration
 import java.util.concurrent.ConcurrentHashMap
-import kotlin.math.max

 /**
 * Aggregation of multiple upstreams responding to a single blockchain
@@ -68,29 +63,22 @@ abstract class Multistream(
        private const val metrics = "upstreams"
    }

+    private val state = MultistreamState { onUpstreamsUpdated() }
+
    protected val log = LoggerFactory.getLogger(this::class.java)

    private var started = false

    private var cacheSubscription: Disposable? = null

-    @Volatile
-    private var callMethods: DisabledCallMethods? = null
    private var callMethodsFactory: Factory<CallMethods> = Factory {
-        return@Factory callMethods ?: throw FunctorException("Not initialized yet")
+        return@Factory state.getCallMethods() ?: throw FunctorException("Not initialized yet")
    }
    private var stopSignal = Sinks.many().multicast().directBestEffort<Boolean>()
    private var seq = 0
    protected var lagObserver: HeadLagObserver? = null
    private var subscription: Disposable? = null

-    @Volatile
-    private var capabilities: Set<Capability> = emptySet()
-
-    private val lowerBounds = ConcurrentHashMap<LowerBoundType, LowerBoundData>()
-
-    @Volatile
-    private var quorumLabels: List<QuorumForLabels.QuorumItem>? = null
    private val meters: MutableMap<String, List<Meter.Id>> = HashMap()
    private val observedUpstreams = Sinks.many()
        .multicast()
@@ -232,41 +220,7 @@ abstract class Multistream(

    protected open fun onUpstreamsUpdated() {
        val upstreams = getAll()
-        val availableUpstreams = upstreams.filter { it.isAvailable() }
-        availableUpstreams.map { it.getMethods() }.let {
-            if (callMethods == null) {
-                callMethods = DisabledCallMethods(this, multistreamUnavailableMethodDisableDuration, AggregatedCallMethods(it))
-            } else {
-                callMethods = DisabledCallMethods(
-                    this,
-                    multistreamUnavailableMethodDisableDuration,
-                    AggregatedCallMethods(it),
-                    callMethods!!.disabledMethods,
-                )
-            }
-        }
-        capabilities = if (upstreams.isEmpty()) {
-            emptySet()
-        } else {
-            availableUpstreams.map { up ->
-                up.getCapabilities()
-            }.let {
-                if (it.isNotEmpty()) {
-                    it.reduce { acc, curr -> acc + curr }
-                } else {
-                    emptySet()
-                }
-            }
-        }
-        quorumLabels = getQuorumLabels(availableUpstreams)
-
-        availableUpstreams
-            .flatMap { it.getLowerBounds() }
-            .groupBy { it.type }
-            .forEach { entry ->
-                val min = entry.value.minBy { it.lowerBound }
-                lowerBounds[entry.key] = min
-            }
+        state.updateState(upstreams, getSubscriptionTopics())

        when {
            upstreams.size == 1 -> {
@@ -279,17 +233,7 @@ abstract class Multistream(
        }
    }

-    private fun getQuorumLabels(ups: List<Upstream>): List<QuorumForLabels.QuorumItem> {
-        val nodes = QuorumForLabels()
-        ups.forEach { up ->
-            if (up is DefaultUpstream) {
-                nodes.add(up.getQuorumByLabel())
-            }
-        }
-        return nodes.getAll()
-    }
-
-    fun getQuorumLabels(): List<QuorumForLabels.QuorumItem> = quorumLabels ?: emptyList()
+    open fun getQuorumLabels(): List<QuorumForLabels.QuorumItem> = state.getQuorumLabels() ?: emptyList()

    override fun observeStatus(): Flux<UpstreamAvailability> {
        val upstreamsFluxes = getAll().map { up ->
@@ -314,12 +258,7 @@ abstract class Multistream(
    }

    override fun getStatus(): UpstreamAvailability {
-        val upstreams = getAll()
-        return if (upstreams.isEmpty()) {
-            UpstreamAvailability.UNAVAILABLE
-        } else {
-            upstreams.minOf { it.getStatus() }
-        }
+        return state.getStatus()
    }

    // TODO options for multistream are useless
@@ -333,7 +272,7 @@ abstract class Multistream(
    }

    override fun getMethods(): CallMethods {
-        return callMethods ?: throw IllegalStateException("Methods are not initialized yet")
+        return state.getCallMethods() ?: throw IllegalStateException("Methods are not initialized yet")
    }

    override fun updateMethods(m: CallMethods) {
@@ -359,11 +298,7 @@ abstract class Multistream(
    }

    override fun getFinalizations(): Collection<FinalizationData> {
-        return getAll().flatMap { it.getFinalizations() }
-            .fold(mutableMapOf<FinalizationType, Long>()) { acc, data ->
-                acc[data.type] = max(acc[data.type] ?: 0, data.height)
-                acc
-            }.toList().map { FinalizationData(it.second, it.first) }
+        return state.getFinalizationData()
    }

    override fun addFinalization(finalization: FinalizationData, upstreamId: String) {
@@ -371,11 +306,11 @@ abstract class Multistream(
    }

    override fun getLowerBounds(): Collection<LowerBoundData> {
-        return lowerBounds.values
+        return state.getLowerBounds()
    }

    override fun getLowerBound(lowerBoundType: LowerBoundType): LowerBoundData? {
-        return lowerBounds[lowerBoundType]
+        return state.getLowerBound(lowerBoundType)
    }

    override fun getUpstreamSettingsData(): Upstream.UpstreamSettingsData? {
@@ -447,7 +382,7 @@ abstract class Multistream(
    }

    override fun getCapabilities(): Set<Capability> {
-        return this.capabilities
+        return state.getCapabilities()
    }

    override fun isGrpc(): Boolean {
@@ -482,7 +417,7 @@ abstract class Multistream(
        val weak = getUpstreams()
            .filter { it.getStatus() != UpstreamAvailability.OK }
            .joinToString(", ") { it.getId() }
-        val lowerBlockData = lowerBounds.entries.joinToString(", ") { "${it.key}=${it.value.lowerBound}" }
+        val lowerBlockData = state.lowerBoundsToString()

        val instance = System.identityHashCode(this).toString(16)
        log.info("State of ${chain.chainCode}: height=${height ?: '?'}, status=[$statuses], lag=[$lag], lower bounds=[$lowerBlockData], weak=[$weak] ($instance)")
@@ -571,6 +506,8 @@ abstract class Multistream(
    fun subscribeUpdatedUpstreams(): Flux<Upstream> =
        updateUpstreams.asFlux()

+    fun stateEvents(): Flux<Collection<MultistreamStateEvent>> = state.stateEvents()
+
    abstract fun makeLagObserver(): HeadLagObserver

    open fun tryProxySubscribe(
@@ -582,57 +519,6 @@ abstract class Multistream(

    abstract fun getHead(mather: Selector.Matcher): Head

-    // --------------------------------------------------------------------------------------------------------
-
-    class DisabledCallMethods(private val multistream: Multistream, private val defaultDisableTimeout: Long, private val callMethods: CallMethods) : CallMethods {
-        var disabledMethods: Cache<String, Boolean> = Caffeine.newBuilder()
-            .removalListener { key: String?, _: Boolean?, cause ->
-                if (cause.wasEvicted() && key != null) {
-                    multistream.log.info("${multistream.getId()} restoring method $key")
-                    multistream.onUpstreamsUpdated()
-                }
-            }
-            .expireAfterWrite(Duration.ofMinutes(defaultDisableTimeout))
-            .build<String, Boolean>()
-
-        constructor(
-            multistream: Multistream,
-            defaultDisableTimeout: Long,
-            callMethods: CallMethods,
-            disabledMethodsCopy: Cache<String, Boolean>,
-        ) : this(multistream, defaultDisableTimeout, callMethods) {
-            disabledMethods = disabledMethodsCopy
-        }
-
-        override fun createQuorumFor(method: String): CallQuorum {
-            return callMethods.createQuorumFor(method)
-        }
-
-        override fun isCallable(method: String): Boolean {
-            return callMethods.isCallable(method) && disabledMethods.getIfPresent(method) == null
-        }
-
-        override fun getSupportedMethods(): Set<String> {
-            return callMethods.getSupportedMethods() - disabledMethods.asMap().keys
-        }
-
-        override fun isHardcoded(method: String): Boolean {
-            return callMethods.isHardcoded(method)
-        }
-
-        override fun executeHardcoded(method: String): ByteArray {
-            return callMethods.executeHardcoded(method)
-        }
-
-        override fun getGroupMethods(groupName: String): Set<String> {
-            return callMethods.getGroupMethods(groupName)
-        }
-
-        fun disableMethodTemporarily(method: String) {
-            disabledMethods.put(method, true)
-        }
-    }
-
    class UpstreamStatus(val upstream: Upstream, val status: UpstreamAvailability)

    class FilterBestAvailability : java.util.function.Function<UpstreamStatus, UpstreamAvailability> {
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DisabledCallMethods.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DisabledCallMethods.kt
@@ -0,0 +1,58 @@
+package io.emeraldpay.dshackle.upstream.calls
+
+import com.github.benmanes.caffeine.cache.Cache
+import com.github.benmanes.caffeine.cache.Caffeine
+import io.emeraldpay.dshackle.quorum.CallQuorum
+import java.time.Duration
+
+class DisabledCallMethods(
+    defaultDisableTimeout: Long,
+    private val callMethods: CallMethods,
+    private val onUpdate: () -> Unit,
+) : CallMethods {
+    var disabledMethods: Cache<String, Boolean> = Caffeine.newBuilder()
+        .removalListener { key: String?, _: Boolean?, cause ->
+            if (cause.wasEvicted() && key != null) {
+                onUpdate.invoke()
+            }
+        }
+        .expireAfterWrite(Duration.ofMinutes(defaultDisableTimeout))
+        .build<String, Boolean>()
+
+    constructor(
+        onUpdate: () -> Unit,
+        defaultDisableTimeout: Long,
+        callMethods: CallMethods,
+        disabledMethodsCopy: Cache<String, Boolean>,
+    ) : this(defaultDisableTimeout, callMethods, onUpdate) {
+        disabledMethods = disabledMethodsCopy
+    }
+
+    override fun createQuorumFor(method: String): CallQuorum {
+        return callMethods.createQuorumFor(method)
+    }
+
+    override fun isCallable(method: String): Boolean {
+        return callMethods.isCallable(method) && disabledMethods.getIfPresent(method) == null
+    }
+
+    override fun getSupportedMethods(): Set<String> {
+        return callMethods.getSupportedMethods() - disabledMethods.asMap().keys
+    }
+
+    override fun isHardcoded(method: String): Boolean {
+        return callMethods.isHardcoded(method)
+    }
+
+    override fun executeHardcoded(method: String): ByteArray {
+        return callMethods.executeHardcoded(method)
+    }
+
+    override fun getGroupMethods(groupName: String): Set<String> {
+        return callMethods.getGroupMethods(groupName)
+    }
+
+    fun disableMethodTemporarily(method: String) {
+        disabledMethods.put(method, true)
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFinalizationDetector.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFinalizationDetector.kt
@@ -13,6 +13,7 @@ import io.emeraldpay.dshackle.upstream.finalization.FinalizationType
 import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
 import org.slf4j.LoggerFactory
 import reactor.core.publisher.Flux
+import reactor.core.publisher.Sinks
 import java.time.Duration
 import java.util.concurrent.ConcurrentHashMap

@@ -23,64 +24,67 @@ class EthereumFinalizationDetector : FinalizationDetector {

    val data: ConcurrentHashMap<FinalizationType, FinalizationData> = ConcurrentHashMap()

+    private val finalizationSink = Sinks.many().multicast().directBestEffort<FinalizationData>()
+
    override fun detectFinalization(
        upstream: Upstream,
        blockTime: Duration,
    ): Flux<FinalizationData> {
-        val timer =
-            Flux.merge(
-                Flux.just(1),
-                Flux.interval(Duration.ofSeconds(15)),
-            )
-        return timer.flatMap {
-            Flux.fromIterable(
-                listOf(
-                    Pair(
-                        FinalizationType.SAFE_BLOCK,
-                        ChainRequest(
-                            "eth_getBlockByNumber",
-                            ListParams("safe", false),
-                            1,
+        return Flux.merge(
+            finalizationSink.asFlux(),
+            Flux.interval(
+                Duration.ofSeconds(0),
+                Duration.ofSeconds(15),
+            ).flatMap {
+                Flux.fromIterable(
+                    listOf(
+                        Pair(
+                            FinalizationType.SAFE_BLOCK,
+                            ChainRequest(
+                                "eth_getBlockByNumber",
+                                ListParams("safe", false),
+                                1,
+                            ),
+                        ),
+                        Pair(
+                            FinalizationType.FINALIZED_BLOCK,
+                            ChainRequest(
+                                "eth_getBlockByNumber",
+                                ListParams("finalized", false),
+                                2,
+                            ),
                        ),
                    ),
-                    Pair(
-                        FinalizationType.FINALIZED_BLOCK,
-                        ChainRequest(
-                            "eth_getBlockByNumber",
-                            ListParams("finalized", false),
-                            2,
-                        ),
-                    ),
-                ),
-            ).flatMap { (type, req) ->
-                upstream
-                    .getIngressReader()
-                    .read(req)
-                    .flatMap {
-                        it.requireResult().map { result ->
-                            val block =
-                                Global.objectMapper
-                                    .readValue(result, BlockJson::class.java) as BlockJson<TransactionRefJson>?
-                            if (block != null) {
-                                FinalizationData(block.number, type)
-                            } else {
-                                throw RpcException(RpcResponseError.CODE_INVALID_JSON, "can't parse block data")
+                ).flatMap { (type, req) ->
+                    upstream
+                        .getIngressReader()
+                        .read(req)
+                        .flatMap {
+                            it.requireResult().map { result ->
+                                val block =
+                                    Global.objectMapper
+                                        .readValue(result, BlockJson::class.java) as BlockJson<TransactionRefJson>?
+                                if (block != null) {
+                                    FinalizationData(block.number, type)
+                                } else {
+                                    throw RpcException(RpcResponseError.CODE_INVALID_JSON, "can't parse block data")
+                                }
                            }
                        }
-                    }
+                }.onErrorResume {
+                    log.error("Error during retrieving — $it")
+                    Flux.empty()
+                }
+            }.filter {
+                it.height > (data[it.type]?.height ?: 0)
            }.doOnNext {
-                addFinalization(it)
-            }.onErrorResume {
-                log.error("Error during retrieving — $it")
-                Flux.empty()
-            }
-        }
+                data[it.type] = it
+            },
+        )
    }

    override fun addFinalization(finalization: FinalizationData) {
-        data[finalization.type] = maxOf(data[finalization.type], finalization) { a, b ->
-            ((a?.height ?: 0) - (b?.height ?: 0)).toInt()
-        } ?: finalization
+        finalizationSink.emitNext(finalization) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED }
    }

    override fun getFinalizations(): Collection<FinalizationData> {
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt
@@ -208,7 +208,11 @@ open class GenericUpstream(
    }

    private fun detectSettings() {
-        settingsDetector?.detectLabels()?.subscribe { label -> updateLabels(label) }
+        settingsDetector?.detectLabels()
+            ?.subscribe { label ->
+                updateLabels(label)
+                sendUpstreamStateEvent(UPDATED)
+            }

        settingsDetector?.detectClientVersion()
            ?.subscribe {
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/state/MultistreamState.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/state/MultistreamState.kt
@@ -0,0 +1,162 @@
+package io.emeraldpay.dshackle.upstream.state
+
+import io.emeraldpay.dshackle.Defaults
+import io.emeraldpay.dshackle.startup.QuorumForLabels
+import io.emeraldpay.dshackle.upstream.Capability
+import io.emeraldpay.dshackle.upstream.DefaultUpstream
+import io.emeraldpay.dshackle.upstream.Upstream
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods
+import io.emeraldpay.dshackle.upstream.calls.CallMethods
+import io.emeraldpay.dshackle.upstream.calls.DisabledCallMethods
+import io.emeraldpay.dshackle.upstream.finalization.FinalizationData
+import io.emeraldpay.dshackle.upstream.finalization.FinalizationType
+import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
+import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Sinks
+import java.util.concurrent.ConcurrentHashMap
+
+class MultistreamState(
+    private val onMsUpdate: () -> Unit,
+) {
+    @Volatile
+    private var callMethods: DisabledCallMethods? = null
+
+    @Volatile
+    private var capabilities: Set<Capability> = emptySet()
+
+    @Volatile
+    private var quorumLabels: List<QuorumForLabels.QuorumItem>? = null
+
+    @Volatile
+    private var status: UpstreamAvailability = UpstreamAvailability.UNAVAILABLE
+
+    @Volatile
+    private var subs: List<String> = emptyList()
+    private val lowerBounds = ConcurrentHashMap<LowerBoundType, LowerBoundData>()
+    private val finalizationData = ConcurrentHashMap<FinalizationType, FinalizationData>()
+
+    private val stateHandler = MultistreamStateHandler
+    private val stateEvents = Sinks.many().multicast().directBestEffort<Collection<MultistreamStateEvent>>()
+
+    fun getCallMethods(): CallMethods? = callMethods
+
+    fun getQuorumLabels(): List<QuorumForLabels.QuorumItem>? = quorumLabels
+
+    fun getLowerBounds(): Collection<LowerBoundData> = HashSet(lowerBounds.values)
+
+    fun getLowerBound(lowerBoundType: LowerBoundType): LowerBoundData? = lowerBounds[lowerBoundType]
+
+    fun getFinalizationData(): Collection<FinalizationData> = HashSet(finalizationData.values)
+
+    fun getCapabilities(): Set<Capability> = capabilities
+
+    fun lowerBoundsToString(): String =
+        lowerBounds.entries.joinToString(", ") { "${it.key}=${it.value.lowerBound}" }
+
+    fun getStatus(): UpstreamAvailability {
+        return status
+    }
+
+    fun updateState(upstreams: List<Upstream>, subs: List<String>) {
+        val oldState = CurrentMultistreamState(this)
+
+        val availableUpstreams = upstreams.filter { it.isAvailable() }
+        updateMethods(availableUpstreams)
+        updateCapabilities(availableUpstreams)
+        updateQuorumLabels(availableUpstreams)
+        updateUpstreamBounds(availableUpstreams)
+        status = if (upstreams.isEmpty()) UpstreamAvailability.UNAVAILABLE else upstreams.minOf { it.getStatus() }
+        this.subs = subs
+
+        stateEvents.emitNext(
+            stateHandler.compareStates(oldState, CurrentMultistreamState(this)),
+        ) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED }
+    }
+
+    fun stateEvents(): Flux<Collection<MultistreamStateEvent>> = stateEvents.asFlux()
+
+    private fun updateMethods(upstreams: List<Upstream>) {
+        upstreams.map { it.getMethods() }.let {
+            callMethods = if (callMethods == null) {
+                DisabledCallMethods(
+                    Defaults.multistreamUnavailableMethodDisableDuration,
+                    AggregatedCallMethods(it),
+                    onMsUpdate,
+                )
+            } else {
+                DisabledCallMethods(
+                    onMsUpdate,
+                    Defaults.multistreamUnavailableMethodDisableDuration,
+                    AggregatedCallMethods(it),
+                    callMethods!!.disabledMethods,
+                )
+            }
+        }
+    }
+
+    private fun updateCapabilities(upstreams: List<Upstream>) {
+        capabilities = if (upstreams.isEmpty()) {
+            emptySet()
+        } else {
+            upstreams.map { up ->
+                up.getCapabilities()
+            }.let {
+                if (it.isNotEmpty()) {
+                    it.reduce { acc, curr -> acc + curr }
+                } else {
+                    emptySet()
+                }
+            }
+        }
+    }
+
+    private fun updateQuorumLabels(ups: List<Upstream>) {
+        val nodes = QuorumForLabels()
+        ups.forEach { up ->
+            if (up is DefaultUpstream) {
+                nodes.add(up.getQuorumByLabel())
+            }
+        }
+        quorumLabels = nodes.getAll()
+    }
+
+    private fun updateUpstreamBounds(upstreams: List<Upstream>) {
+        upstreams
+            .flatMap { it.getLowerBounds() }
+            .groupBy { it.type }
+            .forEach { entry ->
+                val min = entry.value.minBy { it.lowerBound }
+                lowerBounds[entry.key] = min
+            }
+
+        upstreams
+            .flatMap { it.getFinalizations() }
+            .groupBy { it.type }
+            .forEach { entry ->
+                val max = entry.value.maxBy { it.height }
+                finalizationData[entry.key] = max
+            }
+    }
+
+    data class CurrentMultistreamState(
+        val status: UpstreamAvailability,
+        val methods: Collection<String>,
+        val subs: Collection<String>,
+        val caps: Collection<Capability>,
+        val lowerBounds: Collection<LowerBoundData>,
+        val finalizationData: Collection<FinalizationData>,
+        val nodeDetails: Collection<QuorumForLabels.QuorumItem>,
+    ) {
+        constructor(state: MultistreamState) : this(
+            state.getStatus(),
+            state.getCallMethods()?.getSupportedMethods() ?: emptySet(),
+            state.subs,
+            state.getCapabilities(),
+            state.getLowerBounds(),
+            state.getFinalizationData(),
+            state.getQuorumLabels() ?: emptySet(),
+        )
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/state/MultistreamStateEvent.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/state/MultistreamStateEvent.kt
@@ -0,0 +1,37 @@
+package io.emeraldpay.dshackle.upstream.state
+
+import io.emeraldpay.dshackle.startup.QuorumForLabels
+import io.emeraldpay.dshackle.upstream.Capability
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import io.emeraldpay.dshackle.upstream.finalization.FinalizationData
+import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData
+
+sealed class MultistreamStateEvent {
+    data class StatusEvent(
+        val status: UpstreamAvailability,
+    ) : MultistreamStateEvent()
+
+    data class MethodsEvent(
+        val methods: Collection<String>,
+    ) : MultistreamStateEvent()
+
+    data class SubsEvent(
+        val subs: Collection<String>,
+    ) : MultistreamStateEvent()
+
+    data class CapabilitiesEvent(
+        val caps: Collection<Capability>,
+    ) : MultistreamStateEvent()
+
+    data class LowerBoundsEvent(
+        val lowerBounds: Collection<LowerBoundData>,
+    ) : MultistreamStateEvent()
+
+    data class FinalizationEvent(
+        val finalizationData: Collection<FinalizationData>,
+    ) : MultistreamStateEvent()
+
+    data class NodeDetailsEvent(
+        val details: Collection<QuorumForLabels.QuorumItem>,
+    ) : MultistreamStateEvent()
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/state/MultistreamStateHandler.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/state/MultistreamStateHandler.kt
@@ -0,0 +1,32 @@
+package io.emeraldpay.dshackle.upstream.state
+
+object MultistreamStateHandler {
+
+    fun compareStates(old: MultistreamState.CurrentMultistreamState, new: MultistreamState.CurrentMultistreamState): Collection<MultistreamStateEvent> {
+        val events = mutableListOf<MultistreamStateEvent>()
+
+        if (old.status != new.status) {
+            events.add(MultistreamStateEvent.StatusEvent(new.status))
+        }
+        if (old.methods != new.methods) {
+            events.add(MultistreamStateEvent.MethodsEvent(new.methods))
+        }
+        if (old.subs != new.subs) {
+            events.add(MultistreamStateEvent.SubsEvent(new.subs))
+        }
+        if (old.caps != new.caps) {
+            events.add(MultistreamStateEvent.CapabilitiesEvent(new.caps))
+        }
+        if (old.lowerBounds != new.lowerBounds) {
+            events.add(MultistreamStateEvent.LowerBoundsEvent(new.lowerBounds))
+        }
+        if (old.finalizationData != new.finalizationData) {
+            events.add(MultistreamStateEvent.FinalizationEvent(new.finalizationData))
+        }
+        if (old.nodeDetails != new.nodeDetails) {
+            events.add(MultistreamStateEvent.NodeDetailsEvent(new.nodeDetails))
+        }
+
+        return events
+    }
+}