Merge pull request #122 from emeraldpay/feat/health-check

2021-10-24 18:03:22 -04:00
parent a04c1f3e66 5f159dd6d7
commit 6158d10aac
20 changed files with 712 additions and 39 deletions
--- a/docs/06-monitoring.adoc
+++ b/docs/06-monitoring.adoc
@@ -119,4 +119,61 @@ This dashboard contains:

 - JSON RPC upstream conn seconds 50,75,90,99 percentiles

+== Health Checks

+Dshackle provides a http endpoint to check status of the servers.
+This check is compatible with https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#http-probes[Kubernetes Liveness and Readiness Probes].
+
+By default, it's disabled, and you have to set up which blockchain are required to be available to consider Dshackle alive.
+
+.Example config:
+[source,yaml]
+----
+health:
+  port: 8082 # <1>
+  host: 127.0.0.1 # <2>
+  path: /health # <3>
+  blockchains: # <4>
+    - chain: ethereum # <5>
+      min-available: 2 # <6>
+    - chain: bitcoin
+      min-available: 1
+----
+
+<1> (optional) port to bind the Health server.
+Default: `8082`
+<2> (optional) host to bind the Health server.
+Default: `127.0.0.1`
+<3> (optional) path on the server.
+Default: `/health`.
+I.e., `http://127.0.0.1:8082/health` with default config
+<4> list of blockchain to check availability
+<5> a Blockchain to check
+<6> minimum available (i.e., fully synced) Upstreams for that blockchain
+
+With the config above the server is considered healthy if:
+
+- Dshackle has connected to at least two valid Ethereum upstreams
+- **and** at least one valid Bitcoin upstream.
+
+When the server is healthy is responds with `OK` and 200 as HTTP Status Code.
+When any of the checks failed, it responds with a short description and 503 as HTTP Status Code.
+
+Example of a response for an unhealthy server that doesn't have enough upstreams for a Ethereum Classic Blockchain.
+
+.GET http://127.0.0.1:8082/health
+----
+ETHEREUM_CLASSIC UNAVAILABLE
+----
+
+Optionally, the server can be called with `?detailed` query, which provides a more detailed response:
+
+.GET http://127.0.0.1:8082/health?detailed
+----
+ETHEREUM_CLASSIC UNAVAILABLE
+BITCOIN AVAILABLE
+  local-btc-1 OK with lag=0
+ETHEREUM AVAILABLE
+  local-eth-1 OK with lag=0
+  local-eth-2 OK with lag=0
+----
--- a/docs/reference-configuration.adoc
+++ b/docs/reference-configuration.adoc
@@ -31,6 +31,14 @@ monitoring:
    port: 8081
    path: /metrics

+health:
+  port: 8082
+  host: 127.0.0.1
+  path: /health
+  blockchains:
+   - chain: ethereum
+     min-availability: 1
+
 cache:
  redis:
    enabled: true
@@ -164,6 +172,10 @@ See <<tls>> section
 | Setup Prometheus monitoring.
 See <<monitoring>> section

+| `health`
+|
+| Setup Health Check endpoint See <<health>> section
+
 | `proxy`
 |
 | Setup HTTP proxy that emulates all standard JSON RPC requests.
@@ -285,6 +297,52 @@ _Reserved for future use_, in case of multiple different types of endpoints.

 |===

+[#health]
+== Health Check endpoint
+
+[source,yaml]
+----
+health:
+  port: 8082
+  host: 127.0.0.1
+  path: /health
+  blockchains:
+    - chain: ethereum
+      min-available: 2
+    - chain: bitcoin
+      min-available: 1
+----
+
+[cols="2a,2a,5"]
+|===
+| Option | Default Value | Description
+
+| `port`
+| `8082`
+| HTTP port to bind the server
+
+| `host`
+| `127.0.0.1`
+| HTTP host to bind the server
+
+| `path`
+| `/health`
+| HTTP path to respond on requests
+
+| `blockchains`
+|
+| List of blockchains that must be available to consider the server _healthy_
+
+| `[blockchain].chain`
+|
+| Blockchain id
+
+| `[blockchain].min-available`
+| 1
+| How many _available_ upstreams for the blockchain is required to pass
+
+|===
+
 [#proxy]
 == Proxy config

--- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt
@@ -17,6 +17,7 @@
 package io.emeraldpay.dshackle

 import io.emeraldpay.dshackle.config.CacheConfig
+import io.emeraldpay.dshackle.config.HealthConfig
 import io.emeraldpay.dshackle.config.MainConfig
 import io.emeraldpay.dshackle.config.MainConfigReader
 import io.emeraldpay.dshackle.config.MonitoringConfig
@@ -120,4 +121,9 @@ open class Config(
    open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig {
        return mainConfig.monitoring
    }
+
+    @Bean
+    open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig {
+        return mainConfig.health
+    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt
@@ -27,7 +27,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
 import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
+import io.emeraldpay.grpc.Chain
 import java.text.SimpleDateFormat
+import java.util.Locale
 import java.util.TimeZone
 import java.util.concurrent.Executors
 import java.util.concurrent.ScheduledExecutorService
@@ -38,6 +40,35 @@ class Global {

        var metricsExtended = false

+        val chainNames = mapOf(
+            "ethereum" to Chain.ETHEREUM,
+            "ethereum-classic" to Chain.ETHEREUM_CLASSIC,
+            "eth" to Chain.ETHEREUM,
+            "polygon" to Chain.MATIC,
+            "matic" to Chain.MATIC,
+            "etc" to Chain.ETHEREUM_CLASSIC,
+            "morden" to Chain.TESTNET_MORDEN,
+            "kovan" to Chain.TESTNET_KOVAN,
+            "kovan-testnet" to Chain.TESTNET_KOVAN,
+            "goerli" to Chain.TESTNET_GOERLI,
+            "goerli-testnet" to Chain.TESTNET_GOERLI,
+            "rinkeby" to Chain.TESTNET_RINKEBY,
+            "rinkeby-testnet" to Chain.TESTNET_RINKEBY,
+            "ropsten" to Chain.TESTNET_ROPSTEN,
+            "ropsten-testnet" to Chain.TESTNET_ROPSTEN,
+            "bitcoin" to Chain.BITCOIN,
+            "bitcoin-testnet" to Chain.TESTNET_BITCOIN
+        )
+
+        fun chainById(id: String?): Chain {
+            if (id == null) {
+                return Chain.UNSPECIFIED
+            }
+            return chainNames[
+                id.lowercase(Locale.getDefault()).replace("_", "-").trim()
+            ] ?: Chain.UNSPECIFIED
+        }
+
        @JvmStatic
        val objectMapper: ObjectMapper = createObjectMapper()

--- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.grpc.Chain
+
+class HealthConfig {
+
+    companion object {
+        fun default(): HealthConfig {
+            return HealthConfig()
+        }
+    }
+
+    var port: Int = 8082
+    var host: String = "127.0.0.1"
+    var path: String = "/health"
+    val chains = HashMap<Chain, ChainConfig>()
+
+    fun isEnabled(): Boolean {
+        return chains.isNotEmpty()
+    }
+
+    fun configs(): Collection<ChainConfig> {
+        return chains.values
+    }
+
+    data class ChainConfig(
+        val blockchain: Chain,
+        val minAvailable: Int = 1
+    )
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt
@@ -0,0 +1,79 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.dshackle.Global
+import io.emeraldpay.grpc.Chain
+import org.slf4j.LoggerFactory
+import org.yaml.snakeyaml.nodes.CollectionNode
+import org.yaml.snakeyaml.nodes.MappingNode
+import java.io.InputStream
+
+class HealthConfigReader : YamlConfigReader(), ConfigReader<HealthConfig> {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(HealthConfigReader::class.java)
+    }
+
+    fun read(input: InputStream): HealthConfig {
+        val configNode = readNode(input)
+        return read(configNode)
+    }
+
+    override fun read(input: MappingNode?): HealthConfig {
+        return readInternal(getMapping(input, "health"))
+    }
+
+    fun readInternal(input: MappingNode?): HealthConfig {
+        if (input == null) {
+            return HealthConfig.default()
+        }
+        val config = HealthConfig()
+        getValueAsString(input, "host")?.let {
+            config.host = it
+        }
+        getValueAsInt(input, "port")?.let {
+            config.port = it
+        }
+        getValueAsString(input, "path")?.let {
+            config.path = it
+        }
+        readBlockchains(config, getList<MappingNode>(input, "blockchains"))
+        return config
+    }
+
+    fun readBlockchains(healthConfig: HealthConfig, input: CollectionNode<MappingNode>?) {
+        if (input == null) {
+            return
+        }
+        input.value.forEach { conf ->
+            val chain = getValueAsString(conf, "chain")
+                ?.let { Global.chainById(it) }
+            if (chain == null) {
+                log.warn("Blockchain is not specified for a Health Check")
+                return@forEach
+            }
+            if (chain == Chain.UNSPECIFIED) {
+                log.warn("Using UNSPECIFIED blockchain for Health Check. Always fails")
+            }
+            if (healthConfig.chains.containsKey(chain)) {
+                log.warn("Duplicate Health Check config for $chain. Replace previous with new")
+            }
+            val minAvailable = getValueAsInt(conf, "min-available") ?: 1
+            healthConfig.chains[chain] = HealthConfig.ChainConfig(chain, minAvailable.coerceAtLeast(0))
+        }
+    }
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt
@@ -25,4 +25,5 @@ class MainConfig {
    var tokens: TokensConfig? = null
    var monitoring: MonitoringConfig = MonitoringConfig.default()
    var accessLogConfig: AccessLogConfig = AccessLogConfig.default()
+    var health: HealthConfig = HealthConfig.default()
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt
@@ -35,6 +35,7 @@ class MainConfigReader(
    private val tokensConfigReader = TokensConfigReader()
    private val monitoringConfigReader = MonitoringConfigReader()
    private val accessLogReader = AccessLogReader()
+    private val healthConfigReader = HealthConfigReader()

    fun read(input: InputStream): MainConfig? {
        val configNode = readNode(input)
@@ -71,6 +72,9 @@ class MainConfigReader(
        accessLogReader.read(input).let {
            config.accessLogConfig = it
        }
+        healthConfigReader.read(input).let {
+            config.health = it
+        }
        return config
    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt
@@ -16,6 +16,7 @@
 */
 package io.emeraldpay.dshackle.config

+import io.emeraldpay.dshackle.Global
 import io.emeraldpay.grpc.Chain
 import org.apache.commons.lang3.StringUtils
 import org.slf4j.LoggerFactory
@@ -69,10 +70,10 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
                }
                currentRoutes.add(id)
                val blockchain = getValueAsString(route, "blockchain")
-                if (StringUtils.isEmpty(blockchain) || getBlockchain(blockchain!!) == Chain.UNSPECIFIED) {
+                if (StringUtils.isEmpty(blockchain) || Global.chainById(blockchain!!) == Chain.UNSPECIFIED) {
                    throw InvalidConfigYamlException(filename, route.startMark, "Invalid blockchain or not specified")
                }
-                ProxyConfig.Route(id, getBlockchain(blockchain))
+                ProxyConfig.Route(id, Global.chainById(blockchain))
            }
        }
        if (config.routes.isEmpty()) {
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt
@@ -15,6 +15,7 @@
 */
 package io.emeraldpay.dshackle.config

+import io.emeraldpay.dshackle.Global
 import org.slf4j.LoggerFactory
 import org.yaml.snakeyaml.nodes.MappingNode
 import java.io.InputStream
@@ -34,7 +35,7 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
            val token = TokensConfig.Token()
            token.id = getValueAsString(node, "id")
            token.blockchain = getValueAsString(node, "blockchain")?.let {
-                getBlockchain(it)
+                Global.chainById(it)
            }
            token.address = getValueAsString(node, "address")
            token.name = getValueAsString(node, "name")
--- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt
@@ -16,7 +16,6 @@
 */
 package io.emeraldpay.dshackle.config

-import io.emeraldpay.grpc.Chain
 import org.yaml.snakeyaml.Yaml
 import org.yaml.snakeyaml.nodes.CollectionNode
 import org.yaml.snakeyaml.nodes.MappingNode
@@ -142,14 +141,4 @@ abstract class YamlConfigReader {
            base * multiplier
        }
    }
-
-    // ----
-
-    fun getBlockchain(id: String): Chain {
-        return Chain.values().find { chain ->
-            chain.name == id.uppercase(Locale.getDefault()) ||
-                chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) ||
-                chain.id.toString() == id
-        } ?: Chain.UNSPECIFIED
-    }
 }
--- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt
@@ -0,0 +1,142 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.monitoring
+
+import com.sun.net.httpserver.HttpServer
+import io.emeraldpay.dshackle.config.HealthConfig
+import io.emeraldpay.dshackle.upstream.MultistreamHolder
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.http.HttpStatus
+import org.springframework.stereotype.Service
+import java.io.IOException
+import java.net.InetSocketAddress
+import javax.annotation.PostConstruct
+
+@Service
+class HealthCheckSetup(
+    @Autowired private val healthConfig: HealthConfig,
+    @Autowired private val multistreamHolder: MultistreamHolder
+) {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(HealthCheckSetup::class.java)
+    }
+
+    @PostConstruct
+    fun start() {
+        if (!healthConfig.isEnabled()) {
+            return
+        }
+        // use standard JVM server with a single thread blocking processing
+        // health check is a rare operation, no reason to set up anything complex
+        try {
+            log.info("Run Health Server on ${healthConfig.host}:${healthConfig.port}${healthConfig.path}")
+            val server = HttpServer.create(
+                InetSocketAddress(
+                    healthConfig.host,
+                    healthConfig.port
+                ),
+                0
+            )
+            server.createContext(healthConfig.path) { httpExchange ->
+                val response = if (httpExchange.requestURI.query == "detailed") {
+                    getDetailedHealth()
+                } else {
+                    getHealth()
+                }
+                val ok = response.ok
+                val data = response.details.joinToString("\n")
+                val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE
+                httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong())
+                httpExchange.responseBody.use { os ->
+                    os.write(data.toByteArray())
+                }
+            }
+            Thread(server::start).start()
+        } catch (e: IOException) {
+            log.error("Failed to start Health Server", e)
+        }
+    }
+
+    fun getHealth(): Detailed {
+        val errors = healthConfig.configs().mapNotNull {
+            val up = multistreamHolder.getUpstream(it.blockchain)
+            if (up == null || !up.isAvailable()) {
+                return@mapNotNull "${it.blockchain} UNAVAILABLE"
+            }
+            val avail = up.getAll().count { it.getStatus() == UpstreamAvailability.OK }
+            if (avail < it.minAvailable) {
+                return@mapNotNull "${it.blockchain} LACKS MIN AVAILABILITY"
+            }
+            null
+        }
+        return if (errors.isEmpty()) {
+            Detailed(true, listOf("OK"))
+        } else {
+            Detailed(false, errors)
+        }
+    }
+
+    fun getDetailedHealth(): Detailed {
+        val chains = multistreamHolder.getAvailable()
+        val allEnabled = healthConfig.configs().all { chains.contains(it.blockchain) }
+        var anyUnavailable = false
+        val details = chains.flatMap { chain ->
+            var chainUnavailable = false
+            val up = multistreamHolder.getUpstream(chain)
+            val required = healthConfig.chains[chain]
+            if (up == null || !up.isAvailable()) {
+                if (required != null) {
+                    anyUnavailable = true
+                }
+                listOf("${chain.name} UNAVAILABLE")
+            } else {
+                val ups = up.getAll()
+                val checks = if (required != null) {
+                    val avail = ups.count { it.getStatus() == UpstreamAvailability.OK }
+                    if (avail < required.minAvailable) {
+                        chainUnavailable = true
+                        listOf("  LACKS MIN AVAILABILITY")
+                    } else emptyList()
+                } else emptyList()
+                val upDetails = ups.map {
+                    "  ${it.getId()} ${it.getStatus()} with lag=${it.getLag()}"
+                }
+                val status = if (chainUnavailable) "UNAVAILABLE" else "AVAILABLE"
+                anyUnavailable = anyUnavailable || chainUnavailable
+                listOf("${chain.name} $status") + upDetails + checks
+            }
+        }
+        val detailsUnavailable = if (!allEnabled) {
+            healthConfig.configs()
+                .filter { !chains.contains(it.blockchain) }
+                .map {
+                    "${it.blockchain.name} UNAVAILABLE"
+                }
+        } else emptyList()
+        return Detailed(
+            allEnabled && !anyUnavailable,
+            detailsUnavailable + details
+        )
+    }
+
+    data class Detailed(
+        val ok: Boolean,
+        val details: List<String>
+    )
+}
--- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
+++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
@@ -17,6 +17,7 @@
 package io.emeraldpay.dshackle.startup

 import io.emeraldpay.dshackle.FileResolver
+import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.cache.CachesFactory
 import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.reader.Reader
@@ -57,26 +58,6 @@ open class ConfiguredUpstreams(
    private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
    private var seq = AtomicInteger(0)

-    private val chainNames = mapOf(
-        "ethereum" to Chain.ETHEREUM,
-        "ethereum-classic" to Chain.ETHEREUM_CLASSIC,
-        "eth" to Chain.ETHEREUM,
-        "polygon" to Chain.MATIC,
-        "matic" to Chain.MATIC,
-        "etc" to Chain.ETHEREUM_CLASSIC,
-        "morden" to Chain.TESTNET_MORDEN,
-        "kovan" to Chain.TESTNET_KOVAN,
-        "kovan-testnet" to Chain.TESTNET_KOVAN,
-        "goerli" to Chain.TESTNET_GOERLI,
-        "goerli-testnet" to Chain.TESTNET_GOERLI,
-        "rinkeby" to Chain.TESTNET_RINKEBY,
-        "rinkeby-testnet" to Chain.TESTNET_RINKEBY,
-        "ropsten" to Chain.TESTNET_ROPSTEN,
-        "ropsten-testnet" to Chain.TESTNET_ROPSTEN,
-        "bitcoin" to Chain.BITCOIN,
-        "bitcoin-testnet" to Chain.TESTNET_BITCOIN
-    )
-
    @PostConstruct
    fun start() {
        log.debug("Starting upstreams")
@@ -87,8 +68,8 @@ open class ConfiguredUpstreams(
                val options = up.options ?: UpstreamsConfig.Options()
                buildGrpcUpstream(up.cast(UpstreamsConfig.GrpcConnection::class.java), options)
            } else {
-                val chain = chainNames[up.chain]
-                if (chain == null) {
+                val chain = Global.chainById(up.chain)
+                if (chain == Chain.UNSPECIFIED) {
                    log.error("Chain is unknown: ${up.chain}")
                    return@forEach
                }
@@ -114,7 +95,7 @@ open class ConfiguredUpstreams(
        val defaultOptions = HashMap<Chain, UpstreamsConfig.Options>()
        config.defaultOptions.forEach { defaultsConfig ->
            defaultsConfig.chains?.forEach { chainName ->
-                chainNames[chainName]?.let { chain ->
+                Global.chainById(chainName).let { chain ->
                    defaultsConfig.options?.let { options ->
                        if (!defaultOptions.containsKey(chain)) {
                            defaultOptions[chain] = options
@@ -274,7 +255,7 @@ open class ConfiguredUpstreams(
                // "unknown" is not supposed to happen
                Tag.of("upstream", config.id ?: "unknown"),
                // UNSPECIFIED shouldn't happen too
-                Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode)
+                Tag.of("chain", (Global.chainById(config.chain).chainCode))
            )
            val metrics = RpcMetrics(
                Timer.builder("upstream.rpc.conn")
--- a/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy
+++ b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy
@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.grpc.Chain
+import spock.lang.Specification
+
+class HealthConfigReaderSpec extends Specification {
+
+    HealthConfigReader reader = new HealthConfigReader()
+
+    def "Read empty"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-empty.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        !act.enabled
+        act.port == 8082
+        act.host == "127.0.0.1"
+        act.path == "/health"
+        act.configs().size() == 0
+    }
+
+    def "Read single"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-1.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        act.enabled
+        act.port == 8082
+        act.host == "127.0.0.1"
+        act.path == "/health"
+        with(act.configs()) {
+            size() == 1
+            with(it[0]) {
+                it.blockchain == Chain.ETHEREUM
+                it.minAvailable == 2
+            }
+        }
+    }
+
+    def "Read multiple"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-2.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        act.enabled
+        act.port == 10003
+        act.host == "0.0.0.0"
+        act.path == "/healtz"
+        with(act.configs().toSorted { it.blockchain.id }) {
+            size() == 2
+            with(it[0]) {
+                it.blockchain == Chain.BITCOIN
+                it.minAvailable == 1
+            }
+            with(it[1]) {
+                it.blockchain == Chain.ETHEREUM
+                it.minAvailable == 2
+            }
+        }
+    }
+}
--- a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy
+++ b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy
@@ -15,7 +15,7 @@
 */
 package io.emeraldpay.dshackle.config

-import io.emeraldpay.dshackle.FileResolver
+
 import io.emeraldpay.dshackle.test.TestingCommons
 import io.emeraldpay.grpc.Chain
 import spock.lang.Specification
@@ -73,6 +73,16 @@ class MainConfigReaderSpec extends Specification {
                blockchain == Chain.TESTNET_RINKEBY
            }
        }
+        with(act.health) {
+            it.enabled
+            with(it.configs()) {
+                it.size() == 1
+                with(it[0]) {
+                    it.blockchain == Chain.ETHEREUM
+                    it.minAvailable == 1
+                }
+            }
+        }
        act.upstreams != null
        with(act.upstreams) {
            defaultOptions != null
--- a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy
+++ b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy
@@ -0,0 +1,160 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.monitoring
+
+import io.emeraldpay.dshackle.config.HealthConfig
+import io.emeraldpay.dshackle.upstream.Multistream
+import io.emeraldpay.dshackle.upstream.MultistreamHolder
+import io.emeraldpay.dshackle.upstream.Upstream
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import io.emeraldpay.grpc.Chain
+import spock.lang.Specification
+
+class HealthCheckSetupSpec extends Specification {
+
+    def "OK when meets availability - 1"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 1
+            )
+        }
+        def up1 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act.ok
+        act.details == ["OK"]
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1]
+        1 * up1.status >> UpstreamAvailability.OK
+    }
+
+    def "OK when meets availability - 1 - bitcoin"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.BITCOIN] = new HealthConfig.ChainConfig(
+                    Chain.BITCOIN, 1
+            )
+        }
+        def up1 = Mock(Upstream)
+        def bitcoinUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act.ok
+        act.details == ["OK"]
+        1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams
+        1 * bitcoinUpstreams.available >> true
+        1 * bitcoinUpstreams.getAll() >> [up1]
+        1 * up1.status >> UpstreamAvailability.OK
+    }
+
+    def "OK when meets availability - 2/3"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 2
+            )
+        }
+        def up1 = Mock(Upstream)
+        def up2 = Mock(Upstream)
+        def up3 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act.ok
+        act.details == ["OK"]
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
+        1 * up1.status >> UpstreamAvailability.OK
+        1 * up2.status >> UpstreamAvailability.SYNCING
+        1 * up3.status >> UpstreamAvailability.OK
+    }
+
+    def "OK when doesn't meet availability - 2/3"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 2
+            )
+        }
+        def up1 = Mock(Upstream)
+        def up2 = Mock(Upstream)
+        def up3 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        !act.ok
+        act.details != ["OK"]
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
+        1 * up1.status >> UpstreamAvailability.OK
+        1 * up2.status >> UpstreamAvailability.SYNCING
+        1 * up3.status >> UpstreamAvailability.LAGGING
+    }
+
+    def "OK when meets availability - 2/3 - detailed"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 2
+            )
+        }
+        def up1 = Mock(Upstream)
+        def up2 = Mock(Upstream)
+        def up3 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.detailedHealth
+
+        then:
+        act.ok
+        act.details.size() > 1
+        1 * multistream.getAvailable() >> [Chain.ETHEREUM]
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
+        _ * up1.status >> UpstreamAvailability.OK
+        _ * up2.status >> UpstreamAvailability.SYNCING
+        _ * up3.status >> UpstreamAvailability.OK
+    }
+}
--- a/src/test/resources/dshackle-full.yaml
+++ b/src/test/resources/dshackle-full.yaml
@@ -16,6 +16,12 @@ cache:
    enabled: true
    host: redis-master

+health:
+  port: 8083
+  blockchains:
+    - chain: ethereum
+      min-available: 1
+
 proxy:
  port: 8082
  tls:
--- a/src/test/resources/dshackle-health-1.yaml
+++ b/src/test/resources/dshackle-health-1.yaml
@@ -0,0 +1,6 @@
+version: v1
+
+health:
+  blockchains:
+    - chain: ethereum
+      min-available: 2
--- a/src/test/resources/dshackle-health-2.yaml
+++ b/src/test/resources/dshackle-health-2.yaml
@@ -0,0 +1,11 @@
+version: v1
+
+health:
+  port: 10003
+  host: 0.0.0.0
+  path: /healtz
+  blockchains:
+    - chain: ethereum
+      min-available: 2
+    - chain: bitcoin
+      min-available: 1
--- a/src/test/resources/dshackle-health-empty.yaml
+++ b/src/test/resources/dshackle-health-empty.yaml
@@ -0,0 +1,3 @@
+version: v1
+
+health: