solution: health check with detailed description of health

src/main/kotlin/io/emeraldpay/dshackle/Global.kt

@@ -27,8 +27,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
 import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
 import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
+import io.emeraldpay.grpc.Chain
 import java.text.SimpleDateFormat
-import java.util.TimeZone
+import java.util.*
 import java.util.concurrent.Executors
 import java.util.concurrent.ScheduledExecutorService
 
@@ -38,6 +39,35 @@ class Global {
 
         var metricsExtended = false
 
+        val chainNames = mapOf(
+            "ethereum" to Chain.ETHEREUM,
+            "ethereum-classic" to Chain.ETHEREUM_CLASSIC,
+            "eth" to Chain.ETHEREUM,
+            "polygon" to Chain.MATIC,
+            "matic" to Chain.MATIC,
+            "etc" to Chain.ETHEREUM_CLASSIC,
+            "morden" to Chain.TESTNET_MORDEN,
+            "kovan" to Chain.TESTNET_KOVAN,
+            "kovan-testnet" to Chain.TESTNET_KOVAN,
+            "goerli" to Chain.TESTNET_GOERLI,
+            "goerli-testnet" to Chain.TESTNET_GOERLI,
+            "rinkeby" to Chain.TESTNET_RINKEBY,
+            "rinkeby-testnet" to Chain.TESTNET_RINKEBY,
+            "ropsten" to Chain.TESTNET_ROPSTEN,
+            "ropsten-testnet" to Chain.TESTNET_ROPSTEN,
+            "bitcoin" to Chain.BITCOIN,
+            "bitcoin-testnet" to Chain.TESTNET_BITCOIN
+        )
+
+        fun chainById(id: String?): Chain {
+            if (id == null) {
+                return Chain.UNSPECIFIED
+            }
+            return chainNames[
+                id.lowercase(Locale.getDefault()).replace("_", "-").trim()
+            ] ?: Chain.UNSPECIFIED
+        }
+
         @JvmStatic
         val objectMapper: ObjectMapper = createObjectMapper()
 

src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt

@@ -15,6 +15,7 @@
  */
 package io.emeraldpay.dshackle.config
 
+import io.emeraldpay.dshackle.Global
 import io.emeraldpay.grpc.Chain
 import org.slf4j.LoggerFactory
 import org.yaml.snakeyaml.nodes.CollectionNode
@@ -60,7 +61,7 @@ class HealthConfigReader : YamlConfigReader(), ConfigReader<HealthConfig> {
         }
         input.value.forEach { conf ->
             val chain = getValueAsString(conf, "chain")
-                ?.let { getBlockchain(it) }
+                ?.let { Global.chainById(it) }
             if (chain == null) {
                 log.warn("Blockchain is not specified for a Health Check")
                 return@forEach

src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt

@@ -16,6 +16,7 @@
  */
 package io.emeraldpay.dshackle.config
 
+import io.emeraldpay.dshackle.Global
 import io.emeraldpay.grpc.Chain
 import org.apache.commons.lang3.StringUtils
 import org.slf4j.LoggerFactory
@@ -69,10 +70,10 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
                 }
                 currentRoutes.add(id)
                 val blockchain = getValueAsString(route, "blockchain")
-                if (StringUtils.isEmpty(blockchain) || getBlockchain(blockchain!!) == Chain.UNSPECIFIED) {
+                if (StringUtils.isEmpty(blockchain) || Global.chainById(blockchain!!) == Chain.UNSPECIFIED) {
                     throw InvalidConfigYamlException(filename, route.startMark, "Invalid blockchain or not specified")
                 }
-                ProxyConfig.Route(id, getBlockchain(blockchain))
+                ProxyConfig.Route(id, Global.chainById(blockchain))
             }
         }
         if (config.routes.isEmpty()) {

src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt

@@ -15,6 +15,7 @@
  */
 package io.emeraldpay.dshackle.config
 
+import io.emeraldpay.dshackle.Global
 import org.slf4j.LoggerFactory
 import org.yaml.snakeyaml.nodes.MappingNode
 import java.io.InputStream
@@ -34,7 +35,7 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
             val token = TokensConfig.Token()
             token.id = getValueAsString(node, "id")
             token.blockchain = getValueAsString(node, "blockchain")?.let {
-                getBlockchain(it)
+                Global.chainById(it)
             }
             token.address = getValueAsString(node, "address")
             token.name = getValueAsString(node, "name")

src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt

@@ -143,13 +143,4 @@ abstract class YamlConfigReader {
         }
     }
 
-    // ----
-
-    fun getBlockchain(id: String): Chain {
-        return Chain.values().find { chain ->
-            chain.name == id.uppercase(Locale.getDefault()) ||
-                chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) ||
-                chain.id.toString() == id
-        } ?: Chain.UNSPECIFIED
-    }
 }

src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt

@@ -54,12 +54,17 @@ class HealthCheckSetup(
                 0
             )
             server.createContext(healthConfig.path) { httpExchange ->
-                val response = getHealth()
+                val response = if (httpExchange.requestURI.query == "detailed") {
-                val ok = response == "OK"
+                    getDetailedHealth()
+                } else {
+                    getHealth()
+                }
+                val ok = response.ok
+                val data = response.details.joinToString("\n")
                 val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE
-                httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong())
+                httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong())
                 httpExchange.responseBody.use { os ->
-                    os.write(response.toByteArray())
+                    os.write(data.toByteArray())
                 }
             }
             Thread(server::start).start()
@@ -68,7 +73,7 @@ class HealthCheckSetup(
         }
     }
 
-    fun getHealth(): String {
+    fun getHealth(): Detailed {
         val errors = healthConfig.configs().mapNotNull {
             val up = multistreamHolder.getUpstream(it.blockchain)
             if (up == null || !up.isAvailable()) {
@@ -81,9 +86,57 @@ class HealthCheckSetup(
             null
         }
         return if (errors.isEmpty()) {
-            "OK"
+            Detailed(true, listOf("OK"))
         } else {
-            errors.joinToString("\n")
+            Detailed(false, errors)
         }
     }
+
+    fun getDetailedHealth(): Detailed {
+        val chains = multistreamHolder.getAvailable()
+        val allEnabled = healthConfig.configs().all { chains.contains(it.blockchain) }
+        var anyUnavailable = false
+        val details = chains.flatMap { chain ->
+            var chainUnavailable = false
+            val up = multistreamHolder.getUpstream(chain)
+            val required = healthConfig.chains[chain]
+            if (up == null || !up.isAvailable()) {
+                if (required != null) {
+                    anyUnavailable = true
+                }
+                listOf("${chain.name} UNAVAILABLE")
+            } else {
+                val ups = up.getAll()
+                val checks = if (required != null) {
+                    val avail = ups.count { it.getStatus() == UpstreamAvailability.OK }
+                    if (avail < required.minAvailable) {
+                        chainUnavailable = true
+                        listOf("  LACKS MIN AVAILABILITY")
+                    } else emptyList()
+                } else emptyList()
+                val upDetails = ups.map {
+                    "  ${it.getId()} ${it.getStatus()} with lag=${it.getLag()}"
+                }
+                val status = if (chainUnavailable) "UNAVAILABLE" else "AVAILABLE"
+                anyUnavailable = anyUnavailable || chainUnavailable
+                listOf("${chain.name} $status") + upDetails + checks
+            }
+        }
+        val detailsUnavailable = if (!allEnabled) {
+            healthConfig.configs()
+                .filter { !chains.contains(it.blockchain) }
+                .map {
+                    "${it.blockchain.name} UNAVAILABLE"
+                }
+        } else emptyList()
+        return Detailed(
+            allEnabled && !anyUnavailable,
+            detailsUnavailable + details
+        )
+    }
+
+    data class Detailed(
+        val ok: Boolean,
+        val details: List<String>
+    )
 }

src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt

@@ -17,6 +17,7 @@
 package io.emeraldpay.dshackle.startup
 
 import io.emeraldpay.dshackle.FileResolver
+import io.emeraldpay.dshackle.Global
 import io.emeraldpay.dshackle.cache.CachesFactory
 import io.emeraldpay.dshackle.config.UpstreamsConfig
 import io.emeraldpay.dshackle.reader.Reader
@@ -57,26 +58,6 @@ open class ConfiguredUpstreams(
     private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
     private var seq = AtomicInteger(0)
 
-    private val chainNames = mapOf(
-        "ethereum" to Chain.ETHEREUM,
-        "ethereum-classic" to Chain.ETHEREUM_CLASSIC,
-        "eth" to Chain.ETHEREUM,
-        "polygon" to Chain.MATIC,
-        "matic" to Chain.MATIC,
-        "etc" to Chain.ETHEREUM_CLASSIC,
-        "morden" to Chain.TESTNET_MORDEN,
-        "kovan" to Chain.TESTNET_KOVAN,
-        "kovan-testnet" to Chain.TESTNET_KOVAN,
-        "goerli" to Chain.TESTNET_GOERLI,
-        "goerli-testnet" to Chain.TESTNET_GOERLI,
-        "rinkeby" to Chain.TESTNET_RINKEBY,
-        "rinkeby-testnet" to Chain.TESTNET_RINKEBY,
-        "ropsten" to Chain.TESTNET_ROPSTEN,
-        "ropsten-testnet" to Chain.TESTNET_ROPSTEN,
-        "bitcoin" to Chain.BITCOIN,
-        "bitcoin-testnet" to Chain.TESTNET_BITCOIN
-    )
-
     @PostConstruct
     fun start() {
         log.debug("Starting upstreams")
@@ -87,8 +68,8 @@ open class ConfiguredUpstreams(
                 val options = up.options ?: UpstreamsConfig.Options()
                 buildGrpcUpstream(up.cast(UpstreamsConfig.GrpcConnection::class.java), options)
             } else {
-                val chain = chainNames[up.chain]
+                val chain = Global.chainById(up.chain)
-                if (chain == null) {
+                if (chain == Chain.UNSPECIFIED) {
                     log.error("Chain is unknown: ${up.chain}")
                     return@forEach
                 }
@@ -114,7 +95,7 @@ open class ConfiguredUpstreams(
         val defaultOptions = HashMap<Chain, UpstreamsConfig.Options>()
         config.defaultOptions.forEach { defaultsConfig ->
             defaultsConfig.chains?.forEach { chainName ->
-                chainNames[chainName]?.let { chain ->
+                Global.chainById(chainName).let { chain ->
                     defaultsConfig.options?.let { options ->
                         if (!defaultOptions.containsKey(chain)) {
                             defaultOptions[chain] = options
@@ -274,7 +255,7 @@ open class ConfiguredUpstreams(
                 // "unknown" is not supposed to happen
                 Tag.of("upstream", config.id ?: "unknown"),
                 // UNSPECIFIED shouldn't happen too
-                Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode)
+                Tag.of("chain", (Global.chainById(config.chain).chainCode))
             )
             val metrics = RpcMetrics(
                 Timer.builder("upstream.rpc.conn")

src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy

@@ -41,7 +41,8 @@ class HealthCheckSetupSpec extends Specification {
         def act = check.health
 
         then:
-        act == "OK"
+        act.ok
+        act.details == ["OK"]
         1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
         1 * ethereumUpstreams.available >> true
         1 * ethereumUpstreams.getAll() >> [up1]
@@ -64,7 +65,8 @@ class HealthCheckSetupSpec extends Specification {
         def act = check.health
 
         then:
-        act == "OK"
+        act.ok
+        act.details == ["OK"]
         1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams
         1 * bitcoinUpstreams.available >> true
         1 * bitcoinUpstreams.getAll() >> [up1]
@@ -89,7 +91,8 @@ class HealthCheckSetupSpec extends Specification {
         def act = check.health
 
         then:
-        act == "OK"
+        act.ok
+        act.details == ["OK"]
         1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
         1 * ethereumUpstreams.available >> true
         1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
@@ -116,7 +119,8 @@ class HealthCheckSetupSpec extends Specification {
         def act = check.health
 
         then:
-        act != "OK"
+        !act.ok
+        act.details != ["OK"]
         1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
         1 * ethereumUpstreams.available >> true
         1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
@@ -124,4 +128,33 @@ class HealthCheckSetupSpec extends Specification {
         1 * up2.status >> UpstreamAvailability.SYNCING
         1 * up3.status >> UpstreamAvailability.LAGGING
     }
+
+    def "OK when meets availability - 2/3 - detailed"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 2
+            )
+        }
+        def up1 = Mock(Upstream)
+        def up2 = Mock(Upstream)
+        def up3 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.detailedHealth
+
+        then:
+        act.ok
+        act.details.size() > 1
+        1 * multistream.getAvailable() >> [Chain.ETHEREUM]
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
+        _ * up1.status >> UpstreamAvailability.OK
+        _ * up2.status >> UpstreamAvailability.SYNCING
+        _ * up3.status >> UpstreamAvailability.OK
+    }
 }