diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index 78650ea8..e45f8920 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -27,8 +27,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.grpc.Chain import java.text.SimpleDateFormat -import java.util.TimeZone +import java.util.* import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService @@ -38,6 +39,35 @@ class Global { var metricsExtended = false + val chainNames = mapOf( + "ethereum" to Chain.ETHEREUM, + "ethereum-classic" to Chain.ETHEREUM_CLASSIC, + "eth" to Chain.ETHEREUM, + "polygon" to Chain.MATIC, + "matic" to Chain.MATIC, + "etc" to Chain.ETHEREUM_CLASSIC, + "morden" to Chain.TESTNET_MORDEN, + "kovan" to Chain.TESTNET_KOVAN, + "kovan-testnet" to Chain.TESTNET_KOVAN, + "goerli" to Chain.TESTNET_GOERLI, + "goerli-testnet" to Chain.TESTNET_GOERLI, + "rinkeby" to Chain.TESTNET_RINKEBY, + "rinkeby-testnet" to Chain.TESTNET_RINKEBY, + "ropsten" to Chain.TESTNET_ROPSTEN, + "ropsten-testnet" to Chain.TESTNET_ROPSTEN, + "bitcoin" to Chain.BITCOIN, + "bitcoin-testnet" to Chain.TESTNET_BITCOIN + ) + + fun chainById(id: String?): Chain { + if (id == null) { + return Chain.UNSPECIFIED + } + return chainNames[ + id.lowercase(Locale.getDefault()).replace("_", "-").trim() + ] ?: Chain.UNSPECIFIED + } + @JvmStatic val objectMapper: ObjectMapper = createObjectMapper() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt index 1058f6f4..ebf35517 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.CollectionNode @@ -60,7 +61,7 @@ class HealthConfigReader : YamlConfigReader(), ConfigReader { } input.value.forEach { conf -> val chain = getValueAsString(conf, "chain") - ?.let { getBlockchain(it) } + ?.let { Global.chainById(it) } if (chain == null) { log.warn("Blockchain is not specified for a Health Check") return@forEach diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt index 4fcc9cae..e75f80c4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import io.emeraldpay.grpc.Chain import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory @@ -69,10 +70,10 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader { } currentRoutes.add(id) val blockchain = getValueAsString(route, "blockchain") - if (StringUtils.isEmpty(blockchain) || getBlockchain(blockchain!!) == Chain.UNSPECIFIED) { + if (StringUtils.isEmpty(blockchain) || Global.chainById(blockchain!!) == Chain.UNSPECIFIED) { throw InvalidConfigYamlException(filename, route.startMark, "Invalid blockchain or not specified") } - ProxyConfig.Route(id, getBlockchain(blockchain)) + ProxyConfig.Route(id, Global.chainById(blockchain)) } } if (config.routes.isEmpty()) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt index 9d4f6da3..85205277 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream @@ -34,7 +35,7 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader { val token = TokensConfig.Token() token.id = getValueAsString(node, "id") token.blockchain = getValueAsString(node, "blockchain")?.let { - getBlockchain(it) + Global.chainById(it) } token.address = getValueAsString(node, "address") token.name = getValueAsString(node, "name") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt index 414e7322..517849b8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt @@ -143,13 +143,4 @@ abstract class YamlConfigReader { } } - // ---- - - fun getBlockchain(id: String): Chain { - return Chain.values().find { chain -> - chain.name == id.uppercase(Locale.getDefault()) || - chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) || - chain.id.toString() == id - } ?: Chain.UNSPECIFIED - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt index 76c62d3b..c333ae01 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -54,12 +54,17 @@ class HealthCheckSetup( 0 ) server.createContext(healthConfig.path) { httpExchange -> - val response = getHealth() - val ok = response == "OK" + val response = if (httpExchange.requestURI.query == "detailed") { + getDetailedHealth() + } else { + getHealth() + } + val ok = response.ok + val data = response.details.joinToString("\n") val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE - httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong()) + httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong()) httpExchange.responseBody.use { os -> - os.write(response.toByteArray()) + os.write(data.toByteArray()) } } Thread(server::start).start() @@ -68,7 +73,7 @@ class HealthCheckSetup( } } - fun getHealth(): String { + fun getHealth(): Detailed { val errors = healthConfig.configs().mapNotNull { val up = multistreamHolder.getUpstream(it.blockchain) if (up == null || !up.isAvailable()) { @@ -81,9 +86,57 @@ class HealthCheckSetup( null } return if (errors.isEmpty()) { - "OK" + Detailed(true, listOf("OK")) } else { - errors.joinToString("\n") + Detailed(false, errors) } } + + fun getDetailedHealth(): Detailed { + val chains = multistreamHolder.getAvailable() + val allEnabled = healthConfig.configs().all { chains.contains(it.blockchain) } + var anyUnavailable = false + val details = chains.flatMap { chain -> + var chainUnavailable = false + val up = multistreamHolder.getUpstream(chain) + val required = healthConfig.chains[chain] + if (up == null || !up.isAvailable()) { + if (required != null) { + anyUnavailable = true + } + listOf("${chain.name} UNAVAILABLE") + } else { + val ups = up.getAll() + val checks = if (required != null) { + val avail = ups.count { it.getStatus() == UpstreamAvailability.OK } + if (avail < required.minAvailable) { + chainUnavailable = true + listOf(" LACKS MIN AVAILABILITY") + } else emptyList() + } else emptyList() + val upDetails = ups.map { + " ${it.getId()} ${it.getStatus()} with lag=${it.getLag()}" + } + val status = if (chainUnavailable) "UNAVAILABLE" else "AVAILABLE" + anyUnavailable = anyUnavailable || chainUnavailable + listOf("${chain.name} $status") + upDetails + checks + } + } + val detailsUnavailable = if (!allEnabled) { + healthConfig.configs() + .filter { !chains.contains(it.blockchain) } + .map { + "${it.blockchain.name} UNAVAILABLE" + } + } else emptyList() + return Detailed( + allEnabled && !anyUnavailable, + detailsUnavailable + details + ) + } + + data class Detailed( + val ok: Boolean, + val details: List + ) } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index fdf1dadf..d7951d0e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.startup import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.cache.CachesFactory import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader @@ -57,26 +58,6 @@ open class ConfiguredUpstreams( private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) private var seq = AtomicInteger(0) - private val chainNames = mapOf( - "ethereum" to Chain.ETHEREUM, - "ethereum-classic" to Chain.ETHEREUM_CLASSIC, - "eth" to Chain.ETHEREUM, - "polygon" to Chain.MATIC, - "matic" to Chain.MATIC, - "etc" to Chain.ETHEREUM_CLASSIC, - "morden" to Chain.TESTNET_MORDEN, - "kovan" to Chain.TESTNET_KOVAN, - "kovan-testnet" to Chain.TESTNET_KOVAN, - "goerli" to Chain.TESTNET_GOERLI, - "goerli-testnet" to Chain.TESTNET_GOERLI, - "rinkeby" to Chain.TESTNET_RINKEBY, - "rinkeby-testnet" to Chain.TESTNET_RINKEBY, - "ropsten" to Chain.TESTNET_ROPSTEN, - "ropsten-testnet" to Chain.TESTNET_ROPSTEN, - "bitcoin" to Chain.BITCOIN, - "bitcoin-testnet" to Chain.TESTNET_BITCOIN - ) - @PostConstruct fun start() { log.debug("Starting upstreams") @@ -87,8 +68,8 @@ open class ConfiguredUpstreams( val options = up.options ?: UpstreamsConfig.Options() buildGrpcUpstream(up.cast(UpstreamsConfig.GrpcConnection::class.java), options) } else { - val chain = chainNames[up.chain] - if (chain == null) { + val chain = Global.chainById(up.chain) + if (chain == Chain.UNSPECIFIED) { log.error("Chain is unknown: ${up.chain}") return@forEach } @@ -114,7 +95,7 @@ open class ConfiguredUpstreams( val defaultOptions = HashMap() config.defaultOptions.forEach { defaultsConfig -> defaultsConfig.chains?.forEach { chainName -> - chainNames[chainName]?.let { chain -> + Global.chainById(chainName).let { chain -> defaultsConfig.options?.let { options -> if (!defaultOptions.containsKey(chain)) { defaultOptions[chain] = options @@ -274,7 +255,7 @@ open class ConfiguredUpstreams( // "unknown" is not supposed to happen Tag.of("upstream", config.id ?: "unknown"), // UNSPECIFIED shouldn't happen too - Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode) + Tag.of("chain", (Global.chainById(config.chain).chainCode)) ) val metrics = RpcMetrics( Timer.builder("upstream.rpc.conn") diff --git a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy index b9f6d4da..c4772852 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy @@ -41,7 +41,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act == "OK" + act.ok + act.details == ["OK"] 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams 1 * ethereumUpstreams.available >> true 1 * ethereumUpstreams.getAll() >> [up1] @@ -64,7 +65,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act == "OK" + act.ok + act.details == ["OK"] 1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams 1 * bitcoinUpstreams.available >> true 1 * bitcoinUpstreams.getAll() >> [up1] @@ -89,7 +91,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act == "OK" + act.ok + act.details == ["OK"] 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams 1 * ethereumUpstreams.available >> true 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] @@ -116,7 +119,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act != "OK" + !act.ok + act.details != ["OK"] 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams 1 * ethereumUpstreams.available >> true 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] @@ -124,4 +128,33 @@ class HealthCheckSetupSpec extends Specification { 1 * up2.status >> UpstreamAvailability.SYNCING 1 * up3.status >> UpstreamAvailability.LAGGING } + + def "OK when meets availability - 2/3 - detailed"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.detailedHealth + + then: + act.ok + act.details.size() > 1 + 1 * multistream.getAvailable() >> [Chain.ETHEREUM] + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + _ * up1.status >> UpstreamAvailability.OK + _ * up2.status >> UpstreamAvailability.SYNCING + _ * up3.status >> UpstreamAvailability.OK + } }