diff --git a/docs/06-monitoring.adoc b/docs/06-monitoring.adoc index df72607c..e19d3129 100644 --- a/docs/06-monitoring.adoc +++ b/docs/06-monitoring.adoc @@ -119,4 +119,61 @@ This dashboard contains: - JSON RPC upstream conn seconds 50,75,90,99 percentiles +== Health Checks +Dshackle provides a http endpoint to check status of the servers. +This check is compatible with https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#http-probes[Kubernetes Liveness and Readiness Probes]. + +By default, it's disabled, and you have to set up which blockchain are required to be available to consider Dshackle alive. + +.Example config: +[source,yaml] +---- +health: + port: 8082 # <1> + host: 127.0.0.1 # <2> + path: /health # <3> + blockchains: # <4> + - chain: ethereum # <5> + min-available: 2 # <6> + - chain: bitcoin + min-available: 1 +---- + +<1> (optional) port to bind the Health server. +Default: `8082` +<2> (optional) host to bind the Health server. +Default: `127.0.0.1` +<3> (optional) path on the server. +Default: `/health`. +I.e., `http://127.0.0.1:8082/health` with default config +<4> list of blockchain to check availability +<5> a Blockchain to check +<6> minimum available (i.e., fully synced) Upstreams for that blockchain + +With the config above the server is considered healthy if: + +- Dshackle has connected to at least two valid Ethereum upstreams +- **and** at least one valid Bitcoin upstream. + +When the server is healthy is responds with `OK` and 200 as HTTP Status Code. +When any of the checks failed, it responds with a short description and 503 as HTTP Status Code. + +Example of a response for an unhealthy server that doesn't have enough upstreams for a Ethereum Classic Blockchain. + +.GET http://127.0.0.1:8082/health +---- +ETHEREUM_CLASSIC UNAVAILABLE +---- + +Optionally, the server can be called with `?detailed` query, which provides a more detailed response: + +.GET http://127.0.0.1:8082/health?detailed +---- +ETHEREUM_CLASSIC UNAVAILABLE +BITCOIN AVAILABLE + local-btc-1 OK with lag=0 +ETHEREUM AVAILABLE + local-eth-1 OK with lag=0 + local-eth-2 OK with lag=0 +---- diff --git a/docs/reference-configuration.adoc b/docs/reference-configuration.adoc index b5b9aea3..16f06420 100644 --- a/docs/reference-configuration.adoc +++ b/docs/reference-configuration.adoc @@ -31,6 +31,14 @@ monitoring: port: 8081 path: /metrics +health: + port: 8082 + host: 127.0.0.1 + path: /health + blockchains: + - chain: ethereum + min-availability: 1 + cache: redis: enabled: true @@ -164,6 +172,10 @@ See <> section | Setup Prometheus monitoring. See <> section +| `health` +| +| Setup Health Check endpoint See <> section + | `proxy` | | Setup HTTP proxy that emulates all standard JSON RPC requests. @@ -285,6 +297,52 @@ _Reserved for future use_, in case of multiple different types of endpoints. |=== +[#health] +== Health Check endpoint + +[source,yaml] +---- +health: + port: 8082 + host: 127.0.0.1 + path: /health + blockchains: + - chain: ethereum + min-available: 2 + - chain: bitcoin + min-available: 1 +---- + +[cols="2a,2a,5"] +|=== +| Option | Default Value | Description + +| `port` +| `8082` +| HTTP port to bind the server + +| `host` +| `127.0.0.1` +| HTTP host to bind the server + +| `path` +| `/health` +| HTTP path to respond on requests + +| `blockchains` +| +| List of blockchains that must be available to consider the server _healthy_ + +| `[blockchain].chain` +| +| Blockchain id + +| `[blockchain].min-available` +| 1 +| How many _available_ upstreams for the blockchain is required to pass + +|=== + [#proxy] == Proxy config diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index a619f181..b27dc9da 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle import io.emeraldpay.dshackle.config.CacheConfig +import io.emeraldpay.dshackle.config.HealthConfig import io.emeraldpay.dshackle.config.MainConfig import io.emeraldpay.dshackle.config.MainConfigReader import io.emeraldpay.dshackle.config.MonitoringConfig @@ -120,4 +121,9 @@ open class Config( open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig { return mainConfig.monitoring } + + @Bean + open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig { + return mainConfig.health + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index 78650ea8..fac11f1e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -27,7 +27,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.grpc.Chain import java.text.SimpleDateFormat +import java.util.Locale import java.util.TimeZone import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService @@ -38,6 +40,35 @@ class Global { var metricsExtended = false + val chainNames = mapOf( + "ethereum" to Chain.ETHEREUM, + "ethereum-classic" to Chain.ETHEREUM_CLASSIC, + "eth" to Chain.ETHEREUM, + "polygon" to Chain.MATIC, + "matic" to Chain.MATIC, + "etc" to Chain.ETHEREUM_CLASSIC, + "morden" to Chain.TESTNET_MORDEN, + "kovan" to Chain.TESTNET_KOVAN, + "kovan-testnet" to Chain.TESTNET_KOVAN, + "goerli" to Chain.TESTNET_GOERLI, + "goerli-testnet" to Chain.TESTNET_GOERLI, + "rinkeby" to Chain.TESTNET_RINKEBY, + "rinkeby-testnet" to Chain.TESTNET_RINKEBY, + "ropsten" to Chain.TESTNET_ROPSTEN, + "ropsten-testnet" to Chain.TESTNET_ROPSTEN, + "bitcoin" to Chain.BITCOIN, + "bitcoin-testnet" to Chain.TESTNET_BITCOIN + ) + + fun chainById(id: String?): Chain { + if (id == null) { + return Chain.UNSPECIFIED + } + return chainNames[ + id.lowercase(Locale.getDefault()).replace("_", "-").trim() + ] ?: Chain.UNSPECIFIED + } + @JvmStatic val objectMapper: ObjectMapper = createObjectMapper() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt new file mode 100644 index 00000000..cf81781c --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain + +class HealthConfig { + + companion object { + fun default(): HealthConfig { + return HealthConfig() + } + } + + var port: Int = 8082 + var host: String = "127.0.0.1" + var path: String = "/health" + val chains = HashMap() + + fun isEnabled(): Boolean { + return chains.isNotEmpty() + } + + fun configs(): Collection { + return chains.values + } + + data class ChainConfig( + val blockchain: Chain, + val minAvailable: Int = 1 + ) +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt new file mode 100644 index 00000000..901c443d --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.Global +import io.emeraldpay.grpc.Chain +import org.slf4j.LoggerFactory +import org.yaml.snakeyaml.nodes.CollectionNode +import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream + +class HealthConfigReader : YamlConfigReader(), ConfigReader { + + companion object { + private val log = LoggerFactory.getLogger(HealthConfigReader::class.java) + } + + fun read(input: InputStream): HealthConfig { + val configNode = readNode(input) + return read(configNode) + } + + override fun read(input: MappingNode?): HealthConfig { + return readInternal(getMapping(input, "health")) + } + + fun readInternal(input: MappingNode?): HealthConfig { + if (input == null) { + return HealthConfig.default() + } + val config = HealthConfig() + getValueAsString(input, "host")?.let { + config.host = it + } + getValueAsInt(input, "port")?.let { + config.port = it + } + getValueAsString(input, "path")?.let { + config.path = it + } + readBlockchains(config, getList(input, "blockchains")) + return config + } + + fun readBlockchains(healthConfig: HealthConfig, input: CollectionNode?) { + if (input == null) { + return + } + input.value.forEach { conf -> + val chain = getValueAsString(conf, "chain") + ?.let { Global.chainById(it) } + if (chain == null) { + log.warn("Blockchain is not specified for a Health Check") + return@forEach + } + if (chain == Chain.UNSPECIFIED) { + log.warn("Using UNSPECIFIED blockchain for Health Check. Always fails") + } + if (healthConfig.chains.containsKey(chain)) { + log.warn("Duplicate Health Check config for $chain. Replace previous with new") + } + val minAvailable = getValueAsInt(conf, "min-available") ?: 1 + healthConfig.chains[chain] = HealthConfig.ChainConfig(chain, minAvailable.coerceAtLeast(0)) + } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index 286bff26..4fa6cf86 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -25,4 +25,5 @@ class MainConfig { var tokens: TokensConfig? = null var monitoring: MonitoringConfig = MonitoringConfig.default() var accessLogConfig: AccessLogConfig = AccessLogConfig.default() + var health: HealthConfig = HealthConfig.default() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index cba114f8..7ed84bd6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -35,6 +35,7 @@ class MainConfigReader( private val tokensConfigReader = TokensConfigReader() private val monitoringConfigReader = MonitoringConfigReader() private val accessLogReader = AccessLogReader() + private val healthConfigReader = HealthConfigReader() fun read(input: InputStream): MainConfig? { val configNode = readNode(input) @@ -71,6 +72,9 @@ class MainConfigReader( accessLogReader.read(input).let { config.accessLogConfig = it } + healthConfigReader.read(input).let { + config.health = it + } return config } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt index 4fcc9cae..e75f80c4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import io.emeraldpay.grpc.Chain import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory @@ -69,10 +70,10 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader { } currentRoutes.add(id) val blockchain = getValueAsString(route, "blockchain") - if (StringUtils.isEmpty(blockchain) || getBlockchain(blockchain!!) == Chain.UNSPECIFIED) { + if (StringUtils.isEmpty(blockchain) || Global.chainById(blockchain!!) == Chain.UNSPECIFIED) { throw InvalidConfigYamlException(filename, route.startMark, "Invalid blockchain or not specified") } - ProxyConfig.Route(id, getBlockchain(blockchain)) + ProxyConfig.Route(id, Global.chainById(blockchain)) } } if (config.routes.isEmpty()) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt index 9d4f6da3..85205277 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream @@ -34,7 +35,7 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader { val token = TokensConfig.Token() token.id = getValueAsString(node, "id") token.blockchain = getValueAsString(node, "blockchain")?.let { - getBlockchain(it) + Global.chainById(it) } token.address = getValueAsString(node, "address") token.name = getValueAsString(node, "name") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt index 414e7322..c358f999 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt @@ -16,7 +16,6 @@ */ package io.emeraldpay.dshackle.config -import io.emeraldpay.grpc.Chain import org.yaml.snakeyaml.Yaml import org.yaml.snakeyaml.nodes.CollectionNode import org.yaml.snakeyaml.nodes.MappingNode @@ -142,14 +141,4 @@ abstract class YamlConfigReader { base * multiplier } } - - // ---- - - fun getBlockchain(id: String): Chain { - return Chain.values().find { chain -> - chain.name == id.uppercase(Locale.getDefault()) || - chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) || - chain.id.toString() == id - } ?: Chain.UNSPECIFIED - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt new file mode 100644 index 00000000..01432fc4 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -0,0 +1,142 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.monitoring + +import com.sun.net.httpserver.HttpServer +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.HttpStatus +import org.springframework.stereotype.Service +import java.io.IOException +import java.net.InetSocketAddress +import javax.annotation.PostConstruct + +@Service +class HealthCheckSetup( + @Autowired private val healthConfig: HealthConfig, + @Autowired private val multistreamHolder: MultistreamHolder +) { + + companion object { + private val log = LoggerFactory.getLogger(HealthCheckSetup::class.java) + } + + @PostConstruct + fun start() { + if (!healthConfig.isEnabled()) { + return + } + // use standard JVM server with a single thread blocking processing + // health check is a rare operation, no reason to set up anything complex + try { + log.info("Run Health Server on ${healthConfig.host}:${healthConfig.port}${healthConfig.path}") + val server = HttpServer.create( + InetSocketAddress( + healthConfig.host, + healthConfig.port + ), + 0 + ) + server.createContext(healthConfig.path) { httpExchange -> + val response = if (httpExchange.requestURI.query == "detailed") { + getDetailedHealth() + } else { + getHealth() + } + val ok = response.ok + val data = response.details.joinToString("\n") + val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE + httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong()) + httpExchange.responseBody.use { os -> + os.write(data.toByteArray()) + } + } + Thread(server::start).start() + } catch (e: IOException) { + log.error("Failed to start Health Server", e) + } + } + + fun getHealth(): Detailed { + val errors = healthConfig.configs().mapNotNull { + val up = multistreamHolder.getUpstream(it.blockchain) + if (up == null || !up.isAvailable()) { + return@mapNotNull "${it.blockchain} UNAVAILABLE" + } + val avail = up.getAll().count { it.getStatus() == UpstreamAvailability.OK } + if (avail < it.minAvailable) { + return@mapNotNull "${it.blockchain} LACKS MIN AVAILABILITY" + } + null + } + return if (errors.isEmpty()) { + Detailed(true, listOf("OK")) + } else { + Detailed(false, errors) + } + } + + fun getDetailedHealth(): Detailed { + val chains = multistreamHolder.getAvailable() + val allEnabled = healthConfig.configs().all { chains.contains(it.blockchain) } + var anyUnavailable = false + val details = chains.flatMap { chain -> + var chainUnavailable = false + val up = multistreamHolder.getUpstream(chain) + val required = healthConfig.chains[chain] + if (up == null || !up.isAvailable()) { + if (required != null) { + anyUnavailable = true + } + listOf("${chain.name} UNAVAILABLE") + } else { + val ups = up.getAll() + val checks = if (required != null) { + val avail = ups.count { it.getStatus() == UpstreamAvailability.OK } + if (avail < required.minAvailable) { + chainUnavailable = true + listOf(" LACKS MIN AVAILABILITY") + } else emptyList() + } else emptyList() + val upDetails = ups.map { + " ${it.getId()} ${it.getStatus()} with lag=${it.getLag()}" + } + val status = if (chainUnavailable) "UNAVAILABLE" else "AVAILABLE" + anyUnavailable = anyUnavailable || chainUnavailable + listOf("${chain.name} $status") + upDetails + checks + } + } + val detailsUnavailable = if (!allEnabled) { + healthConfig.configs() + .filter { !chains.contains(it.blockchain) } + .map { + "${it.blockchain.name} UNAVAILABLE" + } + } else emptyList() + return Detailed( + allEnabled && !anyUnavailable, + detailsUnavailable + details + ) + } + + data class Detailed( + val ok: Boolean, + val details: List + ) +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index fdf1dadf..d7951d0e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.startup import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.cache.CachesFactory import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader @@ -57,26 +58,6 @@ open class ConfiguredUpstreams( private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) private var seq = AtomicInteger(0) - private val chainNames = mapOf( - "ethereum" to Chain.ETHEREUM, - "ethereum-classic" to Chain.ETHEREUM_CLASSIC, - "eth" to Chain.ETHEREUM, - "polygon" to Chain.MATIC, - "matic" to Chain.MATIC, - "etc" to Chain.ETHEREUM_CLASSIC, - "morden" to Chain.TESTNET_MORDEN, - "kovan" to Chain.TESTNET_KOVAN, - "kovan-testnet" to Chain.TESTNET_KOVAN, - "goerli" to Chain.TESTNET_GOERLI, - "goerli-testnet" to Chain.TESTNET_GOERLI, - "rinkeby" to Chain.TESTNET_RINKEBY, - "rinkeby-testnet" to Chain.TESTNET_RINKEBY, - "ropsten" to Chain.TESTNET_ROPSTEN, - "ropsten-testnet" to Chain.TESTNET_ROPSTEN, - "bitcoin" to Chain.BITCOIN, - "bitcoin-testnet" to Chain.TESTNET_BITCOIN - ) - @PostConstruct fun start() { log.debug("Starting upstreams") @@ -87,8 +68,8 @@ open class ConfiguredUpstreams( val options = up.options ?: UpstreamsConfig.Options() buildGrpcUpstream(up.cast(UpstreamsConfig.GrpcConnection::class.java), options) } else { - val chain = chainNames[up.chain] - if (chain == null) { + val chain = Global.chainById(up.chain) + if (chain == Chain.UNSPECIFIED) { log.error("Chain is unknown: ${up.chain}") return@forEach } @@ -114,7 +95,7 @@ open class ConfiguredUpstreams( val defaultOptions = HashMap() config.defaultOptions.forEach { defaultsConfig -> defaultsConfig.chains?.forEach { chainName -> - chainNames[chainName]?.let { chain -> + Global.chainById(chainName).let { chain -> defaultsConfig.options?.let { options -> if (!defaultOptions.containsKey(chain)) { defaultOptions[chain] = options @@ -274,7 +255,7 @@ open class ConfiguredUpstreams( // "unknown" is not supposed to happen Tag.of("upstream", config.id ?: "unknown"), // UNSPECIFIED shouldn't happen too - Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode) + Tag.of("chain", (Global.chainById(config.chain).chainCode)) ) val metrics = RpcMetrics( Timer.builder("upstream.rpc.conn") diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy new file mode 100644 index 00000000..6add6d56 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain +import spock.lang.Specification + +class HealthConfigReaderSpec extends Specification { + + HealthConfigReader reader = new HealthConfigReader() + + def "Read empty"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-empty.yaml") + when: + def act = reader.read(config) + + then: + !act.enabled + act.port == 8082 + act.host == "127.0.0.1" + act.path == "/health" + act.configs().size() == 0 + } + + def "Read single"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-1.yaml") + when: + def act = reader.read(config) + + then: + act.enabled + act.port == 8082 + act.host == "127.0.0.1" + act.path == "/health" + with(act.configs()) { + size() == 1 + with(it[0]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 2 + } + } + } + + def "Read multiple"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-2.yaml") + when: + def act = reader.read(config) + + then: + act.enabled + act.port == 10003 + act.host == "0.0.0.0" + act.path == "/healtz" + with(act.configs().toSorted { it.blockchain.id }) { + size() == 2 + with(it[0]) { + it.blockchain == Chain.BITCOIN + it.minAvailable == 1 + } + with(it[1]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 2 + } + } + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy index b790e348..f8fbdd7a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy @@ -15,7 +15,7 @@ */ package io.emeraldpay.dshackle.config -import io.emeraldpay.dshackle.FileResolver + import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.grpc.Chain import spock.lang.Specification @@ -73,6 +73,16 @@ class MainConfigReaderSpec extends Specification { blockchain == Chain.TESTNET_RINKEBY } } + with(act.health) { + it.enabled + with(it.configs()) { + it.size() == 1 + with(it[0]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 1 + } + } + } act.upstreams != null with(act.upstreams) { defaultOptions != null diff --git a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy new file mode 100644 index 00000000..c4772852 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.monitoring + +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.grpc.Chain +import spock.lang.Specification + +class HealthCheckSetupSpec extends Specification { + + def "OK when meets availability - 1"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 1 + ) + } + def up1 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act.ok + act.details == ["OK"] + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1] + 1 * up1.status >> UpstreamAvailability.OK + } + + def "OK when meets availability - 1 - bitcoin"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.BITCOIN] = new HealthConfig.ChainConfig( + Chain.BITCOIN, 1 + ) + } + def up1 = Mock(Upstream) + def bitcoinUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act.ok + act.details == ["OK"] + 1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams + 1 * bitcoinUpstreams.available >> true + 1 * bitcoinUpstreams.getAll() >> [up1] + 1 * up1.status >> UpstreamAvailability.OK + } + + def "OK when meets availability - 2/3"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act.ok + act.details == ["OK"] + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + 1 * up1.status >> UpstreamAvailability.OK + 1 * up2.status >> UpstreamAvailability.SYNCING + 1 * up3.status >> UpstreamAvailability.OK + } + + def "OK when doesn't meet availability - 2/3"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + !act.ok + act.details != ["OK"] + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + 1 * up1.status >> UpstreamAvailability.OK + 1 * up2.status >> UpstreamAvailability.SYNCING + 1 * up3.status >> UpstreamAvailability.LAGGING + } + + def "OK when meets availability - 2/3 - detailed"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.detailedHealth + + then: + act.ok + act.details.size() > 1 + 1 * multistream.getAvailable() >> [Chain.ETHEREUM] + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + _ * up1.status >> UpstreamAvailability.OK + _ * up2.status >> UpstreamAvailability.SYNCING + _ * up3.status >> UpstreamAvailability.OK + } +} diff --git a/src/test/resources/dshackle-full.yaml b/src/test/resources/dshackle-full.yaml index e16ccde9..a3a9ad8a 100644 --- a/src/test/resources/dshackle-full.yaml +++ b/src/test/resources/dshackle-full.yaml @@ -16,6 +16,12 @@ cache: enabled: true host: redis-master +health: + port: 8083 + blockchains: + - chain: ethereum + min-available: 1 + proxy: port: 8082 tls: diff --git a/src/test/resources/dshackle-health-1.yaml b/src/test/resources/dshackle-health-1.yaml new file mode 100644 index 00000000..ca32dbb1 --- /dev/null +++ b/src/test/resources/dshackle-health-1.yaml @@ -0,0 +1,6 @@ +version: v1 + +health: + blockchains: + - chain: ethereum + min-available: 2 \ No newline at end of file diff --git a/src/test/resources/dshackle-health-2.yaml b/src/test/resources/dshackle-health-2.yaml new file mode 100644 index 00000000..2e78abdd --- /dev/null +++ b/src/test/resources/dshackle-health-2.yaml @@ -0,0 +1,11 @@ +version: v1 + +health: + port: 10003 + host: 0.0.0.0 + path: /healtz + blockchains: + - chain: ethereum + min-available: 2 + - chain: bitcoin + min-available: 1 \ No newline at end of file diff --git a/src/test/resources/dshackle-health-empty.yaml b/src/test/resources/dshackle-health-empty.yaml new file mode 100644 index 00000000..f1a00ccc --- /dev/null +++ b/src/test/resources/dshackle-health-empty.yaml @@ -0,0 +1,3 @@ +version: v1 + +health: \ No newline at end of file