From e205985da31f840008a62e11503954753265fb06 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Wed, 20 Oct 2021 23:10:41 -0400 Subject: [PATCH 1/5] solution: health check endpoint fix: #111 --- .../kotlin/io/emeraldpay/dshackle/Config.kt | 12 +- .../dshackle/config/HealthConfig.kt | 46 +++++++ .../dshackle/config/HealthConfigReader.kt | 79 +++++++++++ .../emeraldpay/dshackle/config/MainConfig.kt | 1 + .../dshackle/config/MainConfigReader.kt | 4 + .../dshackle/monitoring/HealthCheckSetup.kt | 89 ++++++++++++ .../config/HealthConfigReaderSpec.groovy | 82 +++++++++++ .../config/MainConfigReaderSpec.groovy | 12 +- .../monitoring/HealthCheckSetupSpec.groovy | 127 ++++++++++++++++++ src/test/resources/dshackle-full.yaml | 6 + src/test/resources/dshackle-health-1.yaml | 6 + src/test/resources/dshackle-health-2.yaml | 11 ++ src/test/resources/dshackle-health-empty.yaml | 3 + 13 files changed, 471 insertions(+), 7 deletions(-) create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt create mode 100644 src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt create mode 100644 src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy create mode 100644 src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy create mode 100644 src/test/resources/dshackle-health-1.yaml create mode 100644 src/test/resources/dshackle-health-2.yaml create mode 100644 src/test/resources/dshackle-health-empty.yaml diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index a619f181..56a5b072 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -16,12 +16,7 @@ */ package io.emeraldpay.dshackle -import io.emeraldpay.dshackle.config.CacheConfig -import io.emeraldpay.dshackle.config.MainConfig -import io.emeraldpay.dshackle.config.MainConfigReader -import io.emeraldpay.dshackle.config.MonitoringConfig -import io.emeraldpay.dshackle.config.TokensConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.config.* import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier @@ -120,4 +115,9 @@ open class Config( open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig { return mainConfig.monitoring } + + @Bean + open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig { + return mainConfig.health + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt new file mode 100644 index 00000000..ebd66c88 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain + +class HealthConfig { + + companion object { + fun default(): HealthConfig { + return HealthConfig() + } + } + + var port: Int = 8082 + var host: String = "127.0.0.1" + var path: String = "/health" + val chains = HashMap() + + fun isEnabled(): Boolean { + return chains.isNotEmpty() + } + + fun configs(): Collection { + return chains.values + } + + data class ChainConfig( + val blockchain: Chain, + val minAvailable: Int = 1 + ) + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt new file mode 100644 index 00000000..1058f6f4 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain +import org.slf4j.LoggerFactory +import org.yaml.snakeyaml.nodes.CollectionNode +import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream + +class HealthConfigReader : YamlConfigReader(), ConfigReader { + + companion object { + private val log = LoggerFactory.getLogger(HealthConfigReader::class.java) + } + + fun read(input: InputStream): HealthConfig { + val configNode = readNode(input) + return read(configNode) + } + + override fun read(input: MappingNode?): HealthConfig { + return readInternal(getMapping(input, "health")) + } + + fun readInternal(input: MappingNode?): HealthConfig { + if (input == null) { + return HealthConfig.default() + } + val config = HealthConfig() + getValueAsString(input, "host")?.let { + config.host = it + } + getValueAsInt(input, "port")?.let { + config.port = it + } + getValueAsString(input, "path")?.let { + config.path = it + } + readBlockchains(config, getList(input, "blockchains")) + return config + } + + fun readBlockchains(healthConfig: HealthConfig, input: CollectionNode?) { + if (input == null) { + return + } + input.value.forEach { conf -> + val chain = getValueAsString(conf, "chain") + ?.let { getBlockchain(it) } + if (chain == null) { + log.warn("Blockchain is not specified for a Health Check") + return@forEach + } + if (chain == Chain.UNSPECIFIED) { + log.warn("Using UNSPECIFIED blockchain for Health Check. Always fails") + } + if (healthConfig.chains.containsKey(chain)) { + log.warn("Duplicate Health Check config for $chain. Replace previous with new") + } + val minAvailable = getValueAsInt(conf, "min-available") ?: 1 + healthConfig.chains[chain] = HealthConfig.ChainConfig(chain, minAvailable.coerceAtLeast(0)) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index 286bff26..4fa6cf86 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -25,4 +25,5 @@ class MainConfig { var tokens: TokensConfig? = null var monitoring: MonitoringConfig = MonitoringConfig.default() var accessLogConfig: AccessLogConfig = AccessLogConfig.default() + var health: HealthConfig = HealthConfig.default() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index cba114f8..7ed84bd6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -35,6 +35,7 @@ class MainConfigReader( private val tokensConfigReader = TokensConfigReader() private val monitoringConfigReader = MonitoringConfigReader() private val accessLogReader = AccessLogReader() + private val healthConfigReader = HealthConfigReader() fun read(input: InputStream): MainConfig? { val configNode = readNode(input) @@ -71,6 +72,9 @@ class MainConfigReader( accessLogReader.read(input).let { config.accessLogConfig = it } + healthConfigReader.read(input).let { + config.health = it + } return config } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt new file mode 100644 index 00000000..76c62d3b --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.monitoring + +import com.sun.net.httpserver.HttpServer +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.HttpStatus +import org.springframework.stereotype.Service +import java.io.IOException +import java.net.InetSocketAddress +import javax.annotation.PostConstruct + +@Service +class HealthCheckSetup( + @Autowired private val healthConfig: HealthConfig, + @Autowired private val multistreamHolder: MultistreamHolder +) { + + companion object { + private val log = LoggerFactory.getLogger(HealthCheckSetup::class.java) + } + + @PostConstruct + fun start() { + if (!healthConfig.isEnabled()) { + return + } + // use standard JVM server with a single thread blocking processing + // health check is a rare operation, no reason to set up anything complex + try { + log.info("Run Health Server on ${healthConfig.host}:${healthConfig.port}${healthConfig.path}") + val server = HttpServer.create( + InetSocketAddress( + healthConfig.host, + healthConfig.port + ), + 0 + ) + server.createContext(healthConfig.path) { httpExchange -> + val response = getHealth() + val ok = response == "OK" + val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE + httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong()) + httpExchange.responseBody.use { os -> + os.write(response.toByteArray()) + } + } + Thread(server::start).start() + } catch (e: IOException) { + log.error("Failed to start Health Server", e) + } + } + + fun getHealth(): String { + val errors = healthConfig.configs().mapNotNull { + val up = multistreamHolder.getUpstream(it.blockchain) + if (up == null || !up.isAvailable()) { + return@mapNotNull "${it.blockchain} UNAVAILABLE" + } + val avail = up.getAll().count { it.getStatus() == UpstreamAvailability.OK } + if (avail < it.minAvailable) { + return@mapNotNull "${it.blockchain} LACKS MIN AVAILABILITY" + } + null + } + return if (errors.isEmpty()) { + "OK" + } else { + errors.joinToString("\n") + } + } +} \ No newline at end of file diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy new file mode 100644 index 00000000..6add6d56 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain +import spock.lang.Specification + +class HealthConfigReaderSpec extends Specification { + + HealthConfigReader reader = new HealthConfigReader() + + def "Read empty"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-empty.yaml") + when: + def act = reader.read(config) + + then: + !act.enabled + act.port == 8082 + act.host == "127.0.0.1" + act.path == "/health" + act.configs().size() == 0 + } + + def "Read single"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-1.yaml") + when: + def act = reader.read(config) + + then: + act.enabled + act.port == 8082 + act.host == "127.0.0.1" + act.path == "/health" + with(act.configs()) { + size() == 1 + with(it[0]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 2 + } + } + } + + def "Read multiple"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-2.yaml") + when: + def act = reader.read(config) + + then: + act.enabled + act.port == 10003 + act.host == "0.0.0.0" + act.path == "/healtz" + with(act.configs().toSorted { it.blockchain.id }) { + size() == 2 + with(it[0]) { + it.blockchain == Chain.BITCOIN + it.minAvailable == 1 + } + with(it[1]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 2 + } + } + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy index b790e348..f8fbdd7a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy @@ -15,7 +15,7 @@ */ package io.emeraldpay.dshackle.config -import io.emeraldpay.dshackle.FileResolver + import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.grpc.Chain import spock.lang.Specification @@ -73,6 +73,16 @@ class MainConfigReaderSpec extends Specification { blockchain == Chain.TESTNET_RINKEBY } } + with(act.health) { + it.enabled + with(it.configs()) { + it.size() == 1 + with(it[0]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 1 + } + } + } act.upstreams != null with(act.upstreams) { defaultOptions != null diff --git a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy new file mode 100644 index 00000000..b9f6d4da --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy @@ -0,0 +1,127 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.monitoring + +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.grpc.Chain +import spock.lang.Specification + +class HealthCheckSetupSpec extends Specification { + + def "OK when meets availability - 1"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 1 + ) + } + def up1 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act == "OK" + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1] + 1 * up1.status >> UpstreamAvailability.OK + } + + def "OK when meets availability - 1 - bitcoin"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.BITCOIN] = new HealthConfig.ChainConfig( + Chain.BITCOIN, 1 + ) + } + def up1 = Mock(Upstream) + def bitcoinUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act == "OK" + 1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams + 1 * bitcoinUpstreams.available >> true + 1 * bitcoinUpstreams.getAll() >> [up1] + 1 * up1.status >> UpstreamAvailability.OK + } + + def "OK when meets availability - 2/3"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act == "OK" + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + 1 * up1.status >> UpstreamAvailability.OK + 1 * up2.status >> UpstreamAvailability.SYNCING + 1 * up3.status >> UpstreamAvailability.OK + } + + def "OK when doesn't meet availability - 2/3"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act != "OK" + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + 1 * up1.status >> UpstreamAvailability.OK + 1 * up2.status >> UpstreamAvailability.SYNCING + 1 * up3.status >> UpstreamAvailability.LAGGING + } +} diff --git a/src/test/resources/dshackle-full.yaml b/src/test/resources/dshackle-full.yaml index e16ccde9..a3a9ad8a 100644 --- a/src/test/resources/dshackle-full.yaml +++ b/src/test/resources/dshackle-full.yaml @@ -16,6 +16,12 @@ cache: enabled: true host: redis-master +health: + port: 8083 + blockchains: + - chain: ethereum + min-available: 1 + proxy: port: 8082 tls: diff --git a/src/test/resources/dshackle-health-1.yaml b/src/test/resources/dshackle-health-1.yaml new file mode 100644 index 00000000..ca32dbb1 --- /dev/null +++ b/src/test/resources/dshackle-health-1.yaml @@ -0,0 +1,6 @@ +version: v1 + +health: + blockchains: + - chain: ethereum + min-available: 2 \ No newline at end of file diff --git a/src/test/resources/dshackle-health-2.yaml b/src/test/resources/dshackle-health-2.yaml new file mode 100644 index 00000000..2e78abdd --- /dev/null +++ b/src/test/resources/dshackle-health-2.yaml @@ -0,0 +1,11 @@ +version: v1 + +health: + port: 10003 + host: 0.0.0.0 + path: /healtz + blockchains: + - chain: ethereum + min-available: 2 + - chain: bitcoin + min-available: 1 \ No newline at end of file diff --git a/src/test/resources/dshackle-health-empty.yaml b/src/test/resources/dshackle-health-empty.yaml new file mode 100644 index 00000000..f1a00ccc --- /dev/null +++ b/src/test/resources/dshackle-health-empty.yaml @@ -0,0 +1,3 @@ +version: v1 + +health: \ No newline at end of file From 5f4a8a4ac4f4fe0e8a94155795488503443a5fad Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Fri, 22 Oct 2021 22:12:48 -0400 Subject: [PATCH 2/5] solution: health check with detailed description of health --- .../kotlin/io/emeraldpay/dshackle/Global.kt | 32 ++++++++- .../dshackle/config/HealthConfigReader.kt | 3 +- .../dshackle/config/ProxyConfigReader.kt | 5 +- .../dshackle/config/TokensConfigReader.kt | 3 +- .../dshackle/config/YamlConfigReader.kt | 9 --- .../dshackle/monitoring/HealthCheckSetup.kt | 67 +++++++++++++++++-- .../dshackle/startup/ConfiguredUpstreams.kt | 29 ++------ .../monitoring/HealthCheckSetupSpec.groovy | 41 ++++++++++-- 8 files changed, 140 insertions(+), 49 deletions(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index 78650ea8..e45f8920 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -27,8 +27,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.grpc.Chain import java.text.SimpleDateFormat -import java.util.TimeZone +import java.util.* import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService @@ -38,6 +39,35 @@ class Global { var metricsExtended = false + val chainNames = mapOf( + "ethereum" to Chain.ETHEREUM, + "ethereum-classic" to Chain.ETHEREUM_CLASSIC, + "eth" to Chain.ETHEREUM, + "polygon" to Chain.MATIC, + "matic" to Chain.MATIC, + "etc" to Chain.ETHEREUM_CLASSIC, + "morden" to Chain.TESTNET_MORDEN, + "kovan" to Chain.TESTNET_KOVAN, + "kovan-testnet" to Chain.TESTNET_KOVAN, + "goerli" to Chain.TESTNET_GOERLI, + "goerli-testnet" to Chain.TESTNET_GOERLI, + "rinkeby" to Chain.TESTNET_RINKEBY, + "rinkeby-testnet" to Chain.TESTNET_RINKEBY, + "ropsten" to Chain.TESTNET_ROPSTEN, + "ropsten-testnet" to Chain.TESTNET_ROPSTEN, + "bitcoin" to Chain.BITCOIN, + "bitcoin-testnet" to Chain.TESTNET_BITCOIN + ) + + fun chainById(id: String?): Chain { + if (id == null) { + return Chain.UNSPECIFIED + } + return chainNames[ + id.lowercase(Locale.getDefault()).replace("_", "-").trim() + ] ?: Chain.UNSPECIFIED + } + @JvmStatic val objectMapper: ObjectMapper = createObjectMapper() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt index 1058f6f4..ebf35517 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.CollectionNode @@ -60,7 +61,7 @@ class HealthConfigReader : YamlConfigReader(), ConfigReader { } input.value.forEach { conf -> val chain = getValueAsString(conf, "chain") - ?.let { getBlockchain(it) } + ?.let { Global.chainById(it) } if (chain == null) { log.warn("Blockchain is not specified for a Health Check") return@forEach diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt index 4fcc9cae..e75f80c4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import io.emeraldpay.grpc.Chain import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory @@ -69,10 +70,10 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader { } currentRoutes.add(id) val blockchain = getValueAsString(route, "blockchain") - if (StringUtils.isEmpty(blockchain) || getBlockchain(blockchain!!) == Chain.UNSPECIFIED) { + if (StringUtils.isEmpty(blockchain) || Global.chainById(blockchain!!) == Chain.UNSPECIFIED) { throw InvalidConfigYamlException(filename, route.startMark, "Invalid blockchain or not specified") } - ProxyConfig.Route(id, getBlockchain(blockchain)) + ProxyConfig.Route(id, Global.chainById(blockchain)) } } if (config.routes.isEmpty()) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt index 9d4f6da3..85205277 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt @@ -15,6 +15,7 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.dshackle.Global import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream @@ -34,7 +35,7 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader { val token = TokensConfig.Token() token.id = getValueAsString(node, "id") token.blockchain = getValueAsString(node, "blockchain")?.let { - getBlockchain(it) + Global.chainById(it) } token.address = getValueAsString(node, "address") token.name = getValueAsString(node, "name") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt index 414e7322..517849b8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt @@ -143,13 +143,4 @@ abstract class YamlConfigReader { } } - // ---- - - fun getBlockchain(id: String): Chain { - return Chain.values().find { chain -> - chain.name == id.uppercase(Locale.getDefault()) || - chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) || - chain.id.toString() == id - } ?: Chain.UNSPECIFIED - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt index 76c62d3b..c333ae01 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -54,12 +54,17 @@ class HealthCheckSetup( 0 ) server.createContext(healthConfig.path) { httpExchange -> - val response = getHealth() - val ok = response == "OK" + val response = if (httpExchange.requestURI.query == "detailed") { + getDetailedHealth() + } else { + getHealth() + } + val ok = response.ok + val data = response.details.joinToString("\n") val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE - httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong()) + httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong()) httpExchange.responseBody.use { os -> - os.write(response.toByteArray()) + os.write(data.toByteArray()) } } Thread(server::start).start() @@ -68,7 +73,7 @@ class HealthCheckSetup( } } - fun getHealth(): String { + fun getHealth(): Detailed { val errors = healthConfig.configs().mapNotNull { val up = multistreamHolder.getUpstream(it.blockchain) if (up == null || !up.isAvailable()) { @@ -81,9 +86,57 @@ class HealthCheckSetup( null } return if (errors.isEmpty()) { - "OK" + Detailed(true, listOf("OK")) } else { - errors.joinToString("\n") + Detailed(false, errors) } } + + fun getDetailedHealth(): Detailed { + val chains = multistreamHolder.getAvailable() + val allEnabled = healthConfig.configs().all { chains.contains(it.blockchain) } + var anyUnavailable = false + val details = chains.flatMap { chain -> + var chainUnavailable = false + val up = multistreamHolder.getUpstream(chain) + val required = healthConfig.chains[chain] + if (up == null || !up.isAvailable()) { + if (required != null) { + anyUnavailable = true + } + listOf("${chain.name} UNAVAILABLE") + } else { + val ups = up.getAll() + val checks = if (required != null) { + val avail = ups.count { it.getStatus() == UpstreamAvailability.OK } + if (avail < required.minAvailable) { + chainUnavailable = true + listOf(" LACKS MIN AVAILABILITY") + } else emptyList() + } else emptyList() + val upDetails = ups.map { + " ${it.getId()} ${it.getStatus()} with lag=${it.getLag()}" + } + val status = if (chainUnavailable) "UNAVAILABLE" else "AVAILABLE" + anyUnavailable = anyUnavailable || chainUnavailable + listOf("${chain.name} $status") + upDetails + checks + } + } + val detailsUnavailable = if (!allEnabled) { + healthConfig.configs() + .filter { !chains.contains(it.blockchain) } + .map { + "${it.blockchain.name} UNAVAILABLE" + } + } else emptyList() + return Detailed( + allEnabled && !anyUnavailable, + detailsUnavailable + details + ) + } + + data class Detailed( + val ok: Boolean, + val details: List + ) } \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index fdf1dadf..d7951d0e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.startup import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.cache.CachesFactory import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader @@ -57,26 +58,6 @@ open class ConfiguredUpstreams( private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) private var seq = AtomicInteger(0) - private val chainNames = mapOf( - "ethereum" to Chain.ETHEREUM, - "ethereum-classic" to Chain.ETHEREUM_CLASSIC, - "eth" to Chain.ETHEREUM, - "polygon" to Chain.MATIC, - "matic" to Chain.MATIC, - "etc" to Chain.ETHEREUM_CLASSIC, - "morden" to Chain.TESTNET_MORDEN, - "kovan" to Chain.TESTNET_KOVAN, - "kovan-testnet" to Chain.TESTNET_KOVAN, - "goerli" to Chain.TESTNET_GOERLI, - "goerli-testnet" to Chain.TESTNET_GOERLI, - "rinkeby" to Chain.TESTNET_RINKEBY, - "rinkeby-testnet" to Chain.TESTNET_RINKEBY, - "ropsten" to Chain.TESTNET_ROPSTEN, - "ropsten-testnet" to Chain.TESTNET_ROPSTEN, - "bitcoin" to Chain.BITCOIN, - "bitcoin-testnet" to Chain.TESTNET_BITCOIN - ) - @PostConstruct fun start() { log.debug("Starting upstreams") @@ -87,8 +68,8 @@ open class ConfiguredUpstreams( val options = up.options ?: UpstreamsConfig.Options() buildGrpcUpstream(up.cast(UpstreamsConfig.GrpcConnection::class.java), options) } else { - val chain = chainNames[up.chain] - if (chain == null) { + val chain = Global.chainById(up.chain) + if (chain == Chain.UNSPECIFIED) { log.error("Chain is unknown: ${up.chain}") return@forEach } @@ -114,7 +95,7 @@ open class ConfiguredUpstreams( val defaultOptions = HashMap() config.defaultOptions.forEach { defaultsConfig -> defaultsConfig.chains?.forEach { chainName -> - chainNames[chainName]?.let { chain -> + Global.chainById(chainName).let { chain -> defaultsConfig.options?.let { options -> if (!defaultOptions.containsKey(chain)) { defaultOptions[chain] = options @@ -274,7 +255,7 @@ open class ConfiguredUpstreams( // "unknown" is not supposed to happen Tag.of("upstream", config.id ?: "unknown"), // UNSPECIFIED shouldn't happen too - Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode) + Tag.of("chain", (Global.chainById(config.chain).chainCode)) ) val metrics = RpcMetrics( Timer.builder("upstream.rpc.conn") diff --git a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy index b9f6d4da..c4772852 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy @@ -41,7 +41,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act == "OK" + act.ok + act.details == ["OK"] 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams 1 * ethereumUpstreams.available >> true 1 * ethereumUpstreams.getAll() >> [up1] @@ -64,7 +65,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act == "OK" + act.ok + act.details == ["OK"] 1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams 1 * bitcoinUpstreams.available >> true 1 * bitcoinUpstreams.getAll() >> [up1] @@ -89,7 +91,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act == "OK" + act.ok + act.details == ["OK"] 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams 1 * ethereumUpstreams.available >> true 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] @@ -116,7 +119,8 @@ class HealthCheckSetupSpec extends Specification { def act = check.health then: - act != "OK" + !act.ok + act.details != ["OK"] 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams 1 * ethereumUpstreams.available >> true 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] @@ -124,4 +128,33 @@ class HealthCheckSetupSpec extends Specification { 1 * up2.status >> UpstreamAvailability.SYNCING 1 * up3.status >> UpstreamAvailability.LAGGING } + + def "OK when meets availability - 2/3 - detailed"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.detailedHealth + + then: + act.ok + act.details.size() > 1 + 1 * multistream.getAvailable() >> [Chain.ETHEREUM] + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + _ * up1.status >> UpstreamAvailability.OK + _ * up2.status >> UpstreamAvailability.SYNCING + _ * up3.status >> UpstreamAvailability.OK + } } From 710f90ddaa85e12900debef93f67dcc39f328508 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Fri, 22 Oct 2021 22:19:37 -0400 Subject: [PATCH 3/5] problem: formatting --- src/main/kotlin/io/emeraldpay/dshackle/Config.kt | 8 +++++++- .../kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt | 3 +-- .../io/emeraldpay/dshackle/config/HealthConfigReader.kt | 3 +-- .../io/emeraldpay/dshackle/config/YamlConfigReader.kt | 2 -- .../io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index 56a5b072..b27dc9da 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -16,7 +16,13 @@ */ package io.emeraldpay.dshackle -import io.emeraldpay.dshackle.config.* +import io.emeraldpay.dshackle.config.CacheConfig +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.config.MainConfig +import io.emeraldpay.dshackle.config.MainConfigReader +import io.emeraldpay.dshackle.config.MonitoringConfig +import io.emeraldpay.dshackle.config.TokensConfig +import io.emeraldpay.dshackle.config.UpstreamsConfig import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt index ebd66c88..cf81781c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt @@ -42,5 +42,4 @@ class HealthConfig { val blockchain: Chain, val minAvailable: Int = 1 ) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt index ebf35517..901c443d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -76,5 +76,4 @@ class HealthConfigReader : YamlConfigReader(), ConfigReader { healthConfig.chains[chain] = HealthConfig.ChainConfig(chain, minAvailable.coerceAtLeast(0)) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt index 517849b8..c358f999 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt @@ -16,7 +16,6 @@ */ package io.emeraldpay.dshackle.config -import io.emeraldpay.grpc.Chain import org.yaml.snakeyaml.Yaml import org.yaml.snakeyaml.nodes.CollectionNode import org.yaml.snakeyaml.nodes.MappingNode @@ -142,5 +141,4 @@ abstract class YamlConfigReader { base * multiplier } } - } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt index c333ae01..01432fc4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -139,4 +139,4 @@ class HealthCheckSetup( val ok: Boolean, val details: List ) -} \ No newline at end of file +} From 7032a9559f73f9447d0031e835a48fe0939ecb05 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Fri, 22 Oct 2021 22:27:36 -0400 Subject: [PATCH 4/5] problem: lint --- src/main/kotlin/io/emeraldpay/dshackle/Global.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index e45f8920..fac11f1e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -29,7 +29,8 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.grpc.Chain import java.text.SimpleDateFormat -import java.util.* +import java.util.Locale +import java.util.TimeZone import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService From 5f159dd6d76365db18071847d6669c3233462f68 Mon Sep 17 00:00:00 2001 From: Igor Artamonov Date: Sat, 23 Oct 2021 21:05:04 -0400 Subject: [PATCH 5/5] solution: docs for health check --- docs/06-monitoring.adoc | 57 ++++++++++++++++++++++++++++++ docs/reference-configuration.adoc | 58 +++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/docs/06-monitoring.adoc b/docs/06-monitoring.adoc index df72607c..e19d3129 100644 --- a/docs/06-monitoring.adoc +++ b/docs/06-monitoring.adoc @@ -119,4 +119,61 @@ This dashboard contains: - JSON RPC upstream conn seconds 50,75,90,99 percentiles +== Health Checks +Dshackle provides a http endpoint to check status of the servers. +This check is compatible with https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#http-probes[Kubernetes Liveness and Readiness Probes]. + +By default, it's disabled, and you have to set up which blockchain are required to be available to consider Dshackle alive. + +.Example config: +[source,yaml] +---- +health: + port: 8082 # <1> + host: 127.0.0.1 # <2> + path: /health # <3> + blockchains: # <4> + - chain: ethereum # <5> + min-available: 2 # <6> + - chain: bitcoin + min-available: 1 +---- + +<1> (optional) port to bind the Health server. +Default: `8082` +<2> (optional) host to bind the Health server. +Default: `127.0.0.1` +<3> (optional) path on the server. +Default: `/health`. +I.e., `http://127.0.0.1:8082/health` with default config +<4> list of blockchain to check availability +<5> a Blockchain to check +<6> minimum available (i.e., fully synced) Upstreams for that blockchain + +With the config above the server is considered healthy if: + +- Dshackle has connected to at least two valid Ethereum upstreams +- **and** at least one valid Bitcoin upstream. + +When the server is healthy is responds with `OK` and 200 as HTTP Status Code. +When any of the checks failed, it responds with a short description and 503 as HTTP Status Code. + +Example of a response for an unhealthy server that doesn't have enough upstreams for a Ethereum Classic Blockchain. + +.GET http://127.0.0.1:8082/health +---- +ETHEREUM_CLASSIC UNAVAILABLE +---- + +Optionally, the server can be called with `?detailed` query, which provides a more detailed response: + +.GET http://127.0.0.1:8082/health?detailed +---- +ETHEREUM_CLASSIC UNAVAILABLE +BITCOIN AVAILABLE + local-btc-1 OK with lag=0 +ETHEREUM AVAILABLE + local-eth-1 OK with lag=0 + local-eth-2 OK with lag=0 +---- diff --git a/docs/reference-configuration.adoc b/docs/reference-configuration.adoc index b5b9aea3..16f06420 100644 --- a/docs/reference-configuration.adoc +++ b/docs/reference-configuration.adoc @@ -31,6 +31,14 @@ monitoring: port: 8081 path: /metrics +health: + port: 8082 + host: 127.0.0.1 + path: /health + blockchains: + - chain: ethereum + min-availability: 1 + cache: redis: enabled: true @@ -164,6 +172,10 @@ See <> section | Setup Prometheus monitoring. See <> section +| `health` +| +| Setup Health Check endpoint See <> section + | `proxy` | | Setup HTTP proxy that emulates all standard JSON RPC requests. @@ -285,6 +297,52 @@ _Reserved for future use_, in case of multiple different types of endpoints. |=== +[#health] +== Health Check endpoint + +[source,yaml] +---- +health: + port: 8082 + host: 127.0.0.1 + path: /health + blockchains: + - chain: ethereum + min-available: 2 + - chain: bitcoin + min-available: 1 +---- + +[cols="2a,2a,5"] +|=== +| Option | Default Value | Description + +| `port` +| `8082` +| HTTP port to bind the server + +| `host` +| `127.0.0.1` +| HTTP host to bind the server + +| `path` +| `/health` +| HTTP path to respond on requests + +| `blockchains` +| +| List of blockchains that must be available to consider the server _healthy_ + +| `[blockchain].chain` +| +| Blockchain id + +| `[blockchain].min-available` +| 1 +| How many _available_ upstreams for the blockchain is required to pass + +|=== + [#proxy] == Proxy config