diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index a619f181..56a5b072 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -16,12 +16,7 @@ */ package io.emeraldpay.dshackle -import io.emeraldpay.dshackle.config.CacheConfig -import io.emeraldpay.dshackle.config.MainConfig -import io.emeraldpay.dshackle.config.MainConfigReader -import io.emeraldpay.dshackle.config.MonitoringConfig -import io.emeraldpay.dshackle.config.TokensConfig -import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.config.* import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier @@ -120,4 +115,9 @@ open class Config( open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig { return mainConfig.monitoring } + + @Bean + open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig { + return mainConfig.health + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt new file mode 100644 index 00000000..ebd66c88 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain + +class HealthConfig { + + companion object { + fun default(): HealthConfig { + return HealthConfig() + } + } + + var port: Int = 8082 + var host: String = "127.0.0.1" + var path: String = "/health" + val chains = HashMap() + + fun isEnabled(): Boolean { + return chains.isNotEmpty() + } + + fun configs(): Collection { + return chains.values + } + + data class ChainConfig( + val blockchain: Chain, + val minAvailable: Int = 1 + ) + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt new file mode 100644 index 00000000..1058f6f4 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain +import org.slf4j.LoggerFactory +import org.yaml.snakeyaml.nodes.CollectionNode +import org.yaml.snakeyaml.nodes.MappingNode +import java.io.InputStream + +class HealthConfigReader : YamlConfigReader(), ConfigReader { + + companion object { + private val log = LoggerFactory.getLogger(HealthConfigReader::class.java) + } + + fun read(input: InputStream): HealthConfig { + val configNode = readNode(input) + return read(configNode) + } + + override fun read(input: MappingNode?): HealthConfig { + return readInternal(getMapping(input, "health")) + } + + fun readInternal(input: MappingNode?): HealthConfig { + if (input == null) { + return HealthConfig.default() + } + val config = HealthConfig() + getValueAsString(input, "host")?.let { + config.host = it + } + getValueAsInt(input, "port")?.let { + config.port = it + } + getValueAsString(input, "path")?.let { + config.path = it + } + readBlockchains(config, getList(input, "blockchains")) + return config + } + + fun readBlockchains(healthConfig: HealthConfig, input: CollectionNode?) { + if (input == null) { + return + } + input.value.forEach { conf -> + val chain = getValueAsString(conf, "chain") + ?.let { getBlockchain(it) } + if (chain == null) { + log.warn("Blockchain is not specified for a Health Check") + return@forEach + } + if (chain == Chain.UNSPECIFIED) { + log.warn("Using UNSPECIFIED blockchain for Health Check. Always fails") + } + if (healthConfig.chains.containsKey(chain)) { + log.warn("Duplicate Health Check config for $chain. Replace previous with new") + } + val minAvailable = getValueAsInt(conf, "min-available") ?: 1 + healthConfig.chains[chain] = HealthConfig.ChainConfig(chain, minAvailable.coerceAtLeast(0)) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index 286bff26..4fa6cf86 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -25,4 +25,5 @@ class MainConfig { var tokens: TokensConfig? = null var monitoring: MonitoringConfig = MonitoringConfig.default() var accessLogConfig: AccessLogConfig = AccessLogConfig.default() + var health: HealthConfig = HealthConfig.default() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index cba114f8..7ed84bd6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -35,6 +35,7 @@ class MainConfigReader( private val tokensConfigReader = TokensConfigReader() private val monitoringConfigReader = MonitoringConfigReader() private val accessLogReader = AccessLogReader() + private val healthConfigReader = HealthConfigReader() fun read(input: InputStream): MainConfig? { val configNode = readNode(input) @@ -71,6 +72,9 @@ class MainConfigReader( accessLogReader.read(input).let { config.accessLogConfig = it } + healthConfigReader.read(input).let { + config.health = it + } return config } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt new file mode 100644 index 00000000..76c62d3b --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.monitoring + +import com.sun.net.httpserver.HttpServer +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.HttpStatus +import org.springframework.stereotype.Service +import java.io.IOException +import java.net.InetSocketAddress +import javax.annotation.PostConstruct + +@Service +class HealthCheckSetup( + @Autowired private val healthConfig: HealthConfig, + @Autowired private val multistreamHolder: MultistreamHolder +) { + + companion object { + private val log = LoggerFactory.getLogger(HealthCheckSetup::class.java) + } + + @PostConstruct + fun start() { + if (!healthConfig.isEnabled()) { + return + } + // use standard JVM server with a single thread blocking processing + // health check is a rare operation, no reason to set up anything complex + try { + log.info("Run Health Server on ${healthConfig.host}:${healthConfig.port}${healthConfig.path}") + val server = HttpServer.create( + InetSocketAddress( + healthConfig.host, + healthConfig.port + ), + 0 + ) + server.createContext(healthConfig.path) { httpExchange -> + val response = getHealth() + val ok = response == "OK" + val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE + httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong()) + httpExchange.responseBody.use { os -> + os.write(response.toByteArray()) + } + } + Thread(server::start).start() + } catch (e: IOException) { + log.error("Failed to start Health Server", e) + } + } + + fun getHealth(): String { + val errors = healthConfig.configs().mapNotNull { + val up = multistreamHolder.getUpstream(it.blockchain) + if (up == null || !up.isAvailable()) { + return@mapNotNull "${it.blockchain} UNAVAILABLE" + } + val avail = up.getAll().count { it.getStatus() == UpstreamAvailability.OK } + if (avail < it.minAvailable) { + return@mapNotNull "${it.blockchain} LACKS MIN AVAILABILITY" + } + null + } + return if (errors.isEmpty()) { + "OK" + } else { + errors.joinToString("\n") + } + } +} \ No newline at end of file diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy new file mode 100644 index 00000000..6add6d56 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.grpc.Chain +import spock.lang.Specification + +class HealthConfigReaderSpec extends Specification { + + HealthConfigReader reader = new HealthConfigReader() + + def "Read empty"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-empty.yaml") + when: + def act = reader.read(config) + + then: + !act.enabled + act.port == 8082 + act.host == "127.0.0.1" + act.path == "/health" + act.configs().size() == 0 + } + + def "Read single"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-1.yaml") + when: + def act = reader.read(config) + + then: + act.enabled + act.port == 8082 + act.host == "127.0.0.1" + act.path == "/health" + with(act.configs()) { + size() == 1 + with(it[0]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 2 + } + } + } + + def "Read multiple"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-2.yaml") + when: + def act = reader.read(config) + + then: + act.enabled + act.port == 10003 + act.host == "0.0.0.0" + act.path == "/healtz" + with(act.configs().toSorted { it.blockchain.id }) { + size() == 2 + with(it[0]) { + it.blockchain == Chain.BITCOIN + it.minAvailable == 1 + } + with(it[1]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 2 + } + } + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy index b790e348..f8fbdd7a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy @@ -15,7 +15,7 @@ */ package io.emeraldpay.dshackle.config -import io.emeraldpay.dshackle.FileResolver + import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.grpc.Chain import spock.lang.Specification @@ -73,6 +73,16 @@ class MainConfigReaderSpec extends Specification { blockchain == Chain.TESTNET_RINKEBY } } + with(act.health) { + it.enabled + with(it.configs()) { + it.size() == 1 + with(it[0]) { + it.blockchain == Chain.ETHEREUM + it.minAvailable == 1 + } + } + } act.upstreams != null with(act.upstreams) { defaultOptions != null diff --git a/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy new file mode 100644 index 00000000..b9f6d4da --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy @@ -0,0 +1,127 @@ +/** + * Copyright (c) 2021 EmeraldPay, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.monitoring + +import io.emeraldpay.dshackle.config.HealthConfig +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.grpc.Chain +import spock.lang.Specification + +class HealthCheckSetupSpec extends Specification { + + def "OK when meets availability - 1"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 1 + ) + } + def up1 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act == "OK" + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1] + 1 * up1.status >> UpstreamAvailability.OK + } + + def "OK when meets availability - 1 - bitcoin"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.BITCOIN] = new HealthConfig.ChainConfig( + Chain.BITCOIN, 1 + ) + } + def up1 = Mock(Upstream) + def bitcoinUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act == "OK" + 1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams + 1 * bitcoinUpstreams.available >> true + 1 * bitcoinUpstreams.getAll() >> [up1] + 1 * up1.status >> UpstreamAvailability.OK + } + + def "OK when meets availability - 2/3"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act == "OK" + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + 1 * up1.status >> UpstreamAvailability.OK + 1 * up2.status >> UpstreamAvailability.SYNCING + 1 * up3.status >> UpstreamAvailability.OK + } + + def "OK when doesn't meet availability - 2/3"() { + setup: + def config = new HealthConfig().tap { + it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig( + Chain.ETHEREUM, 2 + ) + } + def up1 = Mock(Upstream) + def up2 = Mock(Upstream) + def up3 = Mock(Upstream) + def ethereumUpstreams = Mock(Multistream) + def multistream = Mock(MultistreamHolder) + def check = new HealthCheckSetup(config, multistream) + + when: + def act = check.health + + then: + act != "OK" + 1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams + 1 * ethereumUpstreams.available >> true + 1 * ethereumUpstreams.getAll() >> [up1, up2, up3] + 1 * up1.status >> UpstreamAvailability.OK + 1 * up2.status >> UpstreamAvailability.SYNCING + 1 * up3.status >> UpstreamAvailability.LAGGING + } +} diff --git a/src/test/resources/dshackle-full.yaml b/src/test/resources/dshackle-full.yaml index e16ccde9..a3a9ad8a 100644 --- a/src/test/resources/dshackle-full.yaml +++ b/src/test/resources/dshackle-full.yaml @@ -16,6 +16,12 @@ cache: enabled: true host: redis-master +health: + port: 8083 + blockchains: + - chain: ethereum + min-available: 1 + proxy: port: 8082 tls: diff --git a/src/test/resources/dshackle-health-1.yaml b/src/test/resources/dshackle-health-1.yaml new file mode 100644 index 00000000..ca32dbb1 --- /dev/null +++ b/src/test/resources/dshackle-health-1.yaml @@ -0,0 +1,6 @@ +version: v1 + +health: + blockchains: + - chain: ethereum + min-available: 2 \ No newline at end of file diff --git a/src/test/resources/dshackle-health-2.yaml b/src/test/resources/dshackle-health-2.yaml new file mode 100644 index 00000000..2e78abdd --- /dev/null +++ b/src/test/resources/dshackle-health-2.yaml @@ -0,0 +1,11 @@ +version: v1 + +health: + port: 10003 + host: 0.0.0.0 + path: /healtz + blockchains: + - chain: ethereum + min-available: 2 + - chain: bitcoin + min-available: 1 \ No newline at end of file diff --git a/src/test/resources/dshackle-health-empty.yaml b/src/test/resources/dshackle-health-empty.yaml new file mode 100644 index 00000000..f1a00ccc --- /dev/null +++ b/src/test/resources/dshackle-health-empty.yaml @@ -0,0 +1,3 @@ +version: v1 + +health: \ No newline at end of file