solution: health check endpoint

fix: #111

src/main/kotlin/io/emeraldpay/dshackle/Config.kt

@@ -16,12 +16,7 @@
  */
 package io.emeraldpay.dshackle
 
-import io.emeraldpay.dshackle.config.CacheConfig
+import io.emeraldpay.dshackle.config.*
-import io.emeraldpay.dshackle.config.MainConfig
-import io.emeraldpay.dshackle.config.MainConfigReader
-import io.emeraldpay.dshackle.config.MonitoringConfig
-import io.emeraldpay.dshackle.config.TokensConfig
-import io.emeraldpay.dshackle.config.UpstreamsConfig
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Qualifier
@@ -120,4 +115,9 @@ open class Config(
     open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig {
         return mainConfig.monitoring
     }
+
+    @Bean
+    open fun healthConfig(@Autowired mainConfig: MainConfig): HealthConfig {
+        return mainConfig.health
+    }
 }

src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfig.kt

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.grpc.Chain
+
+class HealthConfig {
+
+    companion object {
+        fun default(): HealthConfig {
+            return HealthConfig()
+        }
+    }
+
+    var port: Int = 8082
+    var host: String = "127.0.0.1"
+    var path: String = "/health"
+    val chains = HashMap<Chain, ChainConfig>()
+
+    fun isEnabled(): Boolean {
+        return chains.isNotEmpty()
+    }
+
+    fun configs(): Collection<ChainConfig> {
+        return chains.values
+    }
+
+    data class ChainConfig(
+        val blockchain: Chain,
+        val minAvailable: Int = 1
+    )
+
+}

src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt

@@ -0,0 +1,79 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.grpc.Chain
+import org.slf4j.LoggerFactory
+import org.yaml.snakeyaml.nodes.CollectionNode
+import org.yaml.snakeyaml.nodes.MappingNode
+import java.io.InputStream
+
+class HealthConfigReader : YamlConfigReader(), ConfigReader<HealthConfig> {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(HealthConfigReader::class.java)
+    }
+
+    fun read(input: InputStream): HealthConfig {
+        val configNode = readNode(input)
+        return read(configNode)
+    }
+
+    override fun read(input: MappingNode?): HealthConfig {
+        return readInternal(getMapping(input, "health"))
+    }
+
+    fun readInternal(input: MappingNode?): HealthConfig {
+        if (input == null) {
+            return HealthConfig.default()
+        }
+        val config = HealthConfig()
+        getValueAsString(input, "host")?.let {
+            config.host = it
+        }
+        getValueAsInt(input, "port")?.let {
+            config.port = it
+        }
+        getValueAsString(input, "path")?.let {
+            config.path = it
+        }
+        readBlockchains(config, getList<MappingNode>(input, "blockchains"))
+        return config
+    }
+
+    fun readBlockchains(healthConfig: HealthConfig, input: CollectionNode<MappingNode>?) {
+        if (input == null) {
+            return
+        }
+        input.value.forEach { conf ->
+            val chain = getValueAsString(conf, "chain")
+                ?.let { getBlockchain(it) }
+            if (chain == null) {
+                log.warn("Blockchain is not specified for a Health Check")
+                return@forEach
+            }
+            if (chain == Chain.UNSPECIFIED) {
+                log.warn("Using UNSPECIFIED blockchain for Health Check. Always fails")
+            }
+            if (healthConfig.chains.containsKey(chain)) {
+                log.warn("Duplicate Health Check config for $chain. Replace previous with new")
+            }
+            val minAvailable = getValueAsInt(conf, "min-available") ?: 1
+            healthConfig.chains[chain] = HealthConfig.ChainConfig(chain, minAvailable.coerceAtLeast(0))
+        }
+    }
+
+}

src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt

@@ -25,4 +25,5 @@ class MainConfig {
     var tokens: TokensConfig? = null
     var monitoring: MonitoringConfig = MonitoringConfig.default()
     var accessLogConfig: AccessLogConfig = AccessLogConfig.default()
+    var health: HealthConfig = HealthConfig.default()
 }

src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt

@@ -35,6 +35,7 @@ class MainConfigReader(
     private val tokensConfigReader = TokensConfigReader()
     private val monitoringConfigReader = MonitoringConfigReader()
     private val accessLogReader = AccessLogReader()
+    private val healthConfigReader = HealthConfigReader()
 
     fun read(input: InputStream): MainConfig? {
         val configNode = readNode(input)
@@ -71,6 +72,9 @@ class MainConfigReader(
         accessLogReader.read(input).let {
             config.accessLogConfig = it
         }
+        healthConfigReader.read(input).let {
+            config.health = it
+        }
         return config
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.monitoring
+
+import com.sun.net.httpserver.HttpServer
+import io.emeraldpay.dshackle.config.HealthConfig
+import io.emeraldpay.dshackle.upstream.MultistreamHolder
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.http.HttpStatus
+import org.springframework.stereotype.Service
+import java.io.IOException
+import java.net.InetSocketAddress
+import javax.annotation.PostConstruct
+
+@Service
+class HealthCheckSetup(
+    @Autowired private val healthConfig: HealthConfig,
+    @Autowired private val multistreamHolder: MultistreamHolder
+) {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(HealthCheckSetup::class.java)
+    }
+
+    @PostConstruct
+    fun start() {
+        if (!healthConfig.isEnabled()) {
+            return
+        }
+        // use standard JVM server with a single thread blocking processing
+        // health check is a rare operation, no reason to set up anything complex
+        try {
+            log.info("Run Health Server on ${healthConfig.host}:${healthConfig.port}${healthConfig.path}")
+            val server = HttpServer.create(
+                InetSocketAddress(
+                    healthConfig.host,
+                    healthConfig.port
+                ),
+                0
+            )
+            server.createContext(healthConfig.path) { httpExchange ->
+                val response = getHealth()
+                val ok = response == "OK"
+                val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE
+                httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong())
+                httpExchange.responseBody.use { os ->
+                    os.write(response.toByteArray())
+                }
+            }
+            Thread(server::start).start()
+        } catch (e: IOException) {
+            log.error("Failed to start Health Server", e)
+        }
+    }
+
+    fun getHealth(): String {
+        val errors = healthConfig.configs().mapNotNull {
+            val up = multistreamHolder.getUpstream(it.blockchain)
+            if (up == null || !up.isAvailable()) {
+                return@mapNotNull "${it.blockchain} UNAVAILABLE"
+            }
+            val avail = up.getAll().count { it.getStatus() == UpstreamAvailability.OK }
+            if (avail < it.minAvailable) {
+                return@mapNotNull "${it.blockchain} LACKS MIN AVAILABILITY"
+            }
+            null
+        }
+        return if (errors.isEmpty()) {
+            "OK"
+        } else {
+            errors.joinToString("\n")
+        }
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/config/HealthConfigReaderSpec.groovy

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.config
+
+import io.emeraldpay.grpc.Chain
+import spock.lang.Specification
+
+class HealthConfigReaderSpec extends Specification {
+
+    HealthConfigReader reader = new HealthConfigReader()
+
+    def "Read empty"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-empty.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        !act.enabled
+        act.port == 8082
+        act.host == "127.0.0.1"
+        act.path == "/health"
+        act.configs().size() == 0
+    }
+
+    def "Read single"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-1.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        act.enabled
+        act.port == 8082
+        act.host == "127.0.0.1"
+        act.path == "/health"
+        with(act.configs()) {
+            size() == 1
+            with(it[0]) {
+                it.blockchain == Chain.ETHEREUM
+                it.minAvailable == 2
+            }
+        }
+    }
+
+    def "Read multiple"() {
+        setup:
+        def config = this.class.getClassLoader().getResourceAsStream("dshackle-health-2.yaml")
+        when:
+        def act = reader.read(config)
+
+        then:
+        act.enabled
+        act.port == 10003
+        act.host == "0.0.0.0"
+        act.path == "/healtz"
+        with(act.configs().toSorted { it.blockchain.id }) {
+            size() == 2
+            with(it[0]) {
+                it.blockchain == Chain.BITCOIN
+                it.minAvailable == 1
+            }
+            with(it[1]) {
+                it.blockchain == Chain.ETHEREUM
+                it.minAvailable == 2
+            }
+        }
+    }
+}

src/test/groovy/io/emeraldpay/dshackle/config/MainConfigReaderSpec.groovy

@@ -15,7 +15,7 @@
  */
 package io.emeraldpay.dshackle.config
 
-import io.emeraldpay.dshackle.FileResolver
+
 import io.emeraldpay.dshackle.test.TestingCommons
 import io.emeraldpay.grpc.Chain
 import spock.lang.Specification
@@ -73,6 +73,16 @@ class MainConfigReaderSpec extends Specification {
                 blockchain == Chain.TESTNET_RINKEBY
             }
         }
+        with(act.health) {
+            it.enabled
+            with(it.configs()) {
+                it.size() == 1
+                with(it[0]) {
+                    it.blockchain == Chain.ETHEREUM
+                    it.minAvailable == 1
+                }
+            }
+        }
         act.upstreams != null
         with(act.upstreams) {
             defaultOptions != null

src/test/groovy/io/emeraldpay/dshackle/monitoring/HealthCheckSetupSpec.groovy

@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) 2021 EmeraldPay, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.emeraldpay.dshackle.monitoring
+
+import io.emeraldpay.dshackle.config.HealthConfig
+import io.emeraldpay.dshackle.upstream.Multistream
+import io.emeraldpay.dshackle.upstream.MultistreamHolder
+import io.emeraldpay.dshackle.upstream.Upstream
+import io.emeraldpay.dshackle.upstream.UpstreamAvailability
+import io.emeraldpay.grpc.Chain
+import spock.lang.Specification
+
+class HealthCheckSetupSpec extends Specification {
+
+    def "OK when meets availability - 1"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 1
+            )
+        }
+        def up1 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act == "OK"
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1]
+        1 * up1.status >> UpstreamAvailability.OK
+    }
+
+    def "OK when meets availability - 1 - bitcoin"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.BITCOIN] = new HealthConfig.ChainConfig(
+                    Chain.BITCOIN, 1
+            )
+        }
+        def up1 = Mock(Upstream)
+        def bitcoinUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act == "OK"
+        1 * multistream.getUpstream(Chain.BITCOIN) >> bitcoinUpstreams
+        1 * bitcoinUpstreams.available >> true
+        1 * bitcoinUpstreams.getAll() >> [up1]
+        1 * up1.status >> UpstreamAvailability.OK
+    }
+
+    def "OK when meets availability - 2/3"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 2
+            )
+        }
+        def up1 = Mock(Upstream)
+        def up2 = Mock(Upstream)
+        def up3 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act == "OK"
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
+        1 * up1.status >> UpstreamAvailability.OK
+        1 * up2.status >> UpstreamAvailability.SYNCING
+        1 * up3.status >> UpstreamAvailability.OK
+    }
+
+    def "OK when doesn't meet availability - 2/3"() {
+        setup:
+        def config = new HealthConfig().tap {
+            it.chains[Chain.ETHEREUM] = new HealthConfig.ChainConfig(
+                    Chain.ETHEREUM, 2
+            )
+        }
+        def up1 = Mock(Upstream)
+        def up2 = Mock(Upstream)
+        def up3 = Mock(Upstream)
+        def ethereumUpstreams = Mock(Multistream)
+        def multistream = Mock(MultistreamHolder)
+        def check = new HealthCheckSetup(config, multistream)
+
+        when:
+        def act = check.health
+
+        then:
+        act != "OK"
+        1 * multistream.getUpstream(Chain.ETHEREUM) >> ethereumUpstreams
+        1 * ethereumUpstreams.available >> true
+        1 * ethereumUpstreams.getAll() >> [up1, up2, up3]
+        1 * up1.status >> UpstreamAvailability.OK
+        1 * up2.status >> UpstreamAvailability.SYNCING
+        1 * up3.status >> UpstreamAvailability.LAGGING
+    }
+}

src/test/resources/dshackle-full.yaml

@@ -16,6 +16,12 @@ cache:
     enabled: true
     host: redis-master
 
+health:
+  port: 8083
+  blockchains:
+    - chain: ethereum
+      min-available: 1
+
 proxy:
   port: 8082
   tls:

src/test/resources/dshackle-health-1.yaml

@@ -0,0 +1,6 @@
+version: v1
+
+health:
+  blockchains:
+    - chain: ethereum
+      min-available: 2

src/test/resources/dshackle-health-2.yaml

@@ -0,0 +1,11 @@
+version: v1
+
+health:
+  port: 10003
+  host: 0.0.0.0
+  path: /healtz
+  blockchains:
+    - chain: ethereum
+      min-available: 2
+    - chain: bitcoin
+      min-available: 1

src/test/resources/dshackle-health-empty.yaml

@@ -0,0 +1,3 @@
+version: v1
+
+health: