StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solution: health check with detailed description of health

Browse Source
This commit is contained in:
Igor Artamonov
2021-10-22 22:12:48 -04:00
parent e205985da3
commit 5f4a8a4ac4
8 changed files with 140 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/kotlin/io/emeraldpay/dshackle/Global.kt
View File

@@ -27,8 +27,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import java.text.SimpleDateFormat
import java.util.TimeZone
import java.util.*
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
@@ -38,6 +39,35 @@ class Global {
var metricsExtended = false
val chainNames = mapOf(
"ethereum" to Chain.ETHEREUM,
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
"eth" to Chain.ETHEREUM,
"polygon" to Chain.MATIC,
"matic" to Chain.MATIC,
"etc" to Chain.ETHEREUM_CLASSIC,
"morden" to Chain.TESTNET_MORDEN,
"kovan" to Chain.TESTNET_KOVAN,
"kovan-testnet" to Chain.TESTNET_KOVAN,
"goerli" to Chain.TESTNET_GOERLI,
"goerli-testnet" to Chain.TESTNET_GOERLI,
"rinkeby" to Chain.TESTNET_RINKEBY,
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
"ropsten" to Chain.TESTNET_ROPSTEN,
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
"bitcoin" to Chain.BITCOIN,
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
)
fun chainById(id: String?): Chain {
if (id == null) {
return Chain.UNSPECIFIED
}
return chainNames[
id.lowercase(Locale.getDefault()).replace("_", "-").trim()
] ?: Chain.UNSPECIFIED
}
@JvmStatic
val objectMapper: ObjectMapper = createObjectMapper()

3
src/main/kotlin/io/emeraldpay/dshackle/config/HealthConfigReader.kt
View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.CollectionNode
@@ -60,7 +61,7 @@ class HealthConfigReader : YamlConfigReader(), ConfigReader<HealthConfig> {
}
input.value.forEach { conf ->
val chain = getValueAsString(conf, "chain")
?.let { getBlockchain(it) }
?.let { Global.chainById(it) }
if (chain == null) {
log.warn("Blockchain is not specified for a Health Check")
return@forEach

5
src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt
View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import io.emeraldpay.grpc.Chain
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
@@ -69,10 +70,10 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader<ProxyConfig> {
}
currentRoutes.add(id)
val blockchain = getValueAsString(route, "blockchain")
if (StringUtils.isEmpty(blockchain) || getBlockchain(blockchain!!) == Chain.UNSPECIFIED) {
if (StringUtils.isEmpty(blockchain) || Global.chainById(blockchain!!) == Chain.UNSPECIFIED) {
throw InvalidConfigYamlException(filename, route.startMark, "Invalid blockchain or not specified")
}
ProxyConfig.Route(id, getBlockchain(blockchain))
ProxyConfig.Route(id, Global.chainById(blockchain))
}
}
if (config.routes.isEmpty()) {

3
src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt
View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Global
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode
import java.io.InputStream
@@ -34,7 +35,7 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader<TokensConfig> {
val token = TokensConfig.Token()
token.id = getValueAsString(node, "id")
token.blockchain = getValueAsString(node, "blockchain")?.let {
getBlockchain(it)
Global.chainById(it)
}
token.address = getValueAsString(node, "address")
token.name = getValueAsString(node, "name")

9
src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt
View File

@@ -143,13 +143,4 @@ abstract class YamlConfigReader {
}
}
// ----
fun getBlockchain(id: String): Chain {
return Chain.values().find { chain ->
chain.name == id.uppercase(Locale.getDefault()) ||
chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) ||
chain.id.toString() == id
} ?: Chain.UNSPECIFIED
}
}

67
src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt
View File

@@ -54,12 +54,17 @@ class HealthCheckSetup(
0
)
server.createContext(healthConfig.path) { httpExchange ->
val response = getHealth()
val ok = response == "OK"
val response = if (httpExchange.requestURI.query == "detailed") {
getDetailedHealth()
} else {
getHealth()
}
val ok = response.ok
val data = response.details.joinToString("\n")
val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE
httpExchange.sendResponseHeaders(code.value(), response.toByteArray().size.toLong())
httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong())
httpExchange.responseBody.use { os ->
os.write(response.toByteArray())
os.write(data.toByteArray())
}
}
Thread(server::start).start()
@@ -68,7 +73,7 @@ class HealthCheckSetup(
}
}
fun getHealth(): String {
fun getHealth(): Detailed {
val errors = healthConfig.configs().mapNotNull {
val up = multistreamHolder.getUpstream(it.blockchain)
if (up == null || !up.isAvailable()) {
@@ -81,9 +86,57 @@ class HealthCheckSetup(
null
}
return if (errors.isEmpty()) {
"OK"
Detailed(true, listOf("OK"))
} else {
errors.joinToString("\n")
Detailed(false, errors)
}
}
fun getDetailedHealth(): Detailed {
val chains = multistreamHolder.getAvailable()
val allEnabled = healthConfig.configs().all { chains.contains(it.blockchain) }
var anyUnavailable = false
val details = chains.flatMap { chain ->
var chainUnavailable = false
val up = multistreamHolder.getUpstream(chain)
val required = healthConfig.chains[chain]
if (up == null || !up.isAvailable()) {
if (required != null) {
anyUnavailable = true
}
listOf("${chain.name} UNAVAILABLE")
} else {
val ups = up.getAll()
val checks = if (required != null) {
val avail = ups.count { it.getStatus() == UpstreamAvailability.OK }
if (avail < required.minAvailable) {
chainUnavailable = true
listOf(" LACKS MIN AVAILABILITY")
} else emptyList()
} else emptyList()
val upDetails = ups.map {
" ${it.getId()} ${it.getStatus()} with lag=${it.getLag()}"
}
val status = if (chainUnavailable) "UNAVAILABLE" else "AVAILABLE"
anyUnavailable = anyUnavailable || chainUnavailable
listOf("${chain.name} $status") + upDetails + checks
}
}
val detailsUnavailable = if (!allEnabled) {
healthConfig.configs()
.filter { !chains.contains(it.blockchain) }
.map {
"${it.blockchain.name} UNAVAILABLE"
}
} else emptyList()
return Detailed(
allEnabled && !anyUnavailable,
detailsUnavailable + details
)
}
data class Detailed(
val ok: Boolean,
val details: List<String>
)
}

29
src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
View File

@@ -17,6 +17,7 @@
package io.emeraldpay.dshackle.startup
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.cache.CachesFactory
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
@@ -57,26 +58,6 @@ open class ConfiguredUpstreams(
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
private var seq = AtomicInteger(0)
private val chainNames = mapOf(
"ethereum" to Chain.ETHEREUM,
"ethereum-classic" to Chain.ETHEREUM_CLASSIC,
"eth" to Chain.ETHEREUM,
"polygon" to Chain.MATIC,
"matic" to Chain.MATIC,
"etc" to Chain.ETHEREUM_CLASSIC,
"morden" to Chain.TESTNET_MORDEN,
"kovan" to Chain.TESTNET_KOVAN,
"kovan-testnet" to Chain.TESTNET_KOVAN,
"goerli" to Chain.TESTNET_GOERLI,
"goerli-testnet" to Chain.TESTNET_GOERLI,
"rinkeby" to Chain.TESTNET_RINKEBY,
"rinkeby-testnet" to Chain.TESTNET_RINKEBY,
"ropsten" to Chain.TESTNET_ROPSTEN,
"ropsten-testnet" to Chain.TESTNET_ROPSTEN,
"bitcoin" to Chain.BITCOIN,
"bitcoin-testnet" to Chain.TESTNET_BITCOIN
)
@PostConstruct
fun start() {
log.debug("Starting upstreams")
@@ -87,8 +68,8 @@ open class ConfiguredUpstreams(
val options = up.options ?: UpstreamsConfig.Options()
buildGrpcUpstream(up.cast(UpstreamsConfig.GrpcConnection::class.java), options)
} else {
val chain = chainNames[up.chain]
if (chain == null) {
val chain = Global.chainById(up.chain)
if (chain == Chain.UNSPECIFIED) {
log.error("Chain is unknown: ${up.chain}")
return@forEach
}
@@ -114,7 +95,7 @@ open class ConfiguredUpstreams(
val defaultOptions = HashMap<Chain, UpstreamsConfig.Options>()
config.defaultOptions.forEach { defaultsConfig ->
defaultsConfig.chains?.forEach { chainName ->
chainNames[chainName]?.let { chain ->
Global.chainById(chainName).let { chain ->
defaultsConfig.options?.let { options ->
if (!defaultOptions.containsKey(chain)) {
defaultOptions[chain] = options
@@ -274,7 +255,7 @@ open class ConfiguredUpstreams(
// "unknown" is not supposed to happen
Tag.of("upstream", config.id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode)
Tag.of("chain", (Global.chainById(config.chain).chainCode))
)
val metrics = RpcMetrics(
Timer.builder("upstream.rpc.conn")