StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solution: metrics for upstream calls

Browse Source
This commit is contained in:
Igor Artamonov
2021-02-10 14:36:44 -05:00
parent 3df27bee5d
commit 981a5dc589
8 changed files with 144 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
View File

@@ -32,10 +32,15 @@ import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstreams
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer
import java.net.URI
import java.util.*
import java.util.concurrent.atomic.AtomicInteger
@@ -235,9 +240,27 @@ open class ConfiguredUpstreams(
fileResolver.resolve(ca).readBytes()
}
}
val metricsTags = listOf(
// "unknown" is not supposed to happen
Tag.of("upstream", config.id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED ).chainCode)
)
val metrics = RpcMetrics(
Timer.builder("upstream.rpc.conn")
.description("Request time through a HTTP JSON RPC connection")
.tags(metricsTags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry),
Counter.builder("upstream.rpc.err")
.description("Errors received on request through HTTP JSON RPC connection")
.tags(metricsTags)
.register(Metrics.globalRegistry)
)
urls.add(endpoint.url)
JsonRpcHttpClient(
endpoint.url.toString(),
metrics,
conn.rpc?.basicAuth,
tls
)

36
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt
View File

@@ -26,9 +26,14 @@ import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.startup.UpstreamChange
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.Chain
import io.grpc.ManagedChannelBuilder
import io.grpc.netty.NettyChannelBuilder
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer
import io.netty.handler.ssl.*
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.exception.ExceptionUtils
@@ -157,21 +162,40 @@ class GrpcUpstreams(
}
fun getOrCreate(chain: Chain): UpstreamChange {
val metricsTags = listOf(
// "unknown" is not supposed to happen
Tag.of("upstream", id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", chain.chainCode)
)
val metrics = RpcMetrics(
Timer.builder("upstream.grpc.conn")
.description("Request time through a gRPC connection")
.tags(metricsTags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry),
Counter.builder("upstream.grpc.err")
.description("Errors received on request through gRPC connection")
.tags(metricsTags)
.register(Metrics.globalRegistry)
)
val blockchainType = BlockchainType.fromBlockchain(chain)
if (blockchainType == BlockchainType.ETHEREUM) {
return getOrCreateEthereum(chain)
return getOrCreateEthereum(chain, metrics)
} else if (blockchainType == BlockchainType.BITCOIN) {
return getOrCreateBitcoin(chain)
return getOrCreateBitcoin(chain, metrics)
} else {
throw IllegalArgumentException("Unsupported blockchain: $chain")
}
}
fun getOrCreateEthereum(chain: Chain): UpstreamChange {
fun getOrCreateEthereum(chain: Chain, metrics: RpcMetrics): UpstreamChange {
lock.withLock {
val current = known[chain]
return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain)
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = EthereumGrpcUpstream(id, chain, client!!, rpcClient)
created.timeout = this.timeout
known[chain] = created
@@ -183,11 +207,11 @@ class GrpcUpstreams(
}
}
fun getOrCreateBitcoin(chain: Chain): UpstreamChange {
fun getOrCreateBitcoin(chain: Chain, metrics: RpcMetrics): UpstreamChange {
lock.withLock {
val current = known[chain]
return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain)
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = BitcoinGrpcUpstream(id, chain, client!!, rpcClient)
created.timeout = this.timeout
known[chain] = created

41
src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt
View File

@@ -27,11 +27,14 @@ import io.grpc.Channel
import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.RpcResponseError
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.util.concurrent.TimeUnit
class JsonRpcGrpcClient(
private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val chain: Chain
private val chain: Chain,
private val metrics: RpcMetrics
) {
companion object {
@@ -39,18 +42,18 @@ class JsonRpcGrpcClient(
}
fun forSelector(matcher: Selector.Matcher): Reader<JsonRpcRequest, JsonRpcResponse> {
return Executor(stub, chain, matcher)
return Executor(stub, chain, matcher, metrics)
}
class Executor(
private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val chain: Chain,
private val matcher: Selector.Matcher
private val matcher: Selector.Matcher,
private val metrics: RpcMetrics
) : Reader<JsonRpcRequest, JsonRpcResponse> {
private val parser = JsonRpcParser()
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> {
var startTime: Long = 0
val req = BlockchainOuterClass.NativeCallRequest.newBuilder()
.setChainValue(chain.id)
@@ -68,14 +71,26 @@ class JsonRpcGrpcClient(
req.addItems(it)
}
return stub.nativeCall(req.build())
.single()
.flatMap { resp ->
if (resp.succeed) {
val bytes = resp.payload.toByteArray()
Mono.just(JsonRpcResponse(bytes, null))
} else {
Mono.error(RpcException(RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR, resp.errorMessage))
return Mono.just(key)
.doOnNext {
startTime = System.nanoTime()
}.flatMap {
stub.nativeCall(req.build())
.single()
.flatMap { resp ->
if (resp.succeed) {
val bytes = resp.payload.toByteArray()
Mono.just(JsonRpcResponse(bytes, null))
} else {
metrics.errors.increment()
Mono.error(RpcException(RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR, resp.errorMessage))
}
}
}
.doOnNext {
if (startTime > 0) {
val now = System.nanoTime()
metrics.timer.record(now - startTime, TimeUnit.NANOSECONDS)
}
}
}

16
src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClient.kt
View File

@@ -19,6 +19,8 @@ import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.reader.Reader
import io.infinitape.etherjar.rpc.RpcException
import io.infinitape.etherjar.rpc.RpcResponseError
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Timer
import io.netty.buffer.Unpooled
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpHeaders
@@ -31,6 +33,7 @@ import java.security.KeyStore
import java.security.cert.CertificateFactory
import java.security.cert.X509Certificate
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.function.Consumer
/**
@@ -38,6 +41,7 @@ import java.util.function.Consumer
*/
class JsonRpcHttpClient(
private val target: String,
private val metrics: RpcMetrics,
basicAuth: AuthConfig.ClientBasicAuth? = null,
tlsCAAuth: ByteArray? = null
) : Reader<JsonRpcRequest, JsonRpcResponse> {
@@ -98,9 +102,19 @@ class JsonRpcHttpClient(
}
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> {
var startTime: Long = 0
return Mono.just(key)
.map(JsonRpcRequest::toJson)
.doOnNext {
startTime = System.nanoTime()
}
.flatMap(this@JsonRpcHttpClient::execute)
.doOnNext {
if (startTime > 0) {
val now = System.nanoTime()
metrics.timer.record(now - startTime, TimeUnit.NANOSECONDS)
}
}
.map(parser::parse)
.onErrorResume { t ->
val err = when (t) {
@@ -108,7 +122,9 @@ class JsonRpcHttpClient(
is JsonRpcException -> JsonRpcResponse.error(t.error, JsonRpcResponse.NumberId(1))
else -> JsonRpcResponse.error(1, t.message ?: t.javaClass.name)
}
metrics.errors.increment()
Mono.just(err)
}
}
}

24
src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/RpcMetrics.kt Normal file
View File

@@ -0,0 +1,24 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.rpcclient
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Timer
class RpcMetrics(
val timer: Timer,
val errors: Counter
)