Support beacon chain (#436)

This commit is contained in:
KirillPamPam
2024-03-15 17:46:29 +04:00
committed by GitHub
parent d5414f947e
commit d44b72ec69
152 changed files with 2498 additions and 1382 deletions

View File

@@ -1,5 +1,11 @@
import com.squareup.kotlinpoet.*
import com.squareup.kotlinpoet.ClassName
import com.squareup.kotlinpoet.FileSpec
import com.squareup.kotlinpoet.FunSpec
import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy
import com.squareup.kotlinpoet.PropertySpec
import com.squareup.kotlinpoet.TypeSpec
import com.squareup.kotlinpoet.asClassName
import io.emeraldpay.dshackle.BlockchainType import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.ChainsConfigReader import io.emeraldpay.dshackle.config.ChainsConfigReader
@@ -119,6 +125,7 @@ open class CodeGen(private val config: ChainsConfig) {
"polkadot" -> "BlockchainType.POLKADOT" "polkadot" -> "BlockchainType.POLKADOT"
"solana" -> "BlockchainType.SOLANA" "solana" -> "BlockchainType.SOLANA"
"near" -> "BlockchainType.NEAR" "near" -> "BlockchainType.NEAR"
"eth-beacon-chain" -> "BlockchainType.ETHEREUM_BEACON_CHAIN"
else -> throw IllegalArgumentException("unknown blockchain type $type") else -> throw IllegalArgumentException("unknown blockchain type $type")
} }
} }

View File

@@ -1,5 +1,18 @@
package io.emeraldpay.dshackle package io.emeraldpay.dshackle
enum class BlockchainType { enum class BlockchainType(
UNKNOWN, BITCOIN, ETHEREUM, STARKNET, POLKADOT, SOLANA, NEAR; val apiType: ApiType,
) {
UNKNOWN(ApiType.JSON_RPC),
BITCOIN(ApiType.JSON_RPC),
ETHEREUM(ApiType.JSON_RPC),
STARKNET(ApiType.JSON_RPC),
POLKADOT(ApiType.JSON_RPC),
SOLANA(ApiType.JSON_RPC),
NEAR(ApiType.JSON_RPC),
ETHEREUM_BEACON_CHAIN(ApiType.REST);
}
enum class ApiType {
JSON_RPC, REST;
} }

View File

@@ -1361,3 +1361,36 @@ chain-settings:
grpcId: 1052 grpcId: 1052
short-names: [dymension] short-names: [dymension]
chain-id: 0x44c chain-id: 0x44c
- id: eth-beacon-chain
label: Ethereum Beacon Chain
type: eth-beacon-chain
settings:
expected-block-time: 12s
lags:
syncing: 6
lagging: 1
chains:
- id: Mainnet
chain-id: 0x0
short-names: [eth-beacon-chain]
code: ETH_BEACON_CHAIN
grpcId: 1053
priority: 100
- id: Goerli
chain-id: 0x0
code: GOERLI_BEACON_CHAIN
grpcId: 10069
priority: 1
short-names: [eth-beacon-chain-goerli]
- id: Sepolia
code: SEPOLIA_BEACON_CHAIN
grpcId: 10070
priority: 10
chain-id: 0x0
short-names: [eth-beacon-chain-sepolia]
- id: Holesky
code: HOLESKY_BEACON_CHAIN
grpcId: 10071
priority: 5
chain-id: 0x0
short-names: [eth-beacon-chain-holesky]

View File

@@ -21,14 +21,16 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.beaconchain.BeaconChainBlockHeader
import io.emeraldpay.dshackle.upstream.beaconchain.BeaconChainBlockHeaderDeserializer
import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspentDeserializer import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspentDeserializer
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
import io.emeraldpay.dshackle.upstream.ethereum.domain.TransactionId import io.emeraldpay.dshackle.upstream.ethereum.domain.TransactionId
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.TransactionIdSerializer import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.TransactionIdSerializer
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.Locale import java.util.Locale
import java.util.TimeZone import java.util.TimeZone
@@ -67,12 +69,13 @@ class Global {
private fun createObjectMapper(): ObjectMapper { private fun createObjectMapper(): ObjectMapper {
val module = SimpleModule("EmeraldDshackle", Version(1, 0, 0, null, null, null)) val module = SimpleModule("EmeraldDshackle", Version(1, 0, 0, null, null, null))
module.addSerializer(JsonRpcResponse::class.java, JsonRpcResponse.ResponseJsonSerializer()) module.addSerializer(ChainResponse::class.java, ChainResponse.ResponseJsonSerializer())
module.addSerializer(TransactionId::class.java, TransactionIdSerializer()) module.addSerializer(TransactionId::class.java, TransactionIdSerializer())
module.addDeserializer(EsploraUnspent::class.java, EsploraUnspentDeserializer()) module.addDeserializer(EsploraUnspent::class.java, EsploraUnspentDeserializer())
module.addDeserializer(RpcUnspent::class.java, RpcUnspentDeserializer()) module.addDeserializer(RpcUnspent::class.java, RpcUnspentDeserializer())
module.addDeserializer(JsonRpcRequest::class.java, JsonRpcRequest.Deserializer()) module.addDeserializer(ChainRequest::class.java, ChainRequest.Deserializer())
module.addDeserializer(BeaconChainBlockHeader::class.java, BeaconChainBlockHeaderDeserializer())
val objectMapper = ObjectMapper() val objectMapper = ObjectMapper()
objectMapper.registerModule(module) objectMapper.registerModule(module)

View File

@@ -20,8 +20,8 @@ import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ProxyConfig import io.emeraldpay.dshackle.config.ProxyConfig
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
import io.emeraldpay.dshackle.rpc.NativeCall import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import org.reactivestreams.Publisher import org.reactivestreams.Publisher
@@ -97,10 +97,10 @@ class HttpHandler(
} }
.onErrorResume(RpcException::class.java) { err -> .onErrorResume(RpcException::class.java) { err ->
val id = err.details?.let { val id = err.details?.let {
if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1) if (it is ChainResponse.Id) it else ChainResponse.NumberId(-1)
} ?: JsonRpcResponse.NumberId(-1) } ?: ChainResponse.NumberId(-1)
val json = JsonRpcResponse.error(err.code, err.rpcMessage, id) val json = ChainResponse.error(err.code, err.rpcMessage, id)
Mono.just(Global.objectMapper.writeValueAsString(json)) Mono.just(Global.objectMapper.writeValueAsString(json))
} }
.map { Unpooled.wrappedBuffer(it.toByteArray()) } .map { Unpooled.wrappedBuffer(it.toByteArray()) }

View File

@@ -20,10 +20,10 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.google.protobuf.ByteString import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.ethereum.json.RequestJson import io.emeraldpay.dshackle.upstream.ethereum.json.RequestJson
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service import org.springframework.stereotype.Service
import java.io.IOException import java.io.IOException
@@ -54,27 +54,27 @@ open class ReadRpcJson : Function<ByteArray, ProxyCall> {
throw RpcException( throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST, RpcResponseError.CODE_INVALID_REQUEST,
"jsonrpc version is not set", "jsonrpc version is not set",
id?.let { JsonRpcResponse.Id.from(it) }, id?.let { ChainResponse.Id.from(it) },
) )
} }
throw RpcException( throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST, RpcResponseError.CODE_INVALID_REQUEST,
"Unsupported JSON RPC version: " + json["jsonrpc"].toString(), "Unsupported JSON RPC version: " + json["jsonrpc"].toString(),
id?.let { JsonRpcResponse.Id.from(it) }, id?.let { ChainResponse.Id.from(it) },
) )
} }
if (!(json["method"] != null && json["method"] is String)) { if (!(json["method"] != null && json["method"] is String)) {
throw RpcException( throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST, RpcResponseError.CODE_INVALID_REQUEST,
"Method is not set", "Method is not set",
id?.let { JsonRpcResponse.Id.from(it) }, id?.let { ChainResponse.Id.from(it) },
) )
} }
if (json.containsKey("params") && json["params"] !is List<*>) { if (json.containsKey("params") && json["params"] !is List<*>) {
throw RpcException( throw RpcException(
RpcResponseError.CODE_INVALID_REQUEST, RpcResponseError.CODE_INVALID_REQUEST,
"Params must be an array", "Params must be an array",
id?.let { JsonRpcResponse.Id.from(it) }, id?.let { ChainResponse.Id.from(it) },
) )
} }
RequestJson<Any>( RequestJson<Any>(

View File

@@ -19,7 +19,7 @@ package io.emeraldpay.dshackle.proxy
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.rpc.NativeCall import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.ChainResponse
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service import org.springframework.stereotype.Service
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
@@ -68,22 +68,22 @@ open class WriteRpcJson {
open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? { open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? {
val id = call.ids[response.id]?.let { val id = call.ids[response.id]?.let {
JsonRpcResponse.Id.from(it) ChainResponse.Id.from(it)
} ?: return null } ?: return null
val json = if (response.isError()) { val json = if (response.isError()) {
val error = response.error!! val error = response.error!!
error.upstreamError?.let { upstreamError -> error.upstreamError?.let { upstreamError ->
JsonRpcResponse.error(upstreamError, id) ChainResponse.error(upstreamError, id)
} ?: JsonRpcResponse.error(-32002, error.message, id) } ?: ChainResponse.error(-32002, error.message, id)
} else { } else {
JsonRpcResponse.ok(response.result!!, id) ChainResponse.ok(response.result!!, id)
} }
return objectMapper.writeValueAsString(json) return objectMapper.writeValueAsString(json)
} }
fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? { fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? {
val id = call.ids[error.id] ?: return null val id = call.ids[error.id] ?: return null
val json = JsonRpcResponse.error(-32003, error.reason.message ?: "", JsonRpcResponse.Id.from(id)) val json = ChainResponse.error(-32003, error.reason.message ?: "", ChainResponse.Id.from(id))
return objectMapper.writeValueAsString(json) return objectMapper.writeValueAsString(json)
} }

View File

@@ -16,17 +16,17 @@
*/ */
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
open class AlwaysQuorum : CallQuorum { open class AlwaysQuorum : CallQuorum {
private var resolved = false private var resolved = false
private var result: JsonRpcResponse? = null private var result: ChainResponse? = null
private var rpcError: JsonRpcError? = null private var rpcError: ChainCallError? = null
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
private val resolvers = ArrayList<Upstream>() private val resolvers = ArrayList<Upstream>()
@@ -43,7 +43,7 @@ open class AlwaysQuorum : CallQuorum {
} }
override fun record( override fun record(
response: JsonRpcResponse, response: ChainResponse,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
): Boolean { ): Boolean {
@@ -55,7 +55,7 @@ open class AlwaysQuorum : CallQuorum {
} }
override fun record( override fun record(
error: JsonRpcException, error: ChainException,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
) { ) {
@@ -64,11 +64,11 @@ open class AlwaysQuorum : CallQuorum {
resolvers.add(upstream) resolvers.add(upstream)
} }
override fun getResponse(): JsonRpcResponse? { override fun getResponse(): ChainResponse? {
return result return result
} }
override fun getError(): JsonRpcError? { override fun getError(): ChainCallError? {
return rpcError return rpcError
} }

View File

@@ -16,13 +16,13 @@
*/ */
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
open class BroadcastQuorum() : CallQuorum, ValueAwareQuorum<String>(String::class.java) { open class BroadcastQuorum() : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
private var result: JsonRpcResponse? = null private var result: ChainResponse? = null
private var txid: String? = null private var txid: String? = null
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
@@ -34,7 +34,7 @@ open class BroadcastQuorum() : CallQuorum, ValueAwareQuorum<String>(String::clas
return result == null return result == null
} }
override fun getResponse(): JsonRpcResponse? { override fun getResponse(): ChainResponse? {
return result return result
} }
@@ -43,7 +43,7 @@ open class BroadcastQuorum() : CallQuorum, ValueAwareQuorum<String>(String::clas
} }
override fun recordValue( override fun recordValue(
response: JsonRpcResponse, response: ChainResponse,
responseValue: String?, responseValue: String?,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,

View File

@@ -16,10 +16,10 @@
*/ */
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
interface CallQuorum { interface CallQuorum {
@@ -27,19 +27,19 @@ interface CallQuorum {
fun isFailed(): Boolean fun isFailed(): Boolean
fun record( fun record(
response: JsonRpcResponse, response: ChainResponse,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
): Boolean ): Boolean
fun record( fun record(
error: JsonRpcException, error: ChainException,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
) )
fun getSignature(): ResponseSigner.Signature? fun getSignature(): ResponseSigner.Signature?
fun getResponse(): JsonRpcResponse? fun getResponse(): ChainResponse?
fun getError(): JsonRpcError? fun getError(): ChainCallError?
fun getResolvedBy(): Collection<Upstream> fun getResolvedBy(): Collection<Upstream>
} }

View File

@@ -1,13 +1,13 @@
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.ethereum.hex.HexQuantity import io.emeraldpay.dshackle.upstream.ethereum.hex.HexQuantity
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
class MaximumValueQuorum : CallQuorum, ValueAwareQuorum<String>(String::class.java) { class MaximumValueQuorum : CallQuorum, ValueAwareQuorum<String>(String::class.java) {
private var max: Long? = null private var max: Long? = null
private var result: JsonRpcResponse? = null private var result: ChainResponse? = null
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
override fun isResolved(): Boolean { override fun isResolved(): Boolean {
@@ -18,7 +18,7 @@ class MaximumValueQuorum : CallQuorum, ValueAwareQuorum<String>(String::class.ja
return result == null return result == null
} }
override fun getResponse(): JsonRpcResponse? { override fun getResponse(): ChainResponse? {
return result return result
} }
@@ -26,7 +26,7 @@ class MaximumValueQuorum : CallQuorum, ValueAwareQuorum<String>(String::class.ja
return sig return sig
} }
override fun recordValue( override fun recordValue(
response: JsonRpcResponse, response: ChainResponse,
responseValue: String?, responseValue: String?,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,

View File

@@ -16,10 +16,10 @@
*/ */
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
@@ -30,9 +30,9 @@ import java.util.concurrent.atomic.AtomicReference
*/ */
class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum { class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
private val result: AtomicReference<JsonRpcResponse> = AtomicReference() private val result: AtomicReference<ChainResponse> = AtomicReference()
private val failed = AtomicReference(false) private val failed = AtomicReference(false)
private var rpcError: JsonRpcError? = null private var rpcError: ChainCallError? = null
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
private val resolvers = ArrayList<Upstream>() private val resolvers = ArrayList<Upstream>()
@@ -45,7 +45,7 @@ class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
} }
override fun record( override fun record(
response: JsonRpcResponse, response: ChainResponse,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
): Boolean { ): Boolean {
@@ -60,7 +60,7 @@ class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
} }
override fun record( override fun record(
error: JsonRpcException, error: ChainException,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
) { ) {
@@ -76,11 +76,11 @@ class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
return sig return sig
} }
override fun getResponse(): JsonRpcResponse { override fun getResponse(): ChainResponse {
return result.get() return result.get()
} }
override fun getError(): JsonRpcError? { override fun getError(): ChainCallError? {
return rpcError return rpcError
} }

View File

@@ -1,16 +1,16 @@
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
class NotNullQuorum : CallQuorum { class NotNullQuorum : CallQuorum {
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
private var result: JsonRpcResponse? = null private var result: ChainResponse? = null
private var rpcError: JsonRpcError? = null private var rpcError: ChainCallError? = null
private val resolvers = ArrayList<Upstream>() private val resolvers = ArrayList<Upstream>()
private var allFailed = true private var allFailed = true
private val seenUpstreams = HashSet<String>() // just to prevent calling retry upstreams in FilteredApis private val seenUpstreams = HashSet<String>() // just to prevent calling retry upstreams in FilteredApis
@@ -20,7 +20,7 @@ class NotNullQuorum : CallQuorum {
override fun isFailed(): Boolean = rpcError != null override fun isFailed(): Boolean = rpcError != null
override fun record( override fun record(
response: JsonRpcResponse, response: ChainResponse,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
): Boolean { ): Boolean {
@@ -37,13 +37,13 @@ class NotNullQuorum : CallQuorum {
return false return false
} }
override fun record(error: JsonRpcException, signature: ResponseSigner.Signature?, upstream: Upstream) { override fun record(error: ChainException, signature: ResponseSigner.Signature?, upstream: Upstream) {
val upId = upstream.getId() val upId = upstream.getId()
if (seenUpstreams.contains(upId)) { if (seenUpstreams.contains(upId)) {
if (allFailed) { if (allFailed) {
rpcError = error.error rpcError = error.error
} else { } else {
result = JsonRpcResponse(Global.nullValue, null) result = ChainResponse(Global.nullValue, null)
} }
sig = signature sig = signature
} }
@@ -53,9 +53,9 @@ class NotNullQuorum : CallQuorum {
override fun getSignature(): ResponseSigner.Signature? = sig override fun getSignature(): ResponseSigner.Signature? = sig
override fun getResponse(): JsonRpcResponse? = result override fun getResponse(): ChainResponse? = result
override fun getError(): JsonRpcError? = rpcError override fun getError(): ChainCallError? = rpcError
override fun getResolvedBy(): Collection<Upstream> = resolvers override fun getResolvedBy(): Collection<Upstream> = resolvers

View File

@@ -20,15 +20,15 @@ import io.emeraldpay.dshackle.commons.API_READER
import io.emeraldpay.dshackle.commons.SPAN_NO_RESPONSE_MESSAGE import io.emeraldpay.dshackle.commons.SPAN_NO_RESPONSE_MESSAGE
import io.emeraldpay.dshackle.commons.SPAN_REQUEST_API_TYPE import io.emeraldpay.dshackle.commons.SPAN_REQUEST_API_TYPE
import io.emeraldpay.dshackle.commons.SPAN_REQUEST_UPSTREAM_ID import io.emeraldpay.dshackle.commons.SPAN_REQUEST_UPSTREAM_ID
import io.emeraldpay.dshackle.reader.RpcReader import io.emeraldpay.dshackle.reader.RequestReader
import io.emeraldpay.dshackle.reader.SpannedReader import io.emeraldpay.dshackle.reader.SpannedReader
import io.emeraldpay.dshackle.upstream.ApiSource import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.ChainCallUpstreamException
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcUpstreamException
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.cloud.sleuth.Tracer import org.springframework.cloud.sleuth.Tracer
@@ -45,22 +45,22 @@ import java.util.function.Function
/** /**
* Makes request with applying Quorum * Makes request with applying Quorum
*/ */
class QuorumRpcReader( class QuorumRequestReader(
private val apiControl: ApiSource, private val apiControl: ApiSource,
private val quorum: CallQuorum, private val quorum: CallQuorum,
signer: ResponseSigner?, signer: ResponseSigner?,
private val tracer: Tracer, private val tracer: Tracer,
) : RpcReader(signer) { ) : RequestReader(signer) {
companion object { companion object {
private val log = LoggerFactory.getLogger(QuorumRpcReader::class.java) private val log = LoggerFactory.getLogger(QuorumRequestReader::class.java)
} }
constructor(apiControl: ApiSource, quorum: CallQuorum, tracer: Tracer) : this(apiControl, quorum, null, tracer) constructor(apiControl: ApiSource, quorum: CallQuorum, tracer: Tracer) : this(apiControl, quorum, null, tracer)
override fun attempts(): AtomicInteger = apiControl.attempts() override fun attempts(): AtomicInteger = apiControl.attempts()
override fun read(key: JsonRpcRequest): Mono<Result> { override fun read(key: ChainRequest): Mono<Result> {
// needs at least one response, so start a request // needs at least one response, so start a request
apiControl.request(1) apiControl.request(1)
@@ -99,8 +99,8 @@ class QuorumRpcReader(
.transform(processResult(defaultResult)) .transform(processResult(defaultResult))
} }
private fun execute(key: JsonRpcRequest, retrySpec: reactor.util.retry.Retry): Function<Flux<Upstream>, Mono<CallQuorum>> { private fun execute(key: ChainRequest, retrySpec: reactor.util.retry.Retry): Function<Flux<Upstream>, Mono<CallQuorum>> {
val quorumReduce = BiFunction<CallQuorum, Tuple3<JsonRpcResponse, Optional<ResponseSigner.Signature>, Upstream>, CallQuorum> { res, a -> val quorumReduce = BiFunction<CallQuorum, Tuple3<ChainResponse, Optional<ResponseSigner.Signature>, Upstream>, CallQuorum> { res, a ->
if (res.record(a.t1, a.t2.orElse(null), a.t3)) { if (res.record(a.t1, a.t2.orElse(null), a.t3)) {
log.trace("Quorum is resolved for method ${key.method}") log.trace("Quorum is resolved for method ${key.method}")
apiControl.resolve() apiControl.resolve()
@@ -139,7 +139,7 @@ class QuorumRpcReader(
} }
} }
private fun callApi(api: Upstream, key: JsonRpcRequest): Mono<Tuple3<JsonRpcResponse, Optional<ResponseSigner.Signature>, Upstream>> { private fun callApi(api: Upstream, key: ChainRequest): Mono<Tuple3<ChainResponse, Optional<ResponseSigner.Signature>, Upstream>> {
val apiReader = api.getIngressReader() val apiReader = api.getIngressReader()
val spanParams = mapOf( val spanParams = mapOf(
SPAN_REQUEST_API_TYPE to apiReader.javaClass.name, SPAN_REQUEST_API_TYPE to apiReader.javaClass.name,
@@ -157,7 +157,7 @@ class QuorumRpcReader(
.map { Tuples.of(it.t1, it.t2, api) } .map { Tuples.of(it.t1, it.t2, api) }
} }
private fun withSignatureAndUpstream(api: Upstream, key: JsonRpcRequest, response: JsonRpcResponse): Function<Mono<ByteArray>, Mono<Tuple2<JsonRpcResponse, Optional<ResponseSigner.Signature>>>> { private fun withSignatureAndUpstream(api: Upstream, key: ChainRequest, response: ChainResponse): Function<Mono<ByteArray>, Mono<Tuple2<ChainResponse, Optional<ResponseSigner.Signature>>>> {
return Function { src -> return Function { src ->
src.map { src.map {
// TODO: do streaming signature // TODO: do streaming signature
@@ -171,11 +171,11 @@ class QuorumRpcReader(
} }
} }
private fun <T> withErrorResume(api: Upstream, key: JsonRpcRequest): Function<Mono<T>, Mono<T>> { private fun <T> withErrorResume(api: Upstream, key: ChainRequest): Function<Mono<T>, Mono<T>> {
return Function { src -> return Function { src ->
src.onErrorResume { err -> src.onErrorResume { err ->
val msgError = "Error during call upstream ${api.getId()} with method ${key.method}" val msgError = "Error during call upstream ${api.getId()} with method ${key.method}"
if (err is JsonRpcUpstreamException) { if (err is ChainCallUpstreamException) {
log.debug(msgError, err) log.debug(msgError, err)
} else { } else {
log.warn(msgError, err) log.warn(msgError, err)
@@ -184,12 +184,12 @@ class QuorumRpcReader(
// when the call failed with an error we want to notify the quorum because // when the call failed with an error we want to notify the quorum because
// it may use the error message or other details // it may use the error message or other details
// //
val cleanErr: JsonRpcException = getError(key, err) val cleanErr: ChainException = getError(key, err)
quorum.record(cleanErr, null, api) quorum.record(cleanErr, null, api)
// if it's failed after that, then we don't need more calls, stop api source // if it's failed after that, then we don't need more calls, stop api source
if (quorum.isFailed()) { if (quorum.isFailed()) {
val msgQuorumFailed = "Quorum is failed, stop api source. Upstream ${api.getId()}, method ${key.method}" val msgQuorumFailed = "Quorum is failed, stop api source. Upstream ${api.getId()}, method ${key.method}"
if (cleanErr is JsonRpcUpstreamException) { if (cleanErr is ChainCallUpstreamException) {
log.debug(msgQuorumFailed) log.debug(msgQuorumFailed)
} else { } else {
log.warn(msgQuorumFailed) log.warn(msgQuorumFailed)
@@ -205,7 +205,7 @@ class QuorumRpcReader(
} }
} }
private fun setupDefaultResult(key: JsonRpcRequest): Mono<Result> { private fun setupDefaultResult(key: ChainRequest): Mono<Result> {
return Mono.just(quorum).flatMap { q -> return Mono.just(quorum).flatMap { q ->
if (q.isFailed()) { if (q.isFailed()) {
val resolvedBy = resolvedBy()?.getId() val resolvedBy = resolvedBy()?.getId()

View File

@@ -17,11 +17,11 @@
package io.emeraldpay.dshackle.quorum package io.emeraldpay.dshackle.quorum
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
@@ -30,7 +30,7 @@ abstract class ValueAwareQuorum<T>(
) : CallQuorum { ) : CallQuorum {
private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java) private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java)
private var rpcError: JsonRpcError? = null private var rpcError: ChainCallError? = null
protected val resolvers = ArrayList<Upstream>() protected val resolvers = ArrayList<Upstream>()
fun extractValue(response: ByteArray, clazz: Class<T>): T? { fun extractValue(response: ByteArray, clazz: Class<T>): T? {
@@ -38,7 +38,7 @@ abstract class ValueAwareQuorum<T>(
} }
override fun record( override fun record(
response: JsonRpcResponse, response: ChainResponse,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
): Boolean { ): Boolean {
@@ -58,7 +58,7 @@ abstract class ValueAwareQuorum<T>(
} }
override fun record( override fun record(
error: JsonRpcException, error: ChainException,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
) { ) {
@@ -67,7 +67,7 @@ abstract class ValueAwareQuorum<T>(
} }
abstract fun recordValue( abstract fun recordValue(
response: JsonRpcResponse, response: ChainResponse,
responseValue: T?, responseValue: T?,
signature: ResponseSigner.Signature?, signature: ResponseSigner.Signature?,
upstream: Upstream, upstream: Upstream,
@@ -79,7 +79,7 @@ abstract class ValueAwareQuorum<T>(
upstream: Upstream, upstream: Upstream,
) )
override fun getError(): JsonRpcError? { override fun getError(): ChainCallError? {
return rpcError return rpcError
} }

View File

@@ -3,11 +3,11 @@ package io.emeraldpay.dshackle.reader
import io.emeraldpay.dshackle.commons.BROADCAST_READER import io.emeraldpay.dshackle.commons.BROADCAST_READER
import io.emeraldpay.dshackle.commons.SPAN_REQUEST_UPSTREAM_ID import io.emeraldpay.dshackle.commons.SPAN_REQUEST_UPSTREAM_ID
import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.cloud.sleuth.Tracer import org.springframework.cloud.sleuth.Tracer
@@ -21,7 +21,7 @@ class BroadcastReader(
signer: ResponseSigner?, signer: ResponseSigner?,
private val quorum: CallQuorum, private val quorum: CallQuorum,
private val tracer: Tracer, private val tracer: Tracer,
) : RpcReader(signer) { ) : RequestReader(signer) {
private val internalMatcher = Selector.MultiMatcher( private val internalMatcher = Selector.MultiMatcher(
listOf(Selector.AvailabilityMatcher(), matcher), listOf(Selector.AvailabilityMatcher(), matcher),
) )
@@ -34,7 +34,7 @@ class BroadcastReader(
return AtomicInteger(1) return AtomicInteger(1)
} }
override fun read(key: JsonRpcRequest): Mono<Result> { override fun read(key: ChainRequest): Mono<Result> {
return Flux.fromIterable(upstreams) return Flux.fromIterable(upstreams)
.filter { internalMatcher.matches(it) } .filter { internalMatcher.matches(it) }
.flatMap { up -> .flatMap { up ->
@@ -44,7 +44,7 @@ class BroadcastReader(
val sig = getSignature(key, it.jsonRpcResponse, it.upstream.getId()) val sig = getSignature(key, it.jsonRpcResponse, it.upstream.getId())
quorum.record(it.jsonRpcResponse, sig, it.upstream) quorum.record(it.jsonRpcResponse, sig, it.upstream)
} else { } else {
val err = JsonRpcException(JsonRpcResponse.NumberId(key.id), it.jsonRpcResponse.error!!, it.upstream.getId()) val err = ChainException(ChainResponse.NumberId(key.id), it.jsonRpcResponse.error!!, it.upstream.getId())
quorum.record(err, null, it.upstream) quorum.record(err, null, it.upstream)
} }
quorum quorum
@@ -69,7 +69,7 @@ class BroadcastReader(
} }
private fun execute( private fun execute(
key: JsonRpcRequest, key: ChainRequest,
upstream: Upstream, upstream: Upstream,
): Mono<BroadcastResponse> = ): Mono<BroadcastResponse> =
SpannedReader( SpannedReader(
@@ -83,12 +83,12 @@ class BroadcastReader(
.onErrorResume { .onErrorResume {
log.warn("Error during execution ${key.method} from upstream ${upstream.getId()} with message - ${it.message}") log.warn("Error during execution ${key.method} from upstream ${upstream.getId()} with message - ${it.message}")
Mono.just( Mono.just(
BroadcastResponse(JsonRpcResponse(null, getError(key, it).error), upstream), BroadcastResponse(ChainResponse(null, getError(key, it).error), upstream),
) )
} }
private class BroadcastResponse( private class BroadcastResponse(
val jsonRpcResponse: JsonRpcResponse, val jsonRpcResponse: ChainResponse,
val upstream: Upstream, val upstream: Upstream,
) )
} }

View File

@@ -0,0 +1,6 @@
package io.emeraldpay.dshackle.reader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
typealias ChainReader = Reader<ChainRequest, ChainResponse>

View File

@@ -1,10 +0,0 @@
package io.emeraldpay.dshackle.reader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
typealias JsonRpcReader = Reader<JsonRpcRequest, JsonRpcResponse>
interface JsonRpcHttpReader : JsonRpcReader {
fun onStop()
}

View File

@@ -3,42 +3,42 @@ package io.emeraldpay.dshackle.reader
import io.emeraldpay.dshackle.quorum.BroadcastQuorum import io.emeraldpay.dshackle.quorum.BroadcastQuorum
import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.quorum.MaximumValueQuorum import io.emeraldpay.dshackle.quorum.MaximumValueQuorum
import io.emeraldpay.dshackle.quorum.QuorumRpcReader import io.emeraldpay.dshackle.quorum.QuorumRequestReader
import io.emeraldpay.dshackle.reader.RpcReader.Result import io.emeraldpay.dshackle.reader.RequestReader.Result
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.stream.Chunk
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.dshackle.upstream.stream.Chunk
import org.springframework.cloud.sleuth.Tracer import org.springframework.cloud.sleuth.Tracer
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
abstract class RpcReader( abstract class RequestReader(
private val signer: ResponseSigner?, private val signer: ResponseSigner?,
) : Reader<JsonRpcRequest, Result> { ) : Reader<ChainRequest, Result> {
abstract fun attempts(): AtomicInteger abstract fun attempts(): AtomicInteger
protected fun getError(key: JsonRpcRequest, err: Throwable) = protected fun getError(key: ChainRequest, err: Throwable) =
when (err) { when (err) {
is RpcException -> JsonRpcException.from(err) is RpcException -> ChainException.from(err)
is JsonRpcException -> err is ChainException -> err
else -> JsonRpcException( else -> ChainException(
JsonRpcResponse.NumberId(key.id), ChainResponse.NumberId(key.id),
JsonRpcError(-32603, "Unhandled internal error: ${err.javaClass}: ${err.message}"), ChainCallError(-32603, "Unhandled internal error: ${err.javaClass}: ${err.message}"),
) )
} }
protected fun handleError(error: JsonRpcError?, id: Int, resolvedBy: String?) = protected fun handleError(error: ChainCallError?, id: Int, resolvedBy: String?) =
error?.asException(JsonRpcResponse.NumberId(id), resolvedBy) error?.asException(ChainResponse.NumberId(id), resolvedBy)
?: JsonRpcException(JsonRpcResponse.NumberId(id), JsonRpcError(-32603, "Unhandled Upstream error"), resolvedBy) ?: ChainException(ChainResponse.NumberId(id), ChainCallError(-32603, "Unhandled Upstream error"), resolvedBy)
protected fun getSignature(key: JsonRpcRequest, response: JsonRpcResponse, upstreamId: String) = protected fun getSignature(key: ChainRequest, response: ChainResponse, upstreamId: String) =
response.providedSignature response.providedSignature
?: if (key.nonce != null) { ?: if (key.nonce != null) {
signer?.sign(key.nonce, response.getResult(), upstreamId) signer?.sign(key.nonce, response.getResult(), upstreamId)
@@ -55,29 +55,28 @@ abstract class RpcReader(
) )
} }
interface RpcReaderFactory { interface RequestReaderFactory {
companion object { companion object {
fun default(): RpcReaderFactory { fun default(): RequestReaderFactory {
return Default() return Default()
} }
} }
fun create(data: RpcReaderData): RpcReader fun create(data: ReaderData): RequestReader
class Default : RpcReaderFactory { class Default : RequestReaderFactory {
override fun create(data: RpcReaderData): RpcReader { override fun create(data: ReaderData): RequestReader {
if (data.quorum is MaximumValueQuorum || data.quorum is BroadcastQuorum) { if (data.quorum is MaximumValueQuorum || data.quorum is BroadcastQuorum) {
return BroadcastReader(data.multistream.getAll(), data.matcher, data.signer, data.quorum, data.tracer) return BroadcastReader(data.multistream.getAll(), data.matcher, data.signer, data.quorum, data.tracer)
} }
val apis = data.multistream.getApiSource(data.matcher) val apis = data.multistream.getApiSource(data.matcher)
return QuorumRpcReader(apis, data.quorum, data.signer, data.tracer) return QuorumRequestReader(apis, data.quorum, data.signer, data.tracer)
} }
} }
data class RpcReaderData( data class ReaderData(
val multistream: Multistream, val multistream: Multistream,
val method: String,
val matcher: Selector.Matcher, val matcher: Selector.Matcher,
val quorum: CallQuorum, val quorum: CallQuorum,
val signer: ResponseSigner?, val signer: ResponseSigner?,

View File

@@ -7,7 +7,7 @@ import io.emeraldpay.dshackle.commons.SPAN_REQUEST_CANCELLED
import io.emeraldpay.dshackle.commons.SPAN_REQUEST_INFO import io.emeraldpay.dshackle.commons.SPAN_REQUEST_INFO
import io.emeraldpay.dshackle.commons.SPAN_STATUS_MESSAGE import io.emeraldpay.dshackle.commons.SPAN_STATUS_MESSAGE
import io.emeraldpay.dshackle.data.HashId import io.emeraldpay.dshackle.data.HashId
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.ChainRequest
import org.springframework.cloud.sleuth.Tracer import org.springframework.cloud.sleuth.Tracer
import org.springframework.cloud.sleuth.instrument.reactor.ReactorSleuth import org.springframework.cloud.sleuth.instrument.reactor.ReactorSleuth
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -50,7 +50,7 @@ class SpannedReader<K, D>(
private fun extractInfoFromKey(key: K): String? { private fun extractInfoFromKey(key: K): String? {
return when (key) { return when (key) {
is JsonRpcRequest -> "method: ${key.method}" is ChainRequest -> "method: ${key.method}"
is HashId, Long -> "params: $key" is HashId, Long -> "params: $key"
else -> null else -> null
} }

View File

@@ -32,11 +32,14 @@ import io.emeraldpay.dshackle.commons.SPAN_STATUS_MESSAGE
import io.emeraldpay.dshackle.config.MainConfig import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
import io.emeraldpay.dshackle.reader.RpcReader import io.emeraldpay.dshackle.reader.RequestReader
import io.emeraldpay.dshackle.reader.RpcReaderFactory import io.emeraldpay.dshackle.reader.RequestReaderFactory
import io.emeraldpay.dshackle.reader.RpcReaderFactory.RpcReaderData import io.emeraldpay.dshackle.reader.RequestReaderFactory.ReaderData
import io.emeraldpay.dshackle.reader.SpannedReader import io.emeraldpay.dshackle.reader.SpannedReader
import io.emeraldpay.dshackle.upstream.ApiSource import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
@@ -44,13 +47,11 @@ import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.CallParams import io.emeraldpay.dshackle.upstream.rpcclient.CallParams
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import io.emeraldpay.dshackle.upstream.rpcclient.ObjectParams import io.emeraldpay.dshackle.upstream.rpcclient.ObjectParams
import io.emeraldpay.dshackle.upstream.rpcclient.stream.Chunk import io.emeraldpay.dshackle.upstream.rpcclient.RestParams
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.dshackle.upstream.stream.Chunk
import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Metrics
import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.StringUtils
import org.reactivestreams.Publisher import org.reactivestreams.Publisher
@@ -77,7 +78,7 @@ open class NativeCall(
private val passthrough = config.passthrough private val passthrough = config.passthrough
var rpcReaderFactory: RpcReaderFactory = RpcReaderFactory.default() var requestReaderFactory: RequestReaderFactory = RequestReaderFactory.default()
open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> { open fun nativeCall(requestMono: Mono<BlockchainOuterClass.NativeCallRequest>): Flux<BlockchainOuterClass.NativeCallReplyItem> {
return nativeCallResult(requestMono) return nativeCallResult(requestMono)
@@ -199,9 +200,8 @@ open class NativeCall(
} }
} }
fun parseParams(it: ValidCallContext<RawCallDetails>): ValidCallContext<ParsedCallDetails> { fun parseParams(it: ValidCallContext<ParsedCallDetails>): ValidCallContext<ParsedCallDetails> {
val rawParams = extractParams(it.payload.params) val params = it.requestDecorator.processRequest(it.payload.params)
val params = it.requestDecorator.processRequest(rawParams)
return it.withPayload(ParsedCallDetails(it.payload.method, params)) return it.withPayload(ParsedCallDetails(it.payload.method, params))
} }
@@ -304,7 +304,11 @@ open class NativeCall(
val requestId = requestItem.requestId val requestId = requestItem.requestId
val requestCount = request.itemsCount val requestCount = request.itemsCount
val method = requestItem.method val method = requestItem.method
val params = requestItem.payload.toStringUtf8() val params = if (requestItem.hasPayload()) {
requestItem.payload.toStringUtf8()
} else {
""
}
val availableMethods = upstream.getMethods() val availableMethods = upstream.getMethods()
if (!availableMethods.isAvailable(method)) { if (!availableMethods.isAvailable(method)) {
@@ -314,7 +318,7 @@ open class NativeCall(
CallError( CallError(
requestItem.id, requestItem.id,
errorMessage, errorMessage,
JsonRpcError(RpcResponseError.CODE_METHOD_NOT_EXIST, errorMessage), ChainCallError(RpcResponseError.CODE_METHOD_NOT_EXIST, errorMessage),
null, null,
), ),
requestId, requestId,
@@ -356,7 +360,7 @@ open class NativeCall(
upstream, upstream,
matcher.build(), matcher.build(),
callQuorum, callQuorum,
RawCallDetails(method, params), parsedCallDetails(requestItem),
requestDecorator, requestDecorator,
resultDecorator, resultDecorator,
selector, selector,
@@ -367,6 +371,24 @@ open class NativeCall(
} }
} }
private fun parsedCallDetails(item: BlockchainOuterClass.NativeCallItem): ParsedCallDetails {
return if (item.hasPayload()) {
ParsedCallDetails(item.method, extractParams(item.payload.toStringUtf8()))
} else if (item.hasRestData()) {
ParsedCallDetails(
item.method,
RestParams(
item.restData.headersList.map { it.key to it.value }.toMap(),
item.restData.queryParamsList.map { it.key to it.value }.toMap(),
item.restData.pathParamsList,
item.restData.payload.toByteArray(),
),
)
} else {
throw IllegalStateException("Wrong payload type")
}
}
private fun getRequestDecorator(method: String): RequestDecorator = private fun getRequestDecorator(method: String): RequestDecorator =
if (method in DefaultEthereumMethods.withFilterIdMethods) { if (method in DefaultEthereumMethods.withFilterIdMethods) {
WithFilterIdDecorator() WithFilterIdDecorator()
@@ -381,7 +403,7 @@ open class NativeCall(
return ctx.upstream.getLocalReader() return ctx.upstream.getLocalReader()
.flatMap { api -> .flatMap { api ->
SpannedReader(api, tracer, LOCAL_READER) SpannedReader(api, tracer, LOCAL_READER)
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params, ctx.nonce, ctx.forwardedSelector)) .read(ctx.payload.toChainRequest(ctx.nonce, ctx.forwardedSelector, false))
.map { .map {
val result = it.getResult() val result = it.getResult()
val upstreamId = it.providedUpstreamId ?: ctx.upstream.getId() val upstreamId = it.providedUpstreamId ?: ctx.upstream.getId()
@@ -405,13 +427,13 @@ open class NativeCall(
if (!ctx.upstream.getMethods().isCallable(ctx.payload.method)) { if (!ctx.upstream.getMethods().isCallable(ctx.payload.method)) {
return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))
} }
val reader = rpcReaderFactory.create( val reader = requestReaderFactory.create(
RpcReaderData(ctx.upstream, ctx.payload.method, ctx.matcher, ctx.callQuorum, signer, tracer), ReaderData(ctx.upstream, ctx.matcher, ctx.callQuorum, signer, tracer),
) )
val counter = reader.attempts() val counter = reader.attempts()
return SpannedReader(reader, tracer, RPC_READER) return SpannedReader(reader, tracer, RPC_READER)
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params, ctx.nonce, ctx.forwardedSelector, ctx.streamRequest)) .read(ctx.payload.toChainRequest(ctx.nonce, ctx.forwardedSelector, ctx.streamRequest))
.map { .map {
val upId = it.resolvedBy?.getId() ?: ctx.upstream.getId() val upId = it.resolvedBy?.getId() ?: ctx.upstream.getId()
if (it.stream == null) { if (it.stream == null) {
@@ -497,11 +519,11 @@ open class NativeCall(
} }
interface ResultDecorator { interface ResultDecorator {
fun processResult(result: RpcReader.Result): ByteArray fun processResult(result: RequestReader.Result): ByteArray
} }
open class NoneResultDecorator : ResultDecorator { open class NoneResultDecorator : ResultDecorator {
override fun processResult(result: RpcReader.Result): ByteArray = result.value override fun processResult(result: RequestReader.Result): ByteArray = result.value
} }
open class CreateFilterDecorator : ResultDecorator { open class CreateFilterDecorator : ResultDecorator {
@@ -509,7 +531,7 @@ open class NativeCall(
companion object { companion object {
const val quoteCode = '"'.code.toByte() const val quoteCode = '"'.code.toByte()
} }
override fun processResult(result: RpcReader.Result): ByteArray { override fun processResult(result: RequestReader.Result): ByteArray {
val bytes = result.value val bytes = result.value
if (bytes.last() == quoteCode && result.resolvedBy != null) { if (bytes.last() == quoteCode && result.resolvedBy != null) {
val suffix = result.resolvedBy.nodeId() val suffix = result.resolvedBy.nodeId()
@@ -624,7 +646,7 @@ open class NativeCall(
data class CallError( data class CallError(
val id: Int, val id: Int,
val message: String, val message: String,
val upstreamError: JsonRpcError?, val upstreamError: ChainCallError?,
val data: String?, val data: String?,
val upstreamId: String? = null, val upstreamId: String? = null,
) { ) {
@@ -644,7 +666,7 @@ open class NativeCall(
} }
fun from(t: Throwable): CallError { fun from(t: Throwable): CallError {
return when (t) { return when (t) {
is JsonRpcException -> CallError(t.error.code, t.error.message, t.error, getDataAsSting(t.error.details), t.upstreamId) is ChainException -> CallError(t.error.code, t.error.message, t.error, getDataAsSting(t.error.details), t.upstreamId)
is RpcException -> CallError(t.code, t.rpcMessage, null, getDataAsSting(t.details)) is RpcException -> CallError(t.code, t.rpcMessage, null, getDataAsSting(t.details))
is CallFailure -> CallError(t.id, t.reason.message ?: "Upstream Error", null, null) is CallFailure -> CallError(t.id, t.reason.message ?: "Upstream Error", null, null)
else -> { else -> {
@@ -704,6 +726,13 @@ open class NativeCall(
} }
} }
class RawCallDetails(val method: String, val params: String) class ParsedCallDetails(val method: String, val params: CallParams) {
class ParsedCallDetails(val method: String, val params: CallParams) fun toChainRequest(
nonce: Long?,
selector: BlockchainOuterClass.Selector?,
streamRequest: Boolean,
): ChainRequest {
return ChainRequest(method, params, nonce, selector, streamRequest)
}
}
} }

View File

@@ -1,16 +1,18 @@
package io.emeraldpay.dshackle.startup.configure package io.emeraldpay.dshackle.startup.configure
import io.emeraldpay.dshackle.ApiType
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.HttpRpcFactory import io.emeraldpay.dshackle.upstream.HttpFactory
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory
import org.springframework.stereotype.Component
import java.net.URI import java.net.URI
interface ConnectorFactoryCreator { interface ConnectorFactoryCreator {
fun createConnectorFactoryCreator( fun createConnectorFactory(
id: String, id: String,
conn: UpstreamsConfig.RpcConnection, conn: UpstreamsConfig.RpcConnection,
chain: Chain, chain: Chain,
@@ -19,5 +21,18 @@ interface ConnectorFactoryCreator {
chainsConf: ChainsConfig.ChainConfig, chainsConf: ChainsConfig.ChainConfig,
): ConnectorFactory? ): ConnectorFactory?
fun buildHttpFactory(conn: UpstreamsConfig.HttpEndpoint?, urls: ArrayList<URI>? = null): HttpRpcFactory? fun buildHttpFactory(conn: UpstreamsConfig.HttpEndpoint?, urls: ArrayList<URI>? = null): HttpFactory?
}
@Component
class ConnectorFactoryCreatorResolver(
private val genericConnectorFactoryCreator: GenericConnectorFactoryCreator,
private val restConnectorFactoryCreator: RestConnectorFactoryCreator,
) {
fun resolve(chain: Chain): ConnectorFactoryCreator {
if (chain.type.apiType == ApiType.REST) {
return restConnectorFactoryCreator
}
return genericConnectorFactoryCreator
}
} }

View File

@@ -13,8 +13,8 @@ class EthereumUpstreamCreator(
chainsConfig: ChainsConfig, chainsConfig: ChainsConfig,
indexConfig: IndexConfig, indexConfig: IndexConfig,
callTargets: CallTargetsHolder, callTargets: CallTargetsHolder,
genericConnectorFactoryCreator: ConnectorFactoryCreator, connectorFactoryCreatorResolver: ConnectorFactoryCreatorResolver,
) : GenericUpstreamCreator(chainsConfig, indexConfig, callTargets, genericConnectorFactoryCreator) { ) : GenericUpstreamCreator(chainsConfig, indexConfig, callTargets, connectorFactoryCreatorResolver) {
override fun createUpstream( override fun createUpstream(
upstreamsConfig: UpstreamsConfig.Upstream<*>, upstreamsConfig: UpstreamsConfig.Upstream<*>,

View File

@@ -4,8 +4,9 @@ import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.BasicHttpFactory
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.HttpRpcFactory import io.emeraldpay.dshackle.upstream.HttpFactory
import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionFactory import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionFactory
import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPoolFactory import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPoolFactory
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
@@ -17,16 +18,16 @@ import reactor.core.scheduler.Scheduler
import java.net.URI import java.net.URI
@Component @Component
class GenericConnectorFactoryCreator( open class GenericConnectorFactoryCreator(
private val fileResolver: FileResolver, private val fileResolver: FileResolver,
private val wsConnectionResubscribeScheduler: Scheduler, private val wsConnectionResubscribeScheduler: Scheduler,
private val headScheduler: Scheduler, private val headScheduler: Scheduler,
private val wsScheduler: Scheduler, private val wsScheduler: Scheduler,
private val headLivenessScheduler: Scheduler, private val headLivenessScheduler: Scheduler,
) : ConnectorFactoryCreator { ) : ConnectorFactoryCreator {
private val log = LoggerFactory.getLogger(this::class.java) protected val log = LoggerFactory.getLogger(this::class.java)
override fun createConnectorFactoryCreator( override fun createConnectorFactory(
id: String, id: String,
conn: UpstreamsConfig.RpcConnection, conn: UpstreamsConfig.RpcConnection,
chain: Chain, chain: Chain,
@@ -57,7 +58,7 @@ class GenericConnectorFactoryCreator(
return connectorFactory return connectorFactory
} }
override fun buildHttpFactory(conn: UpstreamsConfig.HttpEndpoint?, urls: ArrayList<URI>?): HttpRpcFactory? { override fun buildHttpFactory(conn: UpstreamsConfig.HttpEndpoint?, urls: ArrayList<URI>?): HttpFactory? {
return conn?.let { endpoint -> return conn?.let { endpoint ->
val tls = conn.tls?.let { tls -> val tls = conn.tls?.let { tls ->
tls.ca?.let { ca -> tls.ca?.let { ca ->
@@ -65,7 +66,7 @@ class GenericConnectorFactoryCreator(
} }
} }
urls?.add(endpoint.url) urls?.add(endpoint.url)
HttpRpcFactory(endpoint.url.toString(), conn.basicAuth, tls) BasicHttpFactory(endpoint.url.toString(), conn.basicAuth, tls)
} }
} }

View File

@@ -21,7 +21,7 @@ open class GenericUpstreamCreator(
chainsConfig: ChainsConfig, chainsConfig: ChainsConfig,
indexConfig: IndexConfig, indexConfig: IndexConfig,
callTargets: CallTargetsHolder, callTargets: CallTargetsHolder,
private val genericConnectorFactoryCreator: ConnectorFactoryCreator, private val connectorFactoryCreatorResolver: ConnectorFactoryCreatorResolver,
) : UpstreamCreator(chainsConfig, indexConfig, callTargets) { ) : UpstreamCreator(chainsConfig, indexConfig, callTargets) {
private val hashes: MutableMap<Byte, Boolean> = HashMap() private val hashes: MutableMap<Byte, Boolean> = HashMap()
@@ -58,7 +58,7 @@ open class GenericUpstreamCreator(
val cs = ChainSpecificRegistry.resolve(chain) val cs = ChainSpecificRegistry.resolve(chain)
val connectorFactory = genericConnectorFactoryCreator.createConnectorFactoryCreator( val connectorFactory = connectorFactoryCreatorResolver.resolve(chain).createConnectorFactory(
config.id!!, config.id!!,
connection, connection,
chain, chain,

View File

@@ -0,0 +1,55 @@
package io.emeraldpay.dshackle.startup.configure
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.generic.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.generic.connectors.RestConnectorFactory
import org.springframework.stereotype.Component
import reactor.core.scheduler.Scheduler
import reactor.core.scheduler.Schedulers
import java.net.URI
@Component
class RestConnectorFactoryCreator(
fileResolver: FileResolver,
private val headScheduler: Scheduler,
private val headLivenessScheduler: Scheduler,
) : GenericConnectorFactoryCreator(
fileResolver,
Schedulers.single(),
headScheduler,
Schedulers.single(),
headLivenessScheduler,
) {
override fun createConnectorFactory(
id: String,
conn: UpstreamsConfig.RpcConnection,
chain: Chain,
forkChoice: ForkChoice,
blockValidator: BlockValidator,
chainsConf: ChainsConfig.ChainConfig,
): ConnectorFactory? {
val urls = ArrayList<URI>()
val httpFactory = buildHttpFactory(conn.rpc, urls)
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
val connectorFactory =
RestConnectorFactory(
httpFactory,
forkChoice,
blockValidator,
headScheduler,
headLivenessScheduler,
chainsConf.expectedBlockTime,
)
if (!connectorFactory.isValid()) {
log.warn("Upstream configuration is invalid - no http endpoint")
return null
}
return connectorFactory
}
}

View File

@@ -0,0 +1,94 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.foundation.ChainOptions
import reactor.core.publisher.Mono
import java.util.concurrent.TimeoutException
import java.util.function.Supplier
abstract class BasicEthUpstreamValidator(
upstream: Upstream,
options: ChainOptions.Options,
) : UpstreamValidator(upstream, options) {
override fun validate(): Mono<UpstreamAvailability> {
return Mono.zip(
validatorFunctions().map { it.get() },
) { a -> a.map { it as UpstreamAvailability } }
.map(::resolve)
.defaultIfEmpty(UpstreamAvailability.UNAVAILABLE)
.onErrorResume {
log.error("Error during upstream validation for ${upstream.getId()}", it)
Mono.just(UpstreamAvailability.UNAVAILABLE)
}
}
protected fun validateSyncing(): Mono<UpstreamAvailability> {
if (!options.validateSyncing) {
return Mono.just(UpstreamAvailability.OK)
}
val validateSyncingRequest = validateSyncingRequest()
return upstream.getIngressReader()
.read(validateSyncingRequest.request)
.flatMap(ChainResponse::requireResult)
.map { validateSyncingRequest.mapper(it) }
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for ${validateSyncingRequest.request.method} from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for Syncing"))),
)
.map {
upstream.getHead().onSyncingNode(it)
if (it) {
UpstreamAvailability.SYNCING
} else {
UpstreamAvailability.OK
}
}
.doOnError { err -> log.error("Error during syncing validation for ${upstream.getId()}", err) }
.onErrorReturn(UpstreamAvailability.UNAVAILABLE)
}
protected fun validatePeers(): Mono<UpstreamAvailability> {
if (!options.validatePeers || options.minPeers == 0) {
return Mono.just(UpstreamAvailability.OK)
}
val validatePeersRequest = validatePeersRequest()
return upstream
.getIngressReader()
.read(validatePeersRequest.request)
.flatMap(ChainResponse::checkError)
.map { validatePeersRequest.mapper(it) }
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for ${validatePeersRequest.request.method} from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for Peers"))),
)
.map { count ->
val minPeers = options.minPeers
if (count < minPeers) {
UpstreamAvailability.IMMATURE
} else {
UpstreamAvailability.OK
}
}
.doOnError { err -> log.error("Error during peer count validation for ${upstream.getId()}", err) }
.onErrorReturn(UpstreamAvailability.UNAVAILABLE)
}
protected abstract fun validateSyncingRequest(): ValidateSyncingRequest
protected abstract fun validatePeersRequest(): ValidatePeersRequest
protected abstract fun validatorFunctions(): List<Supplier<Mono<UpstreamAvailability>>>
data class ValidateSyncingRequest(
val request: ChainRequest,
val mapper: (ByteArray) -> Boolean,
)
data class ValidatePeersRequest(
val request: ChainRequest,
val mapper: (ChainResponse) -> Int,
)
}

View File

@@ -1,28 +1,28 @@
package io.emeraldpay.dshackle.upstream package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.ApiType
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.AuthConfig import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.reader.JsonRpcHttpReader import io.emeraldpay.dshackle.upstream.restclient.RestHttpReader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpReader
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer import io.micrometer.core.instrument.Timer
open class HttpRpcFactory( class BasicHttpFactory(
private val url: String, private val url: String,
private val basicAuth: AuthConfig.ClientBasicAuth?, private val basicAuth: AuthConfig.ClientBasicAuth?,
private val tls: ByteArray?, private val tls: ByteArray?,
) : HttpFactory { ) : HttpFactory {
override fun create(id: String?, chain: Chain): JsonRpcHttpReader { override fun create(id: String?, chain: Chain): HttpReader {
val metricsTags = listOf( val metricsTags = listOf(
// "unknown" is not supposed to happen // "unknown" is not supposed to happen
Tag.of("upstream", id ?: "unknown"), Tag.of("upstream", id ?: "unknown"),
// UNSPECIFIED shouldn't happen too // UNSPECIFIED shouldn't happen too
Tag.of("chain", chain.chainCode), Tag.of("chain", chain.chainCode),
) )
val metrics = RpcMetrics( val metrics = RequestMetrics(
Timer.builder("upstream.rpc.conn") Timer.builder("upstream.rpc.conn")
.description("Request time through a HTTP JSON RPC connection") .description("Request time through a HTTP JSON RPC connection")
.tags(metricsTags) .tags(metricsTags)
@@ -33,11 +33,10 @@ open class HttpRpcFactory(
.tags(metricsTags) .tags(metricsTags)
.register(Metrics.globalRegistry), .register(Metrics.globalRegistry),
) )
return JsonRpcHttpClient(
url, if (chain.type.apiType == ApiType.REST) {
metrics, return RestHttpReader(url, metrics, basicAuth, tls)
basicAuth, }
tls, return JsonRpcHttpReader(url, metrics, basicAuth, tls)
)
} }
} }

View File

@@ -2,6 +2,7 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.BlockchainType.BITCOIN import io.emeraldpay.dshackle.BlockchainType.BITCOIN
import io.emeraldpay.dshackle.BlockchainType.ETHEREUM import io.emeraldpay.dshackle.BlockchainType.ETHEREUM
import io.emeraldpay.dshackle.BlockchainType.ETHEREUM_BEACON_CHAIN
import io.emeraldpay.dshackle.BlockchainType.NEAR import io.emeraldpay.dshackle.BlockchainType.NEAR
import io.emeraldpay.dshackle.BlockchainType.POLKADOT import io.emeraldpay.dshackle.BlockchainType.POLKADOT
import io.emeraldpay.dshackle.BlockchainType.SOLANA import io.emeraldpay.dshackle.BlockchainType.SOLANA
@@ -9,6 +10,7 @@ import io.emeraldpay.dshackle.BlockchainType.STARKNET
import io.emeraldpay.dshackle.BlockchainType.UNKNOWN import io.emeraldpay.dshackle.BlockchainType.UNKNOWN
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultBeaconChainMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultPolkadotMethods import io.emeraldpay.dshackle.upstream.calls.DefaultPolkadotMethods
@@ -31,6 +33,7 @@ class CallTargetsHolder {
POLKADOT -> DefaultPolkadotMethods() POLKADOT -> DefaultPolkadotMethods()
SOLANA -> DefaultSolanaMethods() SOLANA -> DefaultSolanaMethods()
NEAR -> DefaultNearMethods() NEAR -> DefaultNearMethods()
ETHEREUM_BEACON_CHAIN -> DefaultBeaconChainMethods()
UNKNOWN -> throw IllegalArgumentException("unknown chain") UNKNOWN -> throw IllegalArgumentException("unknown chain")
} }
callTargets[chain] = created callTargets[chain] = created

View File

@@ -13,18 +13,18 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
data class JsonRpcError(val code: Int, val message: String, val details: Any?) { data class ChainCallError(val code: Int, val message: String, val details: Any?) {
constructor(code: Int, message: String) : this(code, message, null) constructor(code: Int, message: String) : this(code, message, null)
companion object { companion object {
@JvmStatic @JvmStatic
fun from(err: RpcException): JsonRpcError { fun from(err: RpcException): ChainCallError {
return JsonRpcError( return ChainCallError(
err.code, err.code,
err.rpcMessage, err.rpcMessage,
err.details, err.details,
@@ -32,11 +32,11 @@ data class JsonRpcError(val code: Int, val message: String, val details: Any?) {
} }
} }
fun asException(id: JsonRpcResponse.Id?): JsonRpcException { fun asException(id: ChainResponse.Id?): ChainException {
return JsonRpcUpstreamException(id ?: JsonRpcResponse.NumberId(-1), this) return ChainCallUpstreamException(id ?: ChainResponse.NumberId(-1), this)
} }
fun asException(id: JsonRpcResponse.Id?, upstreamId: String?): JsonRpcException { fun asException(id: ChainResponse.Id?, upstreamId: String?): ChainException {
return JsonRpcException(id ?: JsonRpcResponse.NumberId(-1), this, upstreamId, false) return ChainException(id ?: ChainResponse.NumberId(-1), this, upstreamId, false)
} }
} }

View File

@@ -0,0 +1,6 @@
package io.emeraldpay.dshackle.upstream
class ChainCallUpstreamException(
id: ChainResponse.Id,
error: ChainCallError,
) : ChainException(id, error, null, false)

View File

@@ -13,34 +13,34 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
open class JsonRpcException( open class ChainException(
val id: JsonRpcResponse.Id, val id: ChainResponse.Id,
val error: JsonRpcError, val error: ChainCallError,
val upstreamId: String? = null, val upstreamId: String? = null,
writableStackTrace: Boolean = true, writableStackTrace: Boolean = true,
cause: Throwable? = null, cause: Throwable? = null,
) : Exception(error.message, cause, true, writableStackTrace) { ) : Exception(error.message, cause, true, writableStackTrace) {
constructor(id: Int, message: String) : this(JsonRpcResponse.NumberId(id), JsonRpcError(-32005, message)) constructor(id: Int, message: String) : this(ChainResponse.NumberId(id), ChainCallError(-32005, message))
constructor(id: Int, message: String, cause: Throwable) : this(JsonRpcResponse.NumberId(id), JsonRpcError(-32005, message), cause = cause) constructor(id: Int, message: String, cause: Throwable) : this(ChainResponse.NumberId(id), ChainCallError(-32005, message), cause = cause)
companion object { companion object {
fun from(err: RpcException): JsonRpcException { fun from(err: RpcException): ChainException {
val id = err.details?.let { val id = err.details?.let {
if (it is JsonRpcResponse.Id) { if (it is ChainResponse.Id) {
it it
} else { } else {
JsonRpcResponse.NumberId(-3) ChainResponse.NumberId(-3)
} }
} ?: JsonRpcResponse.NumberId(-4) } ?: ChainResponse.NumberId(-4)
return JsonRpcException( return ChainException(
id, id,
JsonRpcError.from(err), ChainCallError.from(err),
) )
} }
} }

View File

@@ -13,16 +13,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.DeserializationContext
import com.fasterxml.jackson.databind.JsonDeserializer import com.fasterxml.jackson.databind.JsonDeserializer
import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.JsonNode
import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.upstream.rpcclient.CallParams
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
data class JsonRpcRequest( data class ChainRequest(
val method: String, val method: String,
val params: CallParams, val params: CallParams,
val id: Int, val id: Int,
@@ -40,16 +41,7 @@ data class JsonRpcRequest(
) : this(method, params, 1, nonce, selectors, isStreamed) ) : this(method, params, 1, nonce, selectors, isStreamed)
fun toJson(): ByteArray { fun toJson(): ByteArray {
val json = mapOf( return params.toJson(id, method)
"jsonrpc" to "2.0",
"id" to id,
"method" to method,
"params" to when (params) {
is ListParams -> params.list
is ObjectParams -> params.obj
},
)
return Global.objectMapper.writeValueAsBytes(json)
} }
override fun toString(): String { override fun toString(): String {
@@ -57,9 +49,9 @@ data class JsonRpcRequest(
} }
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
class Deserializer : JsonDeserializer<JsonRpcRequest>() { class Deserializer : JsonDeserializer<ChainRequest>() {
override fun deserialize(p: JsonParser, ctxt: DeserializationContext): JsonRpcRequest { override fun deserialize(p: JsonParser, ctxt: DeserializationContext): ChainRequest {
val node: JsonNode = p.readValueAsTree() val node: JsonNode = p.readValueAsTree()
val id = node.get("id").intValue() val id = node.get("id").intValue()
val method = node.get("method").textValue() val method = node.get("method").textValue()
@@ -76,7 +68,7 @@ data class JsonRpcRequest(
throw IllegalStateException("Unsupported param type: ${it.asToken()}") throw IllegalStateException("Unsupported param type: ${it.asToken()}")
} }
} }
return JsonRpcRequest(method, ListParams(params as List<Any>), id, null, null) return ChainRequest(method, ListParams(params as List<Any>), id, null, null)
} }
} }
} }

View File

@@ -13,19 +13,19 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.JsonSerializer import com.fasterxml.jackson.databind.JsonSerializer
import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.SerializerProvider
import io.emeraldpay.dshackle.upstream.rpcclient.stream.Chunk
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.dshackle.upstream.stream.Chunk
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
class JsonRpcResponse( class ChainResponse(
private val result: ByteArray?, private val result: ByteArray?,
val error: JsonRpcError?, val error: ChainCallError?,
val id: Id, val id: Id,
val stream: Flux<Chunk>?, val stream: Flux<Chunk>?,
/** /**
@@ -38,42 +38,42 @@ class JsonRpcResponse(
constructor(stream: Flux<Chunk>, id: Int) : constructor(stream: Flux<Chunk>, id: Int) :
this(null, null, NumberId(id.toLong()), stream, null, null) this(null, null, NumberId(id.toLong()), stream, null, null)
constructor(result: ByteArray?, error: JsonRpcError?) : this(result, error, NumberId(0), null) constructor(result: ByteArray?, error: ChainCallError?) : this(result, error, NumberId(0), null)
constructor(result: ByteArray?, error: JsonRpcError?, resolvedBy: String?) : constructor(result: ByteArray?, error: ChainCallError?, resolvedBy: String?) :
this(result, error, NumberId(0), null, null, resolvedBy) this(result, error, NumberId(0), null, null, resolvedBy)
companion object { companion object {
private val NULL_VALUE = "null".toByteArray() private val NULL_VALUE = "null".toByteArray()
@JvmStatic @JvmStatic
fun ok(value: ByteArray): JsonRpcResponse { fun ok(value: ByteArray): ChainResponse {
return JsonRpcResponse(value, null) return ChainResponse(value, null)
} }
@JvmStatic @JvmStatic
fun ok(value: ByteArray, id: Id): JsonRpcResponse { fun ok(value: ByteArray, id: Id): ChainResponse {
return JsonRpcResponse(value, null, id, null) return ChainResponse(value, null, id, null)
} }
@JvmStatic @JvmStatic
fun ok(value: String): JsonRpcResponse { fun ok(value: String): ChainResponse {
return JsonRpcResponse(value.toByteArray(), null) return ChainResponse(value.toByteArray(), null)
} }
@JvmStatic @JvmStatic
fun error(code: Int, msg: String): JsonRpcResponse { fun error(code: Int, msg: String): ChainResponse {
return JsonRpcResponse(null, JsonRpcError(code, msg)) return ChainResponse(null, ChainCallError(code, msg))
} }
@JvmStatic @JvmStatic
fun error(error: JsonRpcError, id: Id): JsonRpcResponse { fun error(error: ChainCallError, id: Id): ChainResponse {
return JsonRpcResponse(null, error, id, null) return ChainResponse(null, error, id, null)
} }
@JvmStatic @JvmStatic
fun error(code: Int, msg: String, id: Id): JsonRpcResponse { fun error(code: Int, msg: String, id: Id): ChainResponse {
return JsonRpcResponse(null, JsonRpcError(code, msg), id, null) return ChainResponse(null, ChainCallError(code, msg), id, null)
} }
} }
@@ -126,13 +126,21 @@ class JsonRpcResponse(
} }
} }
fun copyWithId(id: Id): JsonRpcResponse { fun checkError(): Mono<ChainResponse> {
return JsonRpcResponse(result, error, id, stream, providedSignature, providedUpstreamId) return if (error != null) {
Mono.error(error.asException(id))
} else {
Mono.just(this)
}
}
fun copyWithId(id: Id): ChainResponse {
return ChainResponse(result, error, id, stream, providedSignature, providedUpstreamId)
} }
override fun equals(other: Any?): Boolean { override fun equals(other: Any?): Boolean {
if (this === other) return true if (this === other) return true
if (other !is JsonRpcResponse) return false if (other !is ChainResponse) return false
if (result != null) { if (result != null) {
if (other.result == null) return false if (other.result == null) return false
@@ -238,8 +246,8 @@ class JsonRpcResponse(
} }
} }
class ResponseJsonSerializer : JsonSerializer<JsonRpcResponse>() { class ResponseJsonSerializer : JsonSerializer<ChainResponse>() {
override fun serialize(value: JsonRpcResponse, gen: JsonGenerator, serializers: SerializerProvider) { override fun serialize(value: ChainResponse, gen: JsonGenerator, serializers: SerializerProvider) {
gen.writeStartObject() gen.writeStartObject()
gen.writeStringField("jsonrpc", "2.0") gen.writeStringField("jsonrpc", "2.0")
if (value.id.isNumber()) { if (value.id.isNumber()) {

View File

@@ -1,8 +1,7 @@
package io.emeraldpay.dshackle.upstream package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.reader.JsonRpcHttpReader
interface HttpFactory { interface HttpFactory {
fun create(id: String?, chain: Chain): JsonRpcHttpReader fun create(id: String?, chain: Chain): HttpReader
} }

View File

@@ -0,0 +1,114 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.micrometer.core.instrument.Metrics
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpHeaders
import io.netty.handler.ssl.SslContextBuilder
import io.netty.resolver.DefaultAddressResolverGroup
import reactor.core.publisher.Mono
import reactor.netty.http.client.HttpClient
import reactor.netty.resources.ConnectionProvider
import java.io.ByteArrayInputStream
import java.security.KeyStore
import java.security.cert.CertificateFactory
import java.security.cert.X509Certificate
import java.util.Base64
import java.util.function.Consumer
import java.util.function.Function
abstract class HttpReader(
protected val target: String,
protected val metrics: RequestMetrics,
basicAuth: AuthConfig.ClientBasicAuth? = null,
tlsCAAuth: ByteArray? = null,
) : ChainReader {
protected val httpClient: HttpClient
init {
val connectionProvider = ConnectionProvider.builder("dshackleConnectionPool")
.maxConnections(1500)
.pendingAcquireMaxCount(10000)
.build()
var build = HttpClient.create(connectionProvider)
.compress(true)
.resolver(DefaultAddressResolverGroup.INSTANCE)
build = build.headers { h ->
h.add(HttpHeaderNames.CONTENT_TYPE, "application/json")
}
basicAuth?.let { auth ->
val authString: String = auth.username + ":" + auth.password
val authBase64 = Base64.getEncoder().encodeToString(authString.toByteArray())
val encodedAuth = "Basic $authBase64"
val headers = Consumer { h: HttpHeaders -> h.add(HttpHeaderNames.AUTHORIZATION, encodedAuth) }
build = build.headers(headers)
}
tlsCAAuth?.let { auth ->
val cf = CertificateFactory.getInstance("X.509")
val cert = cf.generateCertificate(ByteArrayInputStream(auth)) as X509Certificate
val ks = KeyStore.getInstance(KeyStore.getDefaultType())
ks.load(null, "".toCharArray())
ks.setCertificateEntry("server", cert)
val sslContext = SslContextBuilder.forClient().trustManager(cert).build()
build.secure { spec ->
spec.sslContext(sslContext)
}
}
this.httpClient = build
}
override fun read(key: ChainRequest): Mono<ChainResponse> {
return internalRead(key)
.transform(convertErrors(key))
.transform(throwIfError())
}
protected abstract fun internalRead(key: ChainRequest): Mono<ChainResponse>
fun onStop() {
Metrics.globalRegistry.remove(metrics.timer)
Metrics.globalRegistry.remove(metrics.fails)
}
/**
* The subscribers expect to catch an exception if the response contains JSON RPC Error. Convert it here to JsonRpcException
*/
private fun throwIfError(): Function<Mono<ChainResponse>, Mono<ChainResponse>> {
return Function { resp ->
resp.flatMap {
if (it.hasError()) {
Mono.error(ChainCallUpstreamException(it.id, it.error!!))
} else {
Mono.just(it)
}
}
}
}
/**
* Convert internal exceptions to standard JsonRpcException
*/
private fun convertErrors(key: ChainRequest): Function<Mono<ChainResponse>, Mono<ChainResponse>> {
return Function { resp ->
resp.onErrorResume { t ->
val err = when (t) {
is RpcException -> ChainException.from(t)
is ChainException -> t
else -> ChainException(key.id, t.message ?: t.javaClass.name, cause = t)
}
// here we're measure the internal errors, not upstream errors
metrics.fails.increment()
Mono.error(err)
}
}
}
}

View File

@@ -1,10 +1,80 @@
package io.emeraldpay.dshackle.upstream package io.emeraldpay.dshackle.upstream
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.module.kotlin.readValue
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.reader.ChainReader
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
typealias LabelsDetectorBuilder = (Chain, JsonRpcReader) -> LabelsDetector? typealias LabelsDetectorBuilder = (Chain, ChainReader) -> LabelsDetector?
interface LabelsDetector { interface LabelsDetector {
fun detectLabels(): Flux<Pair<String, String>> fun detectLabels(): Flux<Pair<String, String>>
} }
abstract class BasicEthLabelsDetector(
private val reader: ChainReader,
) : LabelsDetector {
private val log = LoggerFactory.getLogger(this::class.java)
protected abstract fun nodeTypeRequest(): NodeTypeRequest
protected fun detectNodeType(): Flux<Pair<String, String>?> {
val nodeTypeRequest = nodeTypeRequest()
return reader
.read(nodeTypeRequest.request)
.flatMap(ChainResponse::requireResult)
.map { Global.objectMapper.readValue<JsonNode>(it) }
.flatMapMany { node ->
val mappedNode = nodeTypeRequest.mapper(node)
val labels = mutableListOf<Pair<String, String>>()
if (mappedNode.isTextual) {
clientType(mappedNode.textValue())?.let {
labels.add("client_type" to it)
}
clientVersion(mappedNode.textValue())?.let {
labels.add("client_version" to it)
}
}
Flux.fromIterable(labels)
}
.onErrorResume {
Flux.empty()
}
}
private fun clientVersion(client: String): String? {
val firstSlash = client.indexOf("/")
val secondSlash = client.indexOf("/", firstSlash + 1)
if (firstSlash == -1 || secondSlash == -1 || secondSlash < firstSlash) {
return null
}
return client.substring(firstSlash + 1, secondSlash)
}
private fun clientType(client: String): String? {
return if (client.contains("erigon", true)) {
"erigon"
} else if (client.contains("geth", true)) {
"geth"
} else if (client.contains("bor", true)) {
"bor"
} else if (client.contains("nethermind", true)) {
"nethermind"
} else if (client.contains("prysm", true)) {
"prysm"
} else if (client.contains("lighthouse", true)) {
"lighthouse"
} else {
log.debug("Unknown client type: {}", client)
null
}
}
data class NodeTypeRequest(
val request: ChainRequest,
val mapper: (JsonNode) -> JsonNode,
)
}

View File

@@ -22,7 +22,7 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods
@@ -216,9 +216,9 @@ abstract class Multistream(
/** /**
* Finds an API that leverages caches and other optimizations/transformations of the request. * Finds an API that leverages caches and other optimizations/transformations of the request.
*/ */
abstract fun getLocalReader(): Mono<JsonRpcReader> abstract fun getLocalReader(): Mono<ChainReader>
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
throw NotImplementedError("Immediate direct API is not implemented for Aggregated Upstream") throw NotImplementedError("Immediate direct API is not implemented for Aggregated Upstream")
} }

View File

@@ -13,12 +13,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream
import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Timer import io.micrometer.core.instrument.Timer
class RpcMetrics( class RequestMetrics(
val timer: Timer, val timer: Timer,
val fails: Counter, val fails: Counter,
) )

View File

@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
@@ -33,7 +33,7 @@ interface Upstream : Lifecycle {
/** /**
* Get an actual reader that access the current upstream * Get an actual reader that access the current upstream
*/ */
fun getIngressReader(): JsonRpcReader fun getIngressReader(): ChainReader
fun getOptions(): ChainOptions.Options fun getOptions(): ChainOptions.Options
fun getRole(): UpstreamsConfig.UpstreamRole fun getRole(): UpstreamsConfig.UpstreamRole

View File

@@ -4,7 +4,6 @@ import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -16,9 +15,8 @@ abstract class UpstreamValidator(
val upstream: Upstream, val upstream: Upstream,
val options: ChainOptions.Options, val options: ChainOptions.Options,
) { ) {
companion object { protected val log = LoggerFactory.getLogger(this::class.java)
private val log = LoggerFactory.getLogger(UpstreamValidator::class.java)
}
fun start(): Flux<UpstreamAvailability> { fun start(): Flux<UpstreamAvailability> {
return Flux.interval( return Flux.interval(
Duration.ZERO, Duration.ZERO,
@@ -39,6 +37,14 @@ abstract class UpstreamValidator(
fun validateUpstreamSettingsOnStartup(): ValidateUpstreamSettingsResult { fun validateUpstreamSettingsOnStartup(): ValidateUpstreamSettingsResult {
return validateUpstreamSettings().block() ?: ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR return validateUpstreamSettings().block() ?: ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR
} }
companion object {
@JvmStatic
fun resolve(results: Iterable<UpstreamAvailability>): UpstreamAvailability {
val cp = Comparator { avail1: UpstreamAvailability, avail2: UpstreamAvailability -> if (avail1.isBetterTo(avail2)) -1 else 1 }
return results.sortedWith(cp).last()
}
}
} }
enum class ValidateUpstreamSettingsResult { enum class ValidateUpstreamSettingsResult {
@@ -48,6 +54,6 @@ enum class ValidateUpstreamSettingsResult {
} }
data class SingleCallValidator( data class SingleCallValidator(
val method: JsonRpcRequest, val method: ChainRequest,
val check: (ByteArray) -> UpstreamAvailability, val check: (ByteArray) -> UpstreamAvailability,
) )

View File

@@ -0,0 +1,27 @@
package io.emeraldpay.dshackle.upstream.beaconchain
import com.fasterxml.jackson.databind.node.NullNode
import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.BasicEthLabelsDetector
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.rpcclient.RestParams
import reactor.core.publisher.Flux
class BeaconChainLabelsDetector(
reader: ChainReader,
) : BasicEthLabelsDetector(reader) {
override fun nodeTypeRequest(): NodeTypeRequest {
return NodeTypeRequest(
ChainRequest("GET#/eth/v1/node/version", RestParams.emptyParams()),
) { node ->
node.get("data")?.get("version") ?: NullNode.instance
}
}
override fun detectLabels(): Flux<Pair<String, String>> {
return Flux.merge(
detectNodeType(),
)
}
}

View File

@@ -0,0 +1,21 @@
package io.emeraldpay.dshackle.upstream.beaconchain
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import reactor.core.publisher.Mono
class BeaconChainLowerBoundBlockDetector(
chain: Chain,
upstream: Upstream,
) : LowerBoundBlockDetector(chain, upstream) {
// TODO: consensus nodes could be launched either in full mode or in archive mode
override fun lowerBlockDetect(): Mono<LowerBlockData> {
return Mono.just(LowerBlockData(1))
}
override fun periodRequest(): Long {
return 120
}
}

View File

@@ -0,0 +1,95 @@
package io.emeraldpay.dshackle.upstream.beaconchain
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.DeserializationContext
import com.fasterxml.jackson.databind.JsonDeserializer
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.module.kotlin.readValue
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.LabelsDetector
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamValidator
import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific
import io.emeraldpay.dshackle.upstream.rpcclient.RestParams
import java.math.BigInteger
import java.time.Instant
object BeaconChainSpecific : AbstractPollChainSpecific() {
override fun parseHeader(data: ByteArray, upstreamId: String): BlockContainer {
throw NotImplementedError()
}
override fun listenNewHeadsRequest(): ChainRequest {
throw NotImplementedError()
}
override fun unsubscribeNewHeadsRequest(subId: String): ChainRequest {
throw NotImplementedError()
}
override fun latestBlockRequest(): ChainRequest {
return ChainRequest("GET#/eth/v1/beacon/headers/head", RestParams.emptyParams())
}
override fun parseBlock(data: ByteArray, upstreamId: String): BlockContainer {
val blockHeader = Global.objectMapper.readValue<BeaconChainBlockHeader>(data)
return BlockContainer(
height = blockHeader.height,
hash = BlockId.from(blockHeader.hash),
difficulty = BigInteger.ZERO,
timestamp = Instant.EPOCH,
full = false,
json = data,
parsed = blockHeader,
transactions = emptyList(),
upstreamId = upstreamId,
parentHash = BlockId.from(blockHeader.parentHash),
)
}
override fun labelDetector(chain: Chain, reader: ChainReader): LabelsDetector {
return BeaconChainLabelsDetector(reader)
}
override fun validator(
chain: Chain,
upstream: Upstream,
options: ChainOptions.Options,
config: ChainsConfig.ChainConfig,
): UpstreamValidator {
return BeaconChainValidator(upstream, options)
}
override fun lowerBoundBlockDetector(chain: Chain, upstream: Upstream): LowerBoundBlockDetector {
return BeaconChainLowerBoundBlockDetector(chain, upstream)
}
}
data class BeaconChainBlockHeader(
val hash: String,
val parentHash: String,
val height: Long,
)
class BeaconChainBlockHeaderDeserializer : JsonDeserializer<BeaconChainBlockHeader>() {
override fun deserialize(p: JsonParser, ctxt: DeserializationContext): BeaconChainBlockHeader {
val node = p.readValueAsTree<JsonNode>()
val data = node["data"]
val hash = data["root"].textValue()
val headerData = data["header"]["message"]
val height = headerData["slot"].textValue().toLong()
val parentHash = headerData["parent_root"].textValue()
return BeaconChainBlockHeader(hash, parentHash, height)
}
}

View File

@@ -0,0 +1,88 @@
package io.emeraldpay.dshackle.upstream.beaconchain
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.BasicEthUpstreamValidator
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult
import io.emeraldpay.dshackle.upstream.rpcclient.RestParams
import reactor.core.publisher.Mono
import java.util.concurrent.TimeoutException
import java.util.function.Supplier
class BeaconChainValidator(
upstream: Upstream,
options: ChainOptions.Options,
) : BasicEthUpstreamValidator(upstream, options) {
override fun validatorFunctions(): List<Supplier<Mono<UpstreamAvailability>>> {
return listOf(
Supplier { validateSyncing() },
Supplier { validateHealth() },
Supplier { validatePeers() },
)
}
override fun validateUpstreamSettings(): Mono<ValidateUpstreamSettingsResult> {
return Mono.just(ValidateUpstreamSettingsResult.UPSTREAM_VALID)
}
private fun validateHealth(): Mono<UpstreamAvailability> {
return upstream.getIngressReader()
.read(ChainRequest("GET#/eth/v1/node/health", RestParams.emptyParams()))
.flatMap(ChainResponse::requireResult)
.map { UpstreamAvailability.OK }
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for /eth/v1/node/health from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for /eth/v1/node/health"))),
)
.doOnError { err -> log.error("Error during /eth/v1/node/health validation for ${upstream.getId()}", err) }
.onErrorReturn(UpstreamAvailability.UNAVAILABLE)
}
override fun validateSyncingRequest(): ValidateSyncingRequest {
return ValidateSyncingRequest(
ChainRequest("GET#/eth/v1/node/syncing", RestParams.emptyParams()),
) { bytes -> Global.objectMapper.readValue(bytes, BeaconChainSyncing::class.java).data.isSyncing }
}
override fun validatePeersRequest(): ValidatePeersRequest {
return ValidatePeersRequest(
ChainRequest("GET#/eth/v1/node/peer_count", RestParams.emptyParams()),
) { resp ->
Global.objectMapper.readValue(
resp.getResult(),
BeaconChainPeers::class.java,
).data.connected.toInt()
}
}
private data class BeaconChainSyncing(
@JsonProperty("data")
val data: BeaconChainSyncingData,
)
@JsonIgnoreProperties(ignoreUnknown = true)
private data class BeaconChainSyncingData(
@JsonProperty("is_syncing")
val isSyncing: Boolean,
)
private data class BeaconChainPeers(
@JsonProperty("data")
val data: BeaconChainPeersData,
)
@JsonIgnoreProperties(ignoreUnknown = true)
private data class BeaconChainPeersData(
@JsonProperty("connected")
val connected: String,
)
}

View File

@@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.CachingReader
import io.emeraldpay.dshackle.upstream.DistanceExtractor import io.emeraldpay.dshackle.upstream.DistanceExtractor
import io.emeraldpay.dshackle.upstream.EgressSubscription import io.emeraldpay.dshackle.upstream.EgressSubscription
@@ -91,7 +91,7 @@ open class BitcoinMultistream(
/** /**
* Finds an API that executed directly on a remote. * Finds an API that executed directly on a remote.
*/ */
open fun getDirectApi(matcher: Selector.Matcher): Mono<JsonRpcReader> { open fun getDirectApi(matcher: Selector.Matcher): Mono<ChainReader> {
val apis = getApiSource(matcher) val apis = getApiSource(matcher)
apis.request(1) apis.request(1)
return Mono.from(apis) return Mono.from(apis)
@@ -99,7 +99,7 @@ open class BitcoinMultistream(
.switchIfEmpty(Mono.error(Exception("No API available for $chain"))) .switchIfEmpty(Mono.error(Exception("No API available for $chain")))
} }
override fun getLocalReader(): Mono<JsonRpcReader> { override fun getLocalReader(): Mono<ChainReader> {
return Mono.just(callRouter) return Mono.just(callRouter)
} }

View File

@@ -18,12 +18,12 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.bitcoinj.core.Address import org.bitcoinj.core.Address
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
@@ -56,16 +56,16 @@ open class BitcoinReader(
} }
open fun getBlock(hash: String): Mono<Map<String, Any>> { open fun getBlock(hash: String): Mono<Map<String, Any>> {
return castedRead(JsonRpcRequest("getblock", ListParams(hash)), Map::class.java).cast() return castedRead(ChainRequest("getblock", ListParams(hash)), Map::class.java).cast()
} }
open fun getBlock(height: Long): Mono<Map<String, Any>> { open fun getBlock(height: Long): Mono<Map<String, Any>> {
return castedRead(JsonRpcRequest("getblockhash", ListParams(height)), String::class.java) return castedRead(ChainRequest("getblockhash", ListParams(height)), String::class.java)
.flatMap(this@BitcoinReader::getBlock) .flatMap(this@BitcoinReader::getBlock)
} }
open fun getTx(txid: String): Mono<Map<String, Any>> { open fun getTx(txid: String): Mono<Map<String, Any>> {
return castedRead(JsonRpcRequest("getrawtransaction", ListParams(txid, true)), Map::class.java).cast() return castedRead(ChainRequest("getrawtransaction", ListParams(txid, true)), Map::class.java).cast()
} }
open fun listUnspent(address: Address): Mono<List<SimpleUnspent>> { open fun listUnspent(address: Address): Mono<List<SimpleUnspent>> {
@@ -84,10 +84,10 @@ open class BitcoinReader(
mempool.stop() mempool.stop()
} }
fun <T> castedRead(req: JsonRpcRequest, clazz: Class<T>): Mono<T> { fun <T> castedRead(req: ChainRequest, clazz: Class<T>): Mono<T> {
return upstreams.getDirectApi(Selector.empty).flatMap { api -> return upstreams.getDirectApi(Selector.empty).flatMap { api ->
api.read(req) api.read(req)
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { .map {
objectMapper.readValue(it, clazz) as T objectMapper.readValue(it, clazz) as T
} }

View File

@@ -16,13 +16,13 @@
package io.emeraldpay.dshackle.upstream.bitcoin package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.AbstractHead import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.springframework.scheduling.concurrent.CustomizableThreadFactory import org.springframework.scheduling.concurrent.CustomizableThreadFactory
import reactor.core.Disposable import reactor.core.Disposable
@@ -34,7 +34,7 @@ import java.time.Duration
import java.util.concurrent.Executors import java.util.concurrent.Executors
class BitcoinRpcHead( class BitcoinRpcHead(
private val api: JsonRpcReader, private val api: ChainReader,
private val extractBlock: ExtractBlock, private val extractBlock: ExtractBlock,
private val interval: Duration = Duration.ofSeconds(15), private val interval: Duration = Duration.ofSeconds(15),
headScheduler: Scheduler, headScheduler: Scheduler,
@@ -60,14 +60,14 @@ class BitcoinRpcHead(
val base = Flux.interval(interval) val base = Flux.interval(interval)
.publishOn(scheduler) .publishOn(scheduler)
.flatMap { .flatMap {
api.read(JsonRpcRequest("getbestblockhash", ListParams())) api.read(ChainRequest("getbestblockhash", ListParams()))
.flatMap(JsonRpcResponse::requireStringResult) .flatMap(ChainResponse::requireStringResult)
.timeout(Defaults.timeout, Mono.error(Exception("Best block hash is not received"))) .timeout(Defaults.timeout, Mono.error(Exception("Best block hash is not received")))
} }
.distinctUntilChanged() .distinctUntilChanged()
.flatMap { hash -> .flatMap { hash ->
api.read(JsonRpcRequest("getblock", ListParams(hash))) api.read(ChainRequest("getblock", ListParams(hash)))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map(extractBlock::extract) .map(extractBlock::extract)
.timeout(Defaults.timeout, Mono.error(Exception("Block data is not received"))) .timeout(Defaults.timeout, Mono.error(Exception("Block data is not received")))
} }

View File

@@ -19,11 +19,11 @@ import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcHttpReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HttpReader
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
@@ -34,7 +34,7 @@ import reactor.core.Disposable
open class BitcoinRpcUpstream( open class BitcoinRpcUpstream(
id: String, id: String,
chain: Chain, chain: Chain,
private val directApi: JsonRpcHttpReader, private val directApi: HttpReader,
private val head: Head, private val head: Head,
options: ChainOptions.Options, options: ChainOptions.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
@@ -56,7 +56,7 @@ open class BitcoinRpcUpstream(
return head return head
} }
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
return directApi return directApi
} }

View File

@@ -16,10 +16,10 @@
package io.emeraldpay.dshackle.upstream.bitcoin package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.scheduling.concurrent.CustomizableThreadFactory import org.springframework.scheduling.concurrent.CustomizableThreadFactory
@@ -30,7 +30,7 @@ import java.time.Duration
import java.util.concurrent.Executors import java.util.concurrent.Executors
class BitcoinUpstreamValidator( class BitcoinUpstreamValidator(
private val api: JsonRpcReader, private val api: ChainReader,
private val options: ChainOptions.Options, private val options: ChainOptions.Options,
) { ) {
@@ -41,8 +41,8 @@ class BitcoinUpstreamValidator(
} }
fun validate(): Mono<UpstreamAvailability> { fun validate(): Mono<UpstreamAvailability> {
return api.read(JsonRpcRequest("getconnectioncount", ListParams())) return api.read(ChainRequest("getconnectioncount", ListParams()))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { Integer.parseInt(String(it)) } .map { Integer.parseInt(String(it)) }
.map { count -> .map { count ->
val minPeers = options.minPeers ?: 1 val minPeers = options.minPeers ?: 1

View File

@@ -2,13 +2,13 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.AbstractHead import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.apache.commons.codec.binary.Hex import org.apache.commons.codec.binary.Hex
import reactor.core.Disposable import reactor.core.Disposable
@@ -20,7 +20,7 @@ import java.time.Duration
class BitcoinZMQHead( class BitcoinZMQHead(
private val server: ZMQServer, private val server: ZMQServer,
private val api: JsonRpcReader, private val api: ChainReader,
private val extractBlock: ExtractBlock, private val extractBlock: ExtractBlock,
headScheduler: Scheduler, headScheduler: Scheduler,
) : Head, AbstractHead(MostWorkForkChoice(), headScheduler, awaitHeadTimeoutMs = 1200_000), Lifecycle { ) : Head, AbstractHead(MostWorkForkChoice(), headScheduler, awaitHeadTimeoutMs = 1200_000), Lifecycle {
@@ -33,11 +33,11 @@ class BitcoinZMQHead(
Hex.encodeHexString(it) Hex.encodeHexString(it)
} }
.flatMap { hash -> .flatMap { hash ->
api.read(JsonRpcRequest("getblock", ListParams(hash))) api.read(ChainRequest("getblock", ListParams(hash)))
.switchIfEmpty(Mono.error(IllegalStateException("Block $hash is not available on upstream"))) .switchIfEmpty(Mono.error(IllegalStateException("Block $hash is not available on upstream")))
.retryWhen(Retry.backoff(5, Duration.ofMillis(100))) .retryWhen(Retry.backoff(5, Duration.ofMillis(100)))
.switchIfEmpty(Mono.fromCallable { log.warn("Block $hash is not available on upstream") }.then(Mono.empty())) .switchIfEmpty(Mono.fromCallable { log.warn("Block $hash is not available on upstream") }.then(Mono.empty()))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map(extractBlock::extract) .map(extractBlock::extract)
.timeout(Defaults.timeout, Mono.error(Exception("Block data is not received"))) .timeout(Defaults.timeout, Mono.error(Exception("Block data is not received")))
} }

View File

@@ -17,11 +17,11 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.Disposable import reactor.core.Disposable
@@ -66,8 +66,8 @@ open class CachingMempoolData(
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
fun fetchFromUpstream(): Mono<List<String>> { fun fetchFromUpstream(): Mono<List<String>> {
return upstreams.getDirectApi(Selector.empty).flatMap { api -> return upstreams.getDirectApi(Selector.empty).flatMap { api ->
api.read(JsonRpcRequest("getrawmempool", ListParams())) api.read(ChainRequest("getrawmempool", ListParams()))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { objectMapper.readValue(it, List::class.java) as List<String> } .map { objectMapper.readValue(it, List::class.java) as List<String> }
} }
} }

View File

@@ -17,14 +17,14 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.bitcoinj.core.Address import org.bitcoinj.core.Address
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
@@ -40,16 +40,16 @@ import reactor.core.publisher.Mono
class LocalCallRouter( class LocalCallRouter(
private val methods: CallMethods, private val methods: CallMethods,
private val reader: BitcoinReader, private val reader: BitcoinReader,
) : JsonRpcReader { ) : ChainReader {
companion object { companion object {
private val log = LoggerFactory.getLogger(LocalCallRouter::class.java) private val log = LoggerFactory.getLogger(LocalCallRouter::class.java)
} }
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> { override fun read(key: ChainRequest): Mono<ChainResponse> {
if (methods.isHardcoded(key.method)) { if (methods.isHardcoded(key.method)) {
return Mono.just(methods.executeHardcoded(key.method)) return Mono.just(methods.executeHardcoded(key.method))
.map { JsonRpcResponse(it, null) } .map { ChainResponse(it, null) }
} }
if (!methods.isCallable(key.method)) { if (!methods.isCallable(key.method)) {
return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))
@@ -63,7 +63,7 @@ class LocalCallRouter(
/** /**
* *
*/ */
fun processUnspentRequest(key: JsonRpcRequest): Mono<JsonRpcResponse> { fun processUnspentRequest(key: ChainRequest): Mono<ChainResponse> {
if (key.params is ListParams) { if (key.params is ListParams) {
if (key.params.list.size < 3) { if (key.params.list.size < 3) {
return Mono.error(SilentException("Invalid call to unspent. Address is missing")) return Mono.error(SilentException("Invalid call to unspent. Address is missing"))
@@ -74,7 +74,7 @@ class LocalCallRouter(
return reader.listUnspent(address).map { return reader.listUnspent(address).map {
val rpc = it.map(convertUnspent(address)) val rpc = it.map(convertUnspent(address))
val json = Global.objectMapper.writeValueAsBytes(rpc) val json = Global.objectMapper.writeValueAsBytes(rpc)
JsonRpcResponse.ok(json, JsonRpcResponse.NumberId(key.id)) ChainResponse.ok(json, ChainResponse.NumberId(key.id))
} }
} }
} }

View File

@@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.bitcoinj.core.Address import org.bitcoinj.core.Address
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
@@ -51,8 +51,8 @@ class RpcUnspentReader(
// //
val address = key.toString() val address = key.toString()
return upstreams.getDirectApi(selector).flatMap { api -> return upstreams.getDirectApi(selector).flatMap { api ->
api.read(JsonRpcRequest("listunspent", ListParams(1, 9999999, listOf(address)))) api.read(ChainRequest("listunspent", ListParams(1, 9999999, listOf(address))))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { .map {
Global.objectMapper.readerFor(RpcUnspent::class.java).readValues<RpcUnspent>(it).readAll() Global.objectMapper.readerFor(RpcUnspent::class.java).readValues<RpcUnspent>(it).readAll()
} }

View File

@@ -0,0 +1,143 @@
package io.emeraldpay.dshackle.upstream.calls
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
class DefaultBeaconChainMethods : CallMethods {
private val beaconMethods = setOf(
getMethod("/eth/v1/beacon/genesis"),
getMethod("/eth/v1/beacon/states/*/root"),
getMethod("/eth/v1/beacon/states/*/fork"),
getMethod("/eth/v1/beacon/states/*/finality_checkpoints"),
getMethod("/eth/v1/beacon/states/*/validators"),
postMethod("/eth/v1/beacon/states/*/validators"),
getMethod("/eth/v1/beacon/states/*/validators/*"),
getMethod("/eth/v1/beacon/states/*/validator_balances"),
postMethod("/eth/v1/beacon/states/*/validator_balances"),
getMethod("/eth/v1/beacon/states/*/committees"),
getMethod("/eth/v1/beacon/states/*/sync_committees"),
getMethod("/eth/v1/beacon/states/*/randao"),
getMethod("/eth/v1/beacon/headers"),
getMethod("/eth/v1/beacon/headers/*"),
postMethod("/eth/v1/beacon/blinded_blocks"),
postMethod("/eth/v2/beacon/blinded_blocks"),
postMethod("/eth/v1/beacon/blocks"),
postMethod("/eth/v2/beacon/blocks"),
getMethod("/eth/v2/beacon/blocks/*"),
getMethod("/eth/v1/beacon/blocks/*/root"),
getMethod("/eth/v1/beacon/blocks/*/attestations"),
getMethod("/eth/v1/beacon/blob_sidecars/*"),
postMethod("/eth/v1/beacon/rewards/sync_committee/*"),
getMethod("/eth/v1/beacon/deposit_snapshot"),
getMethod("/eth/v1/beacon/rewards/blocks/*"),
postMethod("/eth/v1/beacon/rewards/attestations/*"),
getMethod("/eth/v1/beacon/blinded_blocks/*"),
getMethod("/eth/v1/beacon/light_client/bootstrap/*"),
getMethod("/eth/v1/beacon/light_client/updates"),
getMethod("/eth/v1/beacon/light_client/finality_update"),
getMethod("/eth/v1/beacon/light_client/optimistic_update"),
getMethod("/eth/v1/beacon/pool/attestations"),
postMethod("/eth/v1/beacon/pool/attestations"),
getMethod("/eth/v1/beacon/pool/attester_slashings"),
postMethod("/eth/v1/beacon/pool/attester_slashings"),
getMethod("/eth/v1/beacon/pool/proposer_slashings"),
postMethod("/eth/v1/beacon/pool/proposer_slashings"),
postMethod("/eth/v1/beacon/pool/sync_committees"),
getMethod("/eth/v1/beacon/pool/voluntary_exits"),
postMethod("/eth/v1/beacon/pool/voluntary_exits"),
getMethod("/eth/v1/beacon/pool/bls_to_execution_changes"),
postMethod("/eth/v1/beacon/pool/bls_to_execution_changes"),
)
private val builderMethods = setOf(
getMethod("/eth/v1/builder/states/*/expected_withdrawals"),
)
private val configMethods = setOf(
getMethod("/eth/v1/config/fork_schedule"),
getMethod("/eth/v1/config/spec"),
getMethod("/eth/v1/config/deposit_contract"),
)
private val debugMethods = setOf(
getMethod("/eth/v2/debug/beacon/states/*"),
getMethod("/eth/v2/debug/beacon/heads"),
getMethod("/eth/v1/debug/fork_choice"),
)
private val eventMethods = setOf(
getMethod("/eth/v1/events"),
)
// need to think up what to do with these methods, probably hardcode them?
private val nodeMethods = setOf(
getMethod("/eth/v1/node/identity"),
getMethod("/eth/v1/node/peers"),
getMethod("/eth/v1/node/peers/*"),
getMethod("/eth/v1/node/peer_count"),
getMethod("/eth/v1/node/version"),
getMethod("/eth/v1/node/syncing"),
getMethod("/eth/v1/node/health"),
)
private val validatorMethods = setOf(
postMethod("/eth/v1/validator/duties/attester/*"),
getMethod("/eth/v1/validator/duties/proposer/*"),
postMethod("/eth/v1/validator/duties/sync/*"),
getMethod("/eth/v3/validator/blocks/*"),
getMethod("/eth/v1/validator/attestation_data"),
getMethod("/eth/v1/validator/aggregate_attestation"),
postMethod("/eth/v1/validator/aggregate_and_proofs"),
postMethod("/eth/v1/validator/beacon_committee_subscriptions"),
postMethod("/eth/v1/validator/sync_committee_subscriptions"),
postMethod("/eth/v1/validator/sync_committee_subscriptions"),
getMethod("/eth/v1/validator/sync_committee_contribution"),
postMethod("/eth/v1/validator/sync_committee_selections"),
postMethod("/eth/v1/validator/contribution_and_proofs"),
postMethod("/eth/v1/validator/prepare_beacon_proposer"),
postMethod("/eth/v1/validator/register_validator"),
postMethod("/eth/v1/validator/liveness/*"),
)
private val rewardMethods = setOf(
postMethod("/eth/v1/beacon/rewards/sync_committee/*"),
getMethod("/eth/v1/beacon/rewards/blocks/*"),
postMethod("/eth/v1/beacon/rewards/blocks/*"),
)
private val allowedMethods: Set<String> =
beaconMethods + builderMethods + configMethods + debugMethods + eventMethods + nodeMethods + validatorMethods
override fun createQuorumFor(method: String): CallQuorum {
return AlwaysQuorum()
}
override fun isCallable(method: String): Boolean {
return allowedMethods.contains(method)
}
override fun getSupportedMethods(): Set<String> {
return allowedMethods.toSortedSet()
}
override fun isHardcoded(method: String): Boolean {
return false
}
override fun executeHardcoded(method: String): ByteArray {
throw RpcException(-32601, "Method not found")
}
override fun getGroupMethods(groupName: String): Set<String> {
return when (groupName) {
"default" -> getSupportedMethods()
else -> emptyList()
}.toSet()
}
private fun getMethod(method: String) = "GET#$method"
private fun postMethod(method: String) = "POST#$method"
}

View File

@@ -1,10 +1,10 @@
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.ethereum.hex.HexQuantity import io.emeraldpay.dshackle.upstream.ethereum.hex.HexQuantity
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import kotlin.math.max import kotlin.math.max
@@ -14,12 +14,12 @@ private const val OPTIMISM_BEDROCK_BLOCK = "0x645C277" // 105235063
private const val EARLIEST_BLOCK = "0x2710" // 10000 private const val EARLIEST_BLOCK = "0x2710" // 10000
class EthereumArchiveBlockNumberReader( class EthereumArchiveBlockNumberReader(
private val reader: JsonRpcReader, private val reader: ChainReader,
) { ) {
fun readArchiveBlock(): Mono<String> = fun readArchiveBlock(): Mono<String> =
reader.read(JsonRpcRequest("eth_blockNumber", ListParams())) reader.read(ChainRequest("eth_blockNumber", ListParams()))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { .map {
HexQuantity HexQuantity
.from( .from(

View File

@@ -5,8 +5,9 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.foundation.ChainOptions.Options import io.emeraldpay.dshackle.foundation.ChainOptions.Options
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.CachingReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.EgressSubscription import io.emeraldpay.dshackle.upstream.EgressSubscription
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.IngressSubscription
@@ -27,7 +28,6 @@ import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource
import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific
import io.emeraldpay.dshackle.upstream.generic.CachingReaderBuilder import io.emeraldpay.dshackle.upstream.generic.CachingReaderBuilder
import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.generic.GenericUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.springframework.cloud.sleuth.Tracer import org.springframework.cloud.sleuth.Tracer
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -42,17 +42,19 @@ object EthereumChainSpecific : AbstractPollChainSpecific() {
return parseBlock(data, upstreamId) return parseBlock(data, upstreamId)
} }
override fun latestBlockRequest() = JsonRpcRequest("eth_getBlockByNumber", ListParams("latest", false)) override fun latestBlockRequest() =
override fun listenNewHeadsRequest(): JsonRpcRequest = JsonRpcRequest("eth_subscribe", ListParams("newHeads")) ChainRequest("eth_getBlockByNumber", ListParams("latest", false))
override fun unsubscribeNewHeadsRequest(subId: String): JsonRpcRequest = override fun listenNewHeadsRequest(): ChainRequest =
JsonRpcRequest("eth_unsubscribe", ListParams(subId)) ChainRequest("eth_subscribe", ListParams("newHeads"))
override fun unsubscribeNewHeadsRequest(subId: String): ChainRequest =
ChainRequest("eth_unsubscribe", ListParams(subId))
override fun localReaderBuilder( override fun localReaderBuilder(
cachingReader: CachingReader, cachingReader: CachingReader,
methods: CallMethods, methods: CallMethods,
head: Head, head: Head,
logsOracle: LogsOracle?, logsOracle: LogsOracle?,
): Mono<JsonRpcReader> { ): Mono<ChainReader> {
return Mono.just(EthereumLocalReader(cachingReader as EthereumCachingReader, methods, head, logsOracle)) return Mono.just(EthereumLocalReader(cachingReader as EthereumCachingReader, methods, head, logsOracle))
} }
@@ -94,7 +96,7 @@ object EthereumChainSpecific : AbstractPollChainSpecific() {
return EthereumLowerBoundBlockDetector(chain, upstream) return EthereumLowerBoundBlockDetector(chain, upstream)
} }
override fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector { override fun labelDetector(chain: Chain, reader: ChainReader): LabelsDetector {
return EthereumLabelsDetector(reader, chain) return EthereumLabelsDetector(reader, chain)
} }

View File

@@ -12,7 +12,9 @@ import io.emeraldpay.dshackle.data.DefaultContainer
import io.emeraldpay.dshackle.data.TxContainer import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.reader.RpcReaderFactory import io.emeraldpay.dshackle.reader.RequestReaderFactory
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -29,8 +31,6 @@ import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionReceiptJson
import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionRefJson import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionRefJson
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.apache.commons.collections4.Factory import org.apache.commons.collections4.Factory
import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.commons.lang3.exception.ExceptionUtils
@@ -57,7 +57,7 @@ class EthereumDirectReader(
} }
private val objectMapper: ObjectMapper = Global.objectMapper private val objectMapper: ObjectMapper = Global.objectMapper
var rpcReaderFactory: RpcReaderFactory = RpcReaderFactory.default() var requestReaderFactory: RequestReaderFactory = RequestReaderFactory.default()
val blockReader: Reader<BlockHash, Result<BlockContainer>> val blockReader: Reader<BlockHash, Result<BlockContainer>>
val blockByHeightReader: Reader<Long, Result<BlockContainer>> val blockByHeightReader: Reader<Long, Result<BlockContainer>>
@@ -69,20 +69,20 @@ class EthereumDirectReader(
init { init {
blockReader = object : Reader<BlockHash, Result<BlockContainer>> { blockReader = object : Reader<BlockHash, Result<BlockContainer>> {
override fun read(key: BlockHash): Mono<Result<BlockContainer>> { override fun read(key: BlockHash): Mono<Result<BlockContainer>> {
val request = JsonRpcRequest("eth_getBlockByHash", ListParams(key.toHex(), false)) val request = ChainRequest("eth_getBlockByHash", ListParams(key.toHex(), false))
return readBlock(request, key.toHex()) return readBlock(request, key.toHex())
} }
} }
blockByHeightReader = object : Reader<Long, Result<BlockContainer>> { blockByHeightReader = object : Reader<Long, Result<BlockContainer>> {
override fun read(key: Long): Mono<Result<BlockContainer>> { override fun read(key: Long): Mono<Result<BlockContainer>> {
val heightMatcher = Selector.HeightMatcher(key) val heightMatcher = Selector.HeightMatcher(key)
val request = JsonRpcRequest("eth_getBlockByNumber", ListParams(HexQuantity.from(key).toHex(), false)) val request = ChainRequest("eth_getBlockByNumber", ListParams(HexQuantity.from(key).toHex(), false))
return readBlock(request, key.toString(), heightMatcher) return readBlock(request, key.toString(), heightMatcher)
} }
} }
txReader = object : Reader<TransactionId, Result<TxContainer>> { txReader = object : Reader<TransactionId, Result<TxContainer>> {
override fun read(key: TransactionId): Mono<Result<TxContainer>> { override fun read(key: TransactionId): Mono<Result<TxContainer>> {
val request = JsonRpcRequest("eth_getTransactionByHash", ListParams(key.toHex())) val request = ChainRequest("eth_getTransactionByHash", ListParams(key.toHex()))
return readWithQuorum(request) // retries were removed because we use NotNullQuorum which handle errors too return readWithQuorum(request) // retries were removed because we use NotNullQuorum which handle errors too
.timeout(Duration.ofSeconds(5), Mono.error(TimeoutException("Tx not read $key"))) .timeout(Duration.ofSeconds(5), Mono.error(TimeoutException("Tx not read $key")))
.flatMap { result -> .flatMap { result ->
@@ -100,14 +100,14 @@ class EthereumDirectReader(
caches.cache(Caches.Tag.REQUESTED, tx.data) caches.cache(Caches.Tag.REQUESTED, tx.data)
} }
}.onErrorResume { }.onErrorResume {
Mono.error(JsonRpcException(request.id, ExceptionUtils.getRootCauseMessage(it))) Mono.error(ChainException(request.id, ExceptionUtils.getRootCauseMessage(it)))
} }
} }
} }
balanceReader = object : Reader<Address, Result<Wei>> { balanceReader = object : Reader<Address, Result<Wei>> {
override fun read(key: Address): Mono<Result<Wei>> { override fun read(key: Address): Mono<Result<Wei>> {
val height = up.getHead().getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest" val height = up.getHead().getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest"
val request = JsonRpcRequest("eth_getBalance", ListParams(key.toHex(), height)) val request = ChainRequest("eth_getBalance", ListParams(key.toHex(), height))
return readWithQuorum(request) return readWithQuorum(request)
.timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Balance not read $key"))) .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Balance not read $key")))
.map { .map {
@@ -131,7 +131,7 @@ class EthereumDirectReader(
receiptReader = object : Reader<TransactionId, Result<ByteArray>> { receiptReader = object : Reader<TransactionId, Result<ByteArray>> {
override fun read(key: TransactionId): Mono<Result<ByteArray>> { override fun read(key: TransactionId): Mono<Result<ByteArray>> {
val request = JsonRpcRequest("eth_getTransactionReceipt", ListParams(key.toHex())) val request = ChainRequest("eth_getTransactionReceipt", ListParams(key.toHex()))
return readWithQuorum(request) return readWithQuorum(request)
.timeout(Duration.ofSeconds(5), Mono.error(TimeoutException("Receipt not read $key"))) .timeout(Duration.ofSeconds(5), Mono.error(TimeoutException("Receipt not read $key")))
.flatMap { result -> .flatMap { result ->
@@ -155,14 +155,14 @@ class EthereumDirectReader(
) )
} }
}.onErrorResume { }.onErrorResume {
Mono.error(JsonRpcException(request.id, ExceptionUtils.getRootCauseMessage(it))) Mono.error(ChainException(request.id, ExceptionUtils.getRootCauseMessage(it)))
} }
} }
} }
logsByHashReader = object : Reader<BlockId, Result<List<TransactionLogJson>>> { logsByHashReader = object : Reader<BlockId, Result<List<TransactionLogJson>>> {
override fun read(key: BlockId): Mono<Result<List<TransactionLogJson>>> { override fun read(key: BlockId): Mono<Result<List<TransactionLogJson>>> {
val request = JsonRpcRequest( val request = ChainRequest(
"eth_getLogs", "eth_getLogs",
ListParams( ListParams(
mapOf( mapOf(
@@ -190,7 +190,7 @@ class EthereumDirectReader(
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
private fun readBlock( private fun readBlock(
request: JsonRpcRequest, request: ChainRequest,
id: String, id: String,
matcher: Selector.Matcher = Selector.empty, matcher: Selector.Matcher = Selector.empty,
): Mono<Result<BlockContainer>> { ): Mono<Result<BlockContainer>> {
@@ -216,7 +216,7 @@ class EthereumDirectReader(
.doOnNext { block -> .doOnNext { block ->
caches.cache(Caches.Tag.REQUESTED, block.data) caches.cache(Caches.Tag.REQUESTED, block.data)
}.onErrorResume { }.onErrorResume {
Mono.error(JsonRpcException(request.id, ExceptionUtils.getRootCauseMessage(it))) Mono.error(ChainException(request.id, ExceptionUtils.getRootCauseMessage(it)))
} }
} }
@@ -224,19 +224,18 @@ class EthereumDirectReader(
* Read from an Upstream applying a Quorum specific for that request * Read from an Upstream applying a Quorum specific for that request
*/ */
private fun readWithQuorum( private fun readWithQuorum(
request: JsonRpcRequest, request: ChainRequest,
matcher: Selector.Matcher = Selector.empty, matcher: Selector.Matcher = Selector.empty,
): Mono<Result<ByteArray>> { ): Mono<Result<ByteArray>> {
return Mono.just(rpcReaderFactory) return Mono.just(requestReaderFactory)
.map { .map {
val requestMatcher = Selector.Builder() val requestMatcher = Selector.Builder()
.withMatcher(matcher) .withMatcher(matcher)
.forMethod(request.method) .forMethod(request.method)
.build() .build()
it.create( it.create(
RpcReaderFactory.RpcReaderData( RequestReaderFactory.ReaderData(
up, up,
request.method,
requestMatcher, requestMatcher,
callMethodsFactory.create().createQuorumFor(request.method), callMethodsFactory.create().createQuorumFor(request.method),
null, null,

View File

@@ -18,15 +18,15 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Global.Companion.nullValue import io.emeraldpay.dshackle.Global.Companion.nullValue
import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.LogsOracle import io.emeraldpay.dshackle.upstream.LogsOracle
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.hex.HexQuantity import io.emeraldpay.dshackle.upstream.ethereum.hex.HexQuantity
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import reactor.kotlin.core.publisher.switchIfEmpty import reactor.kotlin.core.publisher.switchIfEmpty
@@ -44,12 +44,12 @@ class EthereumLocalReader(
private val methods: CallMethods, private val methods: CallMethods,
private val head: Head, private val head: Head,
private val logsOracle: LogsOracle?, private val logsOracle: LogsOracle?,
) : JsonRpcReader { ) : ChainReader {
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> { override fun read(key: ChainRequest): Mono<ChainResponse> {
if (methods.isHardcoded(key.method)) { if (methods.isHardcoded(key.method)) {
return Mono.just(methods.executeHardcoded(key.method)) return Mono.just(methods.executeHardcoded(key.method))
.map { JsonRpcResponse(it, null) } .map { ChainResponse(it, null) }
} }
if (!methods.isCallable(key.method)) { if (!methods.isCallable(key.method)) {
return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))
@@ -61,7 +61,7 @@ class EthereumLocalReader(
val common = commonRequests(key) val common = commonRequests(key)
?.switchIfEmpty { Mono.just(nullValue to null) } ?.switchIfEmpty { Mono.just(nullValue to null) }
if (common != null) { if (common != null) {
return common.map { JsonRpcResponse(it.first, null, it.second) } return common.map { ChainResponse(it.first, null, it.second) }
} }
return Mono.empty() return Mono.empty()
} }
@@ -71,7 +71,7 @@ class EthereumLocalReader(
* parses JSON into Map. But the purpose of further processing and caching for some of the requests we want * parses JSON into Map. But the purpose of further processing and caching for some of the requests we want
* to have actual data types. * to have actual data types.
*/ */
fun commonRequests(key: JsonRpcRequest): Mono<Pair<ByteArray, String?>>? { fun commonRequests(key: ChainRequest): Mono<Pair<ByteArray, String?>>? {
val method = key.method val method = key.method
val params = key.params val params = key.params
if (params is ListParams) { if (params is ListParams) {

View File

@@ -1,10 +1,10 @@
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.RecursiveLowerBoundBlockDetector import io.emeraldpay.dshackle.upstream.RecursiveLowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import io.emeraldpay.dshackle.upstream.toHex import io.emeraldpay.dshackle.upstream.toHex
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -45,13 +45,13 @@ class EthereumLowerBoundBlockDetector(
return Mono.just(true) return Mono.just(true)
} }
return upstream.getIngressReader().read( return upstream.getIngressReader().read(
JsonRpcRequest( ChainRequest(
"eth_getBalance", "eth_getBalance",
ListParams("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0", blockNumber.toHex()), ListParams("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0", blockNumber.toHex()),
), ),
) )
.retryWhen(retrySpec(nonRetryableErrors)) .retryWhen(retrySpec(nonRetryableErrors))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { true } .map { true }
.onErrorReturn(false) .onErrorReturn(false)
} }

View File

@@ -22,109 +22,56 @@ import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.BasicEthUpstreamValidator
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.UpstreamValidator
import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult
import io.emeraldpay.dshackle.upstream.ethereum.domain.Address import io.emeraldpay.dshackle.upstream.ethereum.domain.Address
import io.emeraldpay.dshackle.upstream.ethereum.hex.HexData import io.emeraldpay.dshackle.upstream.ethereum.hex.HexData
import io.emeraldpay.dshackle.upstream.ethereum.json.SyncingJson import io.emeraldpay.dshackle.upstream.ethereum.json.SyncingJson
import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionCallJson import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionCallJson
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory
import org.springframework.scheduling.concurrent.CustomizableThreadFactory import org.springframework.scheduling.concurrent.CustomizableThreadFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import reactor.core.scheduler.Schedulers import reactor.core.scheduler.Schedulers
import reactor.kotlin.extra.retry.retryRandomBackoff import reactor.kotlin.extra.retry.retryRandomBackoff
import reactor.util.function.Tuple2
import java.time.Duration import java.time.Duration
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.TimeoutException import java.util.concurrent.TimeoutException
import java.util.function.Supplier
open class EthereumUpstreamValidator @JvmOverloads constructor( open class EthereumUpstreamValidator @JvmOverloads constructor(
private val chain: Chain, private val chain: Chain,
upstream: Upstream, upstream: Upstream,
options: ChainOptions.Options, options: ChainOptions.Options,
private val config: ChainConfig, private val config: ChainConfig,
) : UpstreamValidator(upstream, options) { ) : BasicEthUpstreamValidator(upstream, options) {
companion object { companion object {
private val log = LoggerFactory.getLogger(EthereumUpstreamValidator::class.java)
val scheduler = val scheduler =
Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("ethereum-validator"))) Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("ethereum-validator")))
} }
private val objectMapper: ObjectMapper = Global.objectMapper private val objectMapper: ObjectMapper = Global.objectMapper
override fun validate(): Mono<UpstreamAvailability> { override fun validateSyncingRequest(): ValidateSyncingRequest {
return Mono.zip( return ValidateSyncingRequest(
validateSyncing(), ChainRequest("eth_syncing", ListParams()),
validatePeers(), ) { bytes -> objectMapper.readValue(bytes, SyncingJson::class.java).isSyncing }
}
override fun validatePeersRequest(): ValidatePeersRequest {
return ValidatePeersRequest(
ChainRequest("net_peerCount", ListParams()),
) { resp -> Integer.decode(resp.getResultAsProcessedString()) }
}
override fun validatorFunctions(): List<Supplier<Mono<UpstreamAvailability>>> {
return listOf(
Supplier { validateSyncing() },
Supplier { validatePeers() },
) )
.map(::resolve)
.defaultIfEmpty(UpstreamAvailability.UNAVAILABLE)
.onErrorResume {
log.error("Error during upstream validation for ${upstream.getId()}", it)
Mono.just(UpstreamAvailability.UNAVAILABLE)
}
}
fun resolve(results: Tuple2<UpstreamAvailability, UpstreamAvailability>): UpstreamAvailability {
val cp = Comparator { avail1: UpstreamAvailability, avail2: UpstreamAvailability -> if (avail1.isBetterTo(avail2)) -1 else 1 }
return listOf(results.t1, results.t2).sortedWith(cp).last()
}
fun validateSyncing(): Mono<UpstreamAvailability> {
if (!options.validateSyncing) {
return Mono.just(UpstreamAvailability.OK)
}
return upstream.getIngressReader()
.read(JsonRpcRequest("eth_syncing", ListParams()))
.flatMap(JsonRpcResponse::requireResult)
.map { objectMapper.readValue(it, SyncingJson::class.java) }
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for eth_syncing from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for Syncing"))),
)
.map {
val isSyncing = it.isSyncing
upstream.getHead().onSyncingNode(isSyncing)
if (isSyncing) {
UpstreamAvailability.SYNCING
} else {
UpstreamAvailability.OK
}
}
.doOnError { err -> log.error("Error during syncing validation for ${upstream.getId()}", err) }
.onErrorReturn(UpstreamAvailability.UNAVAILABLE)
}
fun validatePeers(): Mono<UpstreamAvailability> {
if (!options.validatePeers || options.minPeers == 0) {
return Mono.just(UpstreamAvailability.OK)
}
return upstream
.getIngressReader()
.read(JsonRpcRequest("net_peerCount", ListParams()))
.flatMap(JsonRpcResponse::requireStringResult)
.map(Integer::decode)
.timeout(
Defaults.timeoutInternal,
Mono.fromCallable { log.warn("No response for net_peerCount from ${upstream.getId()}") }
.then(Mono.error(TimeoutException("Validation timeout for Peers"))),
)
.map { count ->
val minPeers = options.minPeers
if (count < minPeers) {
UpstreamAvailability.IMMATURE
} else {
UpstreamAvailability.OK
}
}
.doOnError { err -> log.error("Error during peer count validation for ${upstream.getId()}", err) }
.onErrorReturn(UpstreamAvailability.UNAVAILABLE)
} }
override fun validateUpstreamSettings(): Mono<ValidateUpstreamSettingsResult> { override fun validateUpstreamSettings(): Mono<ValidateUpstreamSettingsResult> {
@@ -178,7 +125,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
} }
return upstream.getIngressReader() return upstream.getIngressReader()
.read( .read(
JsonRpcRequest( ChainRequest(
"eth_call", "eth_call",
ListParams( ListParams(
TransactionCallJson( TransactionCallJson(
@@ -191,7 +138,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
), ),
), ),
) )
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { ValidateUpstreamSettingsResult.UPSTREAM_VALID } .map { ValidateUpstreamSettingsResult.UPSTREAM_VALID }
.onErrorResume { .onErrorResume {
if (it.message != null && it.message!!.contains("rpc.returndata.limit")) { if (it.message != null && it.message!!.contains("rpc.returndata.limit")) {
@@ -224,8 +171,8 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
.readArchiveBlock() .readArchiveBlock()
.flatMap { .flatMap {
upstream.getIngressReader() upstream.getIngressReader()
.read(JsonRpcRequest("eth_getBlockByNumber", ListParams(it, false))) .read(ChainRequest("eth_getBlockByNumber", ListParams(it, false)))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
} }
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn( log.warn(
@@ -250,7 +197,7 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
private fun chainId(): Mono<String> { private fun chainId(): Mono<String> {
return upstream.getIngressReader() return upstream.getIngressReader()
.read(JsonRpcRequest("eth_chainId", ListParams())) .read(ChainRequest("eth_chainId", ListParams()))
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn( log.warn(
"error during chainId retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " + "error during chainId retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " +
@@ -258,12 +205,12 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
) )
} }
.doOnError { log.error("Error during execution 'eth_chainId' - ${it.message} for ${upstream.getId()}") } .doOnError { log.error("Error during execution 'eth_chainId' - ${it.message} for ${upstream.getId()}") }
.flatMap(JsonRpcResponse::requireStringResult) .flatMap(ChainResponse::requireStringResult)
} }
private fun netVersion(): Mono<String> { private fun netVersion(): Mono<String> {
return upstream.getIngressReader() return upstream.getIngressReader()
.read(JsonRpcRequest("net_version", ListParams())) .read(ChainRequest("net_version", ListParams()))
.retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx ->
log.warn( log.warn(
"error during netVersion retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " + "error during netVersion retrieving for ${upstream.getId()}, iteration ${ctx.iteration()}, " +
@@ -271,6 +218,6 @@ open class EthereumUpstreamValidator @JvmOverloads constructor(
) )
} }
.doOnError { log.error("Error during execution 'net_version' - ${it.message} for ${upstream.getId()}") } .doOnError { log.error("Error during execution 'net_version' - ${it.message} for ${upstream.getId()}") }
.flatMap(JsonRpcResponse::requireStringResult) .flatMap(ChainResponse::requireStringResult)
} }
} }

View File

@@ -17,7 +17,7 @@
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
@@ -36,7 +36,7 @@ import java.util.concurrent.atomic.AtomicReference
class GenericWsHead( class GenericWsHead(
forkChoice: ForkChoice, forkChoice: ForkChoice,
blockValidator: BlockValidator, blockValidator: BlockValidator,
private val api: JsonRpcReader, private val api: ChainReader,
private val wsSubscriptions: WsSubscriptions, private val wsSubscriptions: WsSubscriptions,
private val wsConnectionResubscribeScheduler: Scheduler, private val wsConnectionResubscribeScheduler: Scheduler,
headScheduler: Scheduler, headScheduler: Scheduler,

View File

@@ -15,8 +15,8 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsMessage import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsMessage
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -27,7 +27,7 @@ interface WsConnection : AutoCloseable {
fun connectionId(): String fun connectionId(): String
fun getSubscribeResponses(): Flux<JsonRpcWsMessage> fun getSubscribeResponses(): Flux<JsonRpcWsMessage>
fun callRpc(originalRequest: JsonRpcRequest): Mono<JsonRpcResponse> fun callRpc(originalRequest: ChainRequest): Mono<ChainResponse>
fun connect() fun connect()
fun connectionInfoFlux(): Flux<ConnectionInfo> fun connectionInfoFlux(): Flux<ConnectionInfo>

View File

@@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.AuthConfig import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics import io.emeraldpay.dshackle.upstream.RequestMetrics
import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.Tag
@@ -22,7 +22,7 @@ open class WsConnectionFactory(
var basicAuth: AuthConfig.ClientBasicAuth? = null var basicAuth: AuthConfig.ClientBasicAuth? = null
var config: UpstreamsConfig.WsEndpoint? = null var config: UpstreamsConfig.WsEndpoint? = null
private fun metrics(connIndex: Int): RpcMetrics { private fun metrics(connIndex: Int): RequestMetrics {
val metricsTags = listOf( val metricsTags = listOf(
Tag.of("index", connIndex.toString()), Tag.of("index", connIndex.toString()),
Tag.of("upstream", id), Tag.of("upstream", id),
@@ -30,7 +30,7 @@ open class WsConnectionFactory(
Tag.of("chain", chain.chainCode), Tag.of("chain", chain.chainCode),
) )
return RpcMetrics( return RequestMetrics(
Timer.builder("upstream.ws.conn") Timer.builder("upstream.ws.conn")
.description("Request time through a WebSocket JSON RPC connection") .description("Request time through a WebSocket JSON RPC connection")
.tags(metricsTags) .tags(metricsTags)

View File

@@ -18,16 +18,16 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.AuthConfig import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.RequestMetrics
import io.emeraldpay.dshackle.upstream.ethereum.WsConnection.ConnectionState.CONNECTED import io.emeraldpay.dshackle.upstream.ethereum.WsConnection.ConnectionState.CONNECTED
import io.emeraldpay.dshackle.upstream.ethereum.WsConnection.ConnectionState.DISCONNECTED import io.emeraldpay.dshackle.upstream.ethereum.WsConnection.ConnectionState.DISCONNECTED
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsMessage import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsMessage
import io.emeraldpay.dshackle.upstream.rpcclient.ResponseWSParser import io.emeraldpay.dshackle.upstream.rpcclient.ResponseWSParser
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Metrics
import io.netty.buffer.ByteBufInputStream import io.netty.buffer.ByteBufInputStream
import io.netty.handler.codec.http.HttpHeaderNames import io.netty.handler.codec.http.HttpHeaderNames
@@ -64,7 +64,7 @@ open class WsConnectionImpl(
private val uri: URI, private val uri: URI,
private val origin: URI, private val origin: URI,
private val basicAuth: AuthConfig.ClientBasicAuth?, private val basicAuth: AuthConfig.ClientBasicAuth?,
private val rpcMetrics: RpcMetrics?, private val requestMetrics: RequestMetrics?,
private val scheduler: Scheduler, private val scheduler: Scheduler,
) : AutoCloseable, WsConnection, Cloneable { ) : AutoCloseable, WsConnection, Cloneable {
@@ -108,7 +108,7 @@ open class WsConnectionImpl(
private var rpcSend = Sinks private var rpcSend = Sinks
.many() .many()
.unicast() .unicast()
.onBackpressureBuffer<JsonRpcRequest>() .onBackpressureBuffer<ChainRequest>()
private val disconnects = Sinks private val disconnects = Sinks
.many() .many()
@@ -120,7 +120,7 @@ open class WsConnectionImpl(
.multicast() .multicast()
.directBestEffort<WsConnection.ConnectionInfo>() .directBestEffort<WsConnection.ConnectionInfo>()
private val currentRequests = ConcurrentHashMap<Int, Sinks.One<JsonRpcResponse>>() private val currentRequests = ConcurrentHashMap<Int, Sinks.One<ChainResponse>>()
private val connId = UUID.randomUUID().toString() private val connId = UUID.randomUUID().toString()
private val sendIdSeq = AtomicInteger(IDS_START) private val sendIdSeq = AtomicInteger(IDS_START)
@@ -168,7 +168,7 @@ open class WsConnectionImpl(
rpcSend = Sinks rpcSend = Sinks
.many() .many()
.unicast() .unicast()
.onBackpressureBuffer<JsonRpcRequest>() .onBackpressureBuffer()
resetBackoffTask?.cancel(false) resetBackoffTask?.cancel(false)
val retryInterval = currentBackOff.nextBackOff() val retryInterval = currentBackOff.nextBackOff()
resetBackoffTask = resetBackoffExecutor.schedule<Unit>({ resetBackoffTask = resetBackoffExecutor.schedule<Unit>({
@@ -305,7 +305,7 @@ open class WsConnectionImpl(
} }
private fun onMessageRpc(msg: ResponseWSParser.WsResponse) { private fun onMessageRpc(msg: ResponseWSParser.WsResponse) {
val rpcResponse = JsonRpcResponse( val rpcResponse = ChainResponse(
msg.value, msg.value,
msg.error, msg.error,
msg.id, msg.id,
@@ -346,7 +346,7 @@ open class WsConnectionImpl(
return Flux.from(subscriptionResponses.asFlux()) return Flux.from(subscriptionResponses.asFlux())
} }
override fun callRpc(originalRequest: JsonRpcRequest): Mono<JsonRpcResponse> { override fun callRpc(originalRequest: ChainRequest): Mono<ChainResponse> {
return Mono.fromCallable { return Mono.fromCallable {
val startTime = System.nanoTime() val startTime = System.nanoTime()
// use an internal id sequence, to avoid id conflicts with user calls // use an internal id sequence, to avoid id conflicts with user calls
@@ -358,7 +358,7 @@ open class WsConnectionImpl(
} }
} }
private fun sendRpc(request: JsonRpcRequest) { private fun sendRpc(request: ChainRequest) {
// submit to upstream in a separate thread, to free current thread (needs for subscription, etc) // submit to upstream in a separate thread, to free current thread (needs for subscription, etc)
sendExecutor.execute { sendExecutor.execute {
val result = rpcSend.tryEmitNext(request) val result = rpcSend.tryEmitNext(request)
@@ -368,13 +368,13 @@ open class WsConnectionImpl(
} }
} }
private fun waitForResponse(request: JsonRpcRequest, originalId: Int, startTime: Long): Mono<JsonRpcResponse> { private fun waitForResponse(request: ChainRequest, originalId: Int, startTime: Long): Mono<ChainResponse> {
val internalId = request.id.toLong() val internalId = request.id.toLong()
val onResponse = Sinks.one<JsonRpcResponse>() val onResponse = Sinks.one<ChainResponse>()
currentRequests[internalId.toInt()] = onResponse currentRequests[internalId.toInt()] = onResponse
val noResponse = JsonRpcException( val noResponse = ChainException(
JsonRpcResponse.Id.from(originalId), ChainResponse.Id.from(originalId),
JsonRpcError( ChainCallError(
RpcResponseError.CODE_INTERNAL_ERROR, RpcResponseError.CODE_INTERNAL_ERROR,
"Response not received from WebSocket", "Response not received from WebSocket",
), ),
@@ -384,10 +384,10 @@ open class WsConnectionImpl(
val failOnDisconnect = Mono.from(disconnects.asFlux()) val failOnDisconnect = Mono.from(disconnects.asFlux())
.flatMap { .flatMap {
Mono.error<JsonRpcResponse>( Mono.error<ChainResponse>(
JsonRpcException( ChainException(
JsonRpcResponse.Id.from(originalId), ChainResponse.Id.from(originalId),
JsonRpcError( ChainCallError(
RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR, RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR,
"Disconnected from WebSocket", "Disconnected from WebSocket",
), ),
@@ -398,9 +398,9 @@ open class WsConnectionImpl(
return Mono.from(onResponse.asMono()).or(failOnDisconnect) return Mono.from(onResponse.asMono()).or(failOnDisconnect)
.doOnSubscribe { sendRpc(request) } .doOnSubscribe { sendRpc(request) }
.take(Defaults.timeout) .take(Defaults.timeout)
.doOnNext { rpcMetrics?.timer?.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) } .doOnNext { requestMetrics?.timer?.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) }
.doOnError { rpcMetrics?.fails?.increment() } .doOnError { requestMetrics?.fails?.increment() }
.map { it.copyWithId(JsonRpcResponse.Id.from(originalId)) } .map { it.copyWithId(ChainResponse.Id.from(originalId)) }
.switchIfEmpty( .switchIfEmpty(
Mono.fromCallable { log.warn("No response for ${request.method} ${request.params}") }.then(Mono.error(noResponse)), Mono.fromCallable { log.warn("No response for ${request.method} ${request.params}") }.then(Mono.error(noResponse)),
) )
@@ -414,11 +414,11 @@ open class WsConnectionImpl(
connection?.dispose() connection?.dispose()
connection = null connection = null
currentRequests.clear() currentRequests.clear()
rpcMetrics?.fails?.let { requestMetrics?.fails?.let {
it.close() it.close()
Metrics.globalRegistry.remove(it) Metrics.globalRegistry.remove(it)
} }
rpcMetrics?.timer?.let { requestMetrics?.timer?.let {
it.close() it.close()
Metrics.globalRegistry.remove(it) Metrics.globalRegistry.remove(it)
} }

View File

@@ -15,8 +15,8 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.ChainResponse
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
@@ -41,11 +41,11 @@ interface WsSubscriptions {
/** /**
* Subscribe on remote * Subscribe on remote
*/ */
fun subscribe(request: JsonRpcRequest): SubscribeData fun subscribe(request: ChainRequest): SubscribeData
fun connectionInfoFlux(): Flux<WsConnection.ConnectionInfo> fun connectionInfoFlux(): Flux<WsConnection.ConnectionInfo>
fun unsubscribe(request: JsonRpcRequest): Mono<JsonRpcResponse> fun unsubscribe(request: ChainRequest): Mono<ChainResponse>
data class SubscribeData( data class SubscribeData(
val data: Flux<ByteArray>, val data: Flux<ByteArray>,

View File

@@ -15,9 +15,9 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
@@ -32,7 +32,7 @@ class WsSubscriptionsImpl(
private val log = LoggerFactory.getLogger(WsSubscriptionsImpl::class.java) private val log = LoggerFactory.getLogger(WsSubscriptionsImpl::class.java)
} }
override fun subscribe(request: JsonRpcRequest): WsSubscriptions.SubscribeData { override fun subscribe(request: ChainRequest): WsSubscriptions.SubscribeData {
val subscriptionId = AtomicReference("") val subscriptionId = AtomicReference("")
val conn = wsPool.getConnection() val conn = wsPool.getConnection()
val messages = conn.getSubscribeResponses() val messages = conn.getSubscribeResponses()
@@ -44,7 +44,7 @@ class WsSubscriptionsImpl(
.flatMapMany { .flatMapMany {
if (it.hasError()) { if (it.hasError()) {
log.warn("Failed to establish subscription: ${it.error?.message}") log.warn("Failed to establish subscription: ${it.error?.message}")
Mono.error(JsonRpcException(it.id, it.error!!)) Mono.error(ChainException(it.id, it.error!!))
} else { } else {
subscriptionId.set(it.getResultAsProcessedString()) subscriptionId.set(it.getResultAsProcessedString())
messages messages
@@ -54,7 +54,7 @@ class WsSubscriptionsImpl(
return WsSubscriptions.SubscribeData(messageFlux, conn.connectionId(), subscriptionId) return WsSubscriptions.SubscribeData(messageFlux, conn.connectionId(), subscriptionId)
} }
override fun unsubscribe(request: JsonRpcRequest): Mono<JsonRpcResponse> { override fun unsubscribe(request: ChainRequest): Mono<ChainResponse> {
if (request.params is ListParams && (request.params.list.isEmpty() || request.params.list.contains("")) if (request.params is ListParams && (request.params.list.isEmpty() || request.params.list.contains(""))
) { ) {
return Mono.empty() return Mono.empty()

View File

@@ -1,29 +1,21 @@
package io.emeraldpay.dshackle.upstream.ethereum.subscribe package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.module.kotlin.readValue
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global.Companion.objectMapper import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.upstream.BasicEthLabelsDetector
import io.emeraldpay.dshackle.upstream.LabelsDetector import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.ethereum.EthereumArchiveBlockNumberReader import io.emeraldpay.dshackle.upstream.ethereum.EthereumArchiveBlockNumberReader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
class EthereumLabelsDetector( class EthereumLabelsDetector(
private val reader: JsonRpcReader, private val reader: ChainReader,
private val chain: Chain, private val chain: Chain,
) : LabelsDetector { ) : BasicEthLabelsDetector(reader) {
private val blockNumberReader = EthereumArchiveBlockNumberReader(reader) private val blockNumberReader = EthereumArchiveBlockNumberReader(reader)
companion object {
private val log = LoggerFactory.getLogger(EthereumLabelsDetector::class.java)
}
override fun detectLabels(): Flux<Pair<String, String>> { override fun detectLabels(): Flux<Pair<String, String>> {
return Flux.merge( return Flux.merge(
detectNodeType(), detectNodeType(),
@@ -31,27 +23,6 @@ class EthereumLabelsDetector(
) )
} }
private fun detectNodeType(): Flux<Pair<String, String>?> {
return reader
.read(JsonRpcRequest("web3_clientVersion", ListParams()))
.flatMap(JsonRpcResponse::requireResult)
.map { objectMapper.readValue<JsonNode>(it) }
.flatMapMany { node ->
val labels = mutableListOf<Pair<String, String>>()
if (node.isTextual) {
clientType(node.textValue())?.let {
labels.add("client_type" to it)
}
clientVersion(node.textValue())?.let {
labels.add("client_version" to it)
}
}
Flux.fromIterable(labels)
}
.onErrorResume { Flux.empty() }
}
private fun detectArchiveNode(): Mono<Pair<String, String>> { private fun detectArchiveNode(): Mono<Pair<String, String>> {
return Mono.zip( return Mono.zip(
blockNumberReader.readEarliestBlock(chain).flatMap { haveBalance(it) }, blockNumberReader.readEarliestBlock(chain).flatMap { haveBalance(it) },
@@ -63,34 +34,16 @@ class EthereumLabelsDetector(
private fun haveBalance(blockNumber: String): Mono<ByteArray> { private fun haveBalance(blockNumber: String): Mono<ByteArray> {
return reader.read( return reader.read(
JsonRpcRequest( ChainRequest(
"eth_getBalance", "eth_getBalance",
ListParams("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0", blockNumber), ListParams("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0", blockNumber),
), ),
).flatMap(JsonRpcResponse::requireResult) ).flatMap(ChainResponse::requireResult)
} }
private fun clientVersion(client: String): String? { override fun nodeTypeRequest(): NodeTypeRequest {
val firstSlash = client.indexOf("/") return NodeTypeRequest(
val secondSlash = client.indexOf("/", firstSlash + 1) ChainRequest("web3_clientVersion", ListParams()),
if (firstSlash == -1 || secondSlash == -1 || secondSlash < firstSlash) { ) { node -> node }
return null
}
return client.substring(firstSlash + 1, secondSlash)
}
private fun clientType(client: String): String? {
return if (client.contains("erigon", true)) {
"erigon"
} else if (client.contains("geth", true)) {
"geth"
} else if (client.contains("bor", true)) {
"bor"
} else if (client.contains("nethermind", true)) {
"nethermind"
} else {
log.debug("Unknown client type: {}", client)
null
}
} }
} }

View File

@@ -15,11 +15,11 @@
*/ */
package io.emeraldpay.dshackle.upstream.ethereum.subscribe package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.SubscriptionConnect import io.emeraldpay.dshackle.upstream.SubscriptionConnect
import io.emeraldpay.dshackle.upstream.ethereum.EthereumEgressSubscription import io.emeraldpay.dshackle.upstream.ethereum.EthereumEgressSubscription
import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions
import io.emeraldpay.dshackle.upstream.ethereum.domain.TransactionId import io.emeraldpay.dshackle.upstream.ethereum.domain.TransactionId
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
@@ -35,7 +35,7 @@ class WebsocketPendingTxes(
} }
override fun createConnection(): Flux<TransactionId> { override fun createConnection(): Flux<TransactionId> {
return wsSubscriptions.subscribe(JsonRpcRequest("eth_subscribe", ListParams(EthereumEgressSubscription.METHOD_PENDING_TXES))) return wsSubscriptions.subscribe(ChainRequest("eth_subscribe", ListParams(EthereumEgressSubscription.METHOD_PENDING_TXES)))
.data .data
.timeout(Duration.ofSeconds(60), Mono.empty()) .timeout(Duration.ofSeconds(60), Mono.empty())
.map { .map {

View File

@@ -3,8 +3,9 @@ package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.CachingReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.EgressSubscription import io.emeraldpay.dshackle.upstream.EgressSubscription
import io.emeraldpay.dshackle.upstream.EmptyEgressSubscription import io.emeraldpay.dshackle.upstream.EmptyEgressSubscription
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
@@ -17,7 +18,6 @@ import io.emeraldpay.dshackle.upstream.NoopCachingReader
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.CallSelector import io.emeraldpay.dshackle.upstream.calls.CallSelector
import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import org.springframework.cloud.sleuth.Tracer import org.springframework.cloud.sleuth.Tracer
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import reactor.core.scheduler.Scheduler import reactor.core.scheduler.Scheduler
@@ -29,7 +29,7 @@ abstract class AbstractChainSpecific : ChainSpecific {
methods: CallMethods, methods: CallMethods,
head: Head, head: Head,
logsOracle: LogsOracle?, logsOracle: LogsOracle?,
): Mono<JsonRpcReader> { ): Mono<ChainReader> {
return Mono.just(LocalReader(methods)) return Mono.just(LocalReader(methods))
} }
@@ -37,7 +37,7 @@ abstract class AbstractChainSpecific : ChainSpecific {
return { _, _, _ -> NoopCachingReader } return { _, _, _ -> NoopCachingReader }
} }
override fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector? { override fun labelDetector(chain: Chain, reader: ChainReader): LabelsDetector? {
return null return null
} }
@@ -56,13 +56,13 @@ abstract class AbstractChainSpecific : ChainSpecific {
abstract class AbstractPollChainSpecific : AbstractChainSpecific() { abstract class AbstractPollChainSpecific : AbstractChainSpecific() {
override fun getLatestBlock(api: JsonRpcReader, upstreamId: String): Mono<BlockContainer> { override fun getLatestBlock(api: ChainReader, upstreamId: String): Mono<BlockContainer> {
return api.read(latestBlockRequest()).map { return api.read(latestBlockRequest()).map {
parseBlock(it.getResult(), upstreamId) parseBlock(it.getResult(), upstreamId)
} }
} }
abstract fun latestBlockRequest(): JsonRpcRequest abstract fun latestBlockRequest(): ChainRequest
abstract fun parseBlock(data: ByteArray, upstreamId: String): BlockContainer abstract fun parseBlock(data: ByteArray, upstreamId: String): BlockContainer
} }

View File

@@ -2,6 +2,7 @@ package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.BlockchainType.BITCOIN import io.emeraldpay.dshackle.BlockchainType.BITCOIN
import io.emeraldpay.dshackle.BlockchainType.ETHEREUM import io.emeraldpay.dshackle.BlockchainType.ETHEREUM
import io.emeraldpay.dshackle.BlockchainType.ETHEREUM_BEACON_CHAIN
import io.emeraldpay.dshackle.BlockchainType.NEAR import io.emeraldpay.dshackle.BlockchainType.NEAR
import io.emeraldpay.dshackle.BlockchainType.POLKADOT import io.emeraldpay.dshackle.BlockchainType.POLKADOT
import io.emeraldpay.dshackle.BlockchainType.SOLANA import io.emeraldpay.dshackle.BlockchainType.SOLANA
@@ -12,8 +13,9 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.CachingReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.EgressSubscription import io.emeraldpay.dshackle.upstream.EgressSubscription
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.IngressSubscription
@@ -23,13 +25,13 @@ import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamValidator import io.emeraldpay.dshackle.upstream.UpstreamValidator
import io.emeraldpay.dshackle.upstream.beaconchain.BeaconChainSpecific
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.CallSelector import io.emeraldpay.dshackle.upstream.calls.CallSelector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific
import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions
import io.emeraldpay.dshackle.upstream.near.NearChainSpecific import io.emeraldpay.dshackle.upstream.near.NearChainSpecific
import io.emeraldpay.dshackle.upstream.polkadot.PolkadotChainSpecific import io.emeraldpay.dshackle.upstream.polkadot.PolkadotChainSpecific
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.solana.SolanaChainSpecific import io.emeraldpay.dshackle.upstream.solana.SolanaChainSpecific
import io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific import io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific
import org.apache.commons.collections4.Factory import org.apache.commons.collections4.Factory
@@ -38,19 +40,19 @@ import reactor.core.publisher.Mono
import reactor.core.scheduler.Scheduler import reactor.core.scheduler.Scheduler
typealias SubscriptionBuilder = (Multistream) -> EgressSubscription typealias SubscriptionBuilder = (Multistream) -> EgressSubscription
typealias LocalReaderBuilder = (CachingReader, CallMethods, Head, LogsOracle?) -> Mono<JsonRpcReader> typealias LocalReaderBuilder = (CachingReader, CallMethods, Head, LogsOracle?) -> Mono<ChainReader>
typealias CachingReaderBuilder = (Multistream, Caches, Factory<CallMethods>) -> CachingReader typealias CachingReaderBuilder = (Multistream, Caches, Factory<CallMethods>) -> CachingReader
interface ChainSpecific { interface ChainSpecific {
fun parseHeader(data: ByteArray, upstreamId: String): BlockContainer fun parseHeader(data: ByteArray, upstreamId: String): BlockContainer
fun getLatestBlock(api: JsonRpcReader, upstreamId: String): Mono<BlockContainer> fun getLatestBlock(api: ChainReader, upstreamId: String): Mono<BlockContainer>
fun listenNewHeadsRequest(): JsonRpcRequest fun listenNewHeadsRequest(): ChainRequest
fun unsubscribeNewHeadsRequest(subId: String): JsonRpcRequest fun unsubscribeNewHeadsRequest(subId: String): ChainRequest
fun localReaderBuilder(cachingReader: CachingReader, methods: CallMethods, head: Head, logsOracle: LogsOracle?): Mono<JsonRpcReader> fun localReaderBuilder(cachingReader: CachingReader, methods: CallMethods, head: Head, logsOracle: LogsOracle?): Mono<ChainReader>
fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription
@@ -58,7 +60,7 @@ interface ChainSpecific {
fun validator(chain: Chain, upstream: Upstream, options: ChainOptions.Options, config: ChainConfig): UpstreamValidator fun validator(chain: Chain, upstream: Upstream, options: ChainOptions.Options, config: ChainConfig): UpstreamValidator
fun labelDetector(chain: Chain, reader: JsonRpcReader): LabelsDetector? fun labelDetector(chain: Chain, reader: ChainReader): LabelsDetector?
fun makeIngressSubscription(ws: WsSubscriptions): IngressSubscription fun makeIngressSubscription(ws: WsSubscriptions): IngressSubscription
@@ -77,6 +79,7 @@ object ChainSpecificRegistry {
POLKADOT -> PolkadotChainSpecific POLKADOT -> PolkadotChainSpecific
SOLANA -> SolanaChainSpecific SOLANA -> SolanaChainSpecific
NEAR -> NearChainSpecific NEAR -> NearChainSpecific
ETHEREUM_BEACON_CHAIN -> BeaconChainSpecific
BITCOIN -> throw IllegalArgumentException("bitcoin should use custom streams implementation") BITCOIN -> throw IllegalArgumentException("bitcoin should use custom streams implementation")
UNKNOWN -> throw IllegalArgumentException("unknown chain") UNKNOWN -> throw IllegalArgumentException("unknown chain")
} }

View File

@@ -17,7 +17,7 @@ package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.AbstractHead import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
@@ -33,7 +33,7 @@ open class GenericHead(
private val chainSpecific: ChainSpecific, private val chainSpecific: ChainSpecific,
) : Head, AbstractHead(forkChoice, headScheduler, blockValidator, 60_000, upstreamId) { ) : Head, AbstractHead(forkChoice, headScheduler, blockValidator, 60_000, upstreamId) {
fun getLatestBlock(api: JsonRpcReader): Mono<BlockContainer> { fun getLatestBlock(api: ChainReader): Mono<BlockContainer> {
return chainSpecific.getLatestBlock(api, upstreamId) return chainSpecific.getLatestBlock(api, upstreamId)
.subscribeOn(headScheduler) .subscribeOn(headScheduler)
.timeout(Defaults.timeout, Mono.error(Exception("Block data not received"))) .timeout(Defaults.timeout, Mono.error(Exception("Block data not received")))

View File

@@ -1,10 +1,10 @@
package io.emeraldpay.dshackle.upstream.generic package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.SubscriptionConnect import io.emeraldpay.dshackle.upstream.SubscriptionConnect
import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions
import io.emeraldpay.dshackle.upstream.generic.subscribe.GenericPersistentConnect import io.emeraldpay.dshackle.upstream.generic.subscribe.GenericPersistentConnect
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -38,7 +38,7 @@ class GenericSubscriptionConnect(
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
override fun createConnection(): Flux<Any> { override fun createConnection(): Flux<Any> {
return conn.subscribe(JsonRpcRequest(topic, ListParams(getParams(params)))) return conn.subscribe(ChainRequest(topic, ListParams(getParams(params))))
.data .data
.timeout(Duration.ofSeconds(60), Mono.empty()) .timeout(Duration.ofSeconds(60), Mono.empty())
.onErrorResume { Mono.empty() } as Flux<Any> .onErrorResume { Mono.empty() } as Flux<Any>

View File

@@ -21,7 +21,7 @@ import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.IndexConfig import io.emeraldpay.dshackle.config.IndexConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.CachingReader
import io.emeraldpay.dshackle.upstream.DistanceExtractor import io.emeraldpay.dshackle.upstream.DistanceExtractor
import io.emeraldpay.dshackle.upstream.DynamicMergedHead import io.emeraldpay.dshackle.upstream.DynamicMergedHead
@@ -166,7 +166,7 @@ open class GenericMultistream(
return this as T return this as T
} }
override fun getLocalReader(): Mono<JsonRpcReader> { override fun getLocalReader(): Mono<ChainReader> {
return localReaderBuilder(cachingReader, getMethods(), getHead(), logsOracle) return localReaderBuilder(cachingReader, getMethods(), getHead(), logsOracle)
} }

View File

@@ -16,7 +16,7 @@
*/ */
package io.emeraldpay.dshackle.upstream.generic package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
@@ -26,7 +26,7 @@ import reactor.core.scheduler.Scheduler
import java.time.Duration import java.time.Duration
class GenericRpcHead( class GenericRpcHead(
private val api: JsonRpcReader, private val api: ChainReader,
forkChoice: ForkChoice, forkChoice: ForkChoice,
upstreamId: String, upstreamId: String,
blockValidator: BlockValidator, blockValidator: BlockValidator,

View File

@@ -5,7 +5,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig.Labels import io.emeraldpay.dshackle.config.UpstreamsConfig.Labels
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent.ChangeType.UPDATED import io.emeraldpay.dshackle.startup.UpstreamChangeEvent.ChangeType.UPDATED
@@ -65,7 +65,7 @@ open class GenericUpstream(
return connector.getHead() return connector.getHead()
} }
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
return connector.getIngressReader() return connector.getIngressReader()
} }

View File

@@ -2,14 +2,13 @@ package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.SingleCallValidator import io.emeraldpay.dshackle.upstream.SingleCallValidator
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.UpstreamValidator import io.emeraldpay.dshackle.upstream.UpstreamValidator
import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult
import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult.UPSTREAM_VALID import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult.UPSTREAM_VALID
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.util.concurrent.TimeoutException import java.util.concurrent.TimeoutException
@@ -19,13 +18,10 @@ class GenericUpstreamValidator(
private val validator: SingleCallValidator, private val validator: SingleCallValidator,
) : UpstreamValidator(upstream, options) { ) : UpstreamValidator(upstream, options) {
companion object {
private val log = LoggerFactory.getLogger(GenericUpstreamValidator::class.java)
}
override fun validate(): Mono<UpstreamAvailability> { override fun validate(): Mono<UpstreamAvailability> {
return upstream.getIngressReader() return upstream.getIngressReader()
.read(validator.method) .read(validator.method)
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { validator.check(it) } .map { validator.check(it) }
.timeout( .timeout(
Defaults.timeoutInternal, Defaults.timeoutInternal,

View File

@@ -1,18 +1,18 @@
package io.emeraldpay.dshackle.upstream.generic package io.emeraldpay.dshackle.upstream.generic
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
class LocalReader(private val methods: CallMethods) : JsonRpcReader { class LocalReader(private val methods: CallMethods) : ChainReader {
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> { override fun read(key: ChainRequest): Mono<ChainResponse> {
if (methods.isHardcoded(key.method)) { if (methods.isHardcoded(key.method)) {
return Mono.just(methods.executeHardcoded(key.method)) return Mono.just(methods.executeHardcoded(key.method))
.map { JsonRpcResponse(it, null) } .map { ChainResponse(it, null) }
} }
if (!methods.isCallable(key.method)) { if (!methods.isCallable(key.method)) {
return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method"))

View File

@@ -1,6 +1,6 @@
package io.emeraldpay.dshackle.upstream.generic.connectors package io.emeraldpay.dshackle.upstream.generic.connectors
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
@@ -11,7 +11,7 @@ interface GenericConnector : Lifecycle {
fun hasLiveSubscriptionHead(): Flux<Boolean> fun hasLiveSubscriptionHead(): Flux<Boolean>
fun getIngressReader(): JsonRpcReader fun getIngressReader(): ChainReader
fun getIngressSubscription(): IngressSubscription fun getIngressSubscription(): IngressSubscription
} }

View File

@@ -2,11 +2,11 @@ package io.emeraldpay.dshackle.upstream.generic.connectors
import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.reader.JsonRpcHttpReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.reader.JsonRpcReader
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HttpReader
import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.IngressSubscription
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.MergedHead import io.emeraldpay.dshackle.upstream.MergedHead
@@ -33,7 +33,7 @@ import java.time.Duration
class GenericRpcConnector( class GenericRpcConnector(
connectorType: ConnectorMode, connectorType: ConnectorMode,
private val directReader: JsonRpcHttpReader, private val directReader: HttpReader,
wsFactory: WsConnectionPoolFactory?, wsFactory: WsConnectionPoolFactory?,
upstream: DefaultUpstream, upstream: DefaultUpstream,
forkChoice: ForkChoice, forkChoice: ForkChoice,
@@ -152,7 +152,7 @@ class GenericRpcConnector(
directReader.onStop() directReader.onStop()
} }
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
return directReader return directReader
} }

View File

@@ -1,6 +1,6 @@
package io.emeraldpay.dshackle.upstream.generic.connectors package io.emeraldpay.dshackle.upstream.generic.connectors
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
@@ -29,7 +29,7 @@ class GenericWsConnector(
chainSpecific: ChainSpecific, chainSpecific: ChainSpecific,
) : GenericConnector { ) : GenericConnector {
private val pool: WsConnectionPool private val pool: WsConnectionPool
private val reader: JsonRpcReader private val reader: ChainReader
private val head: GenericWsHead private val head: GenericWsHead
private val subscriptions: IngressSubscription private val subscriptions: IngressSubscription
private val liveness: HeadLivenessValidator private val liveness: HeadLivenessValidator
@@ -68,7 +68,7 @@ class GenericWsConnector(
head.stop() head.stop()
} }
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
return reader return reader
} }

View File

@@ -0,0 +1,47 @@
package io.emeraldpay.dshackle.upstream.generic.connectors
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.HttpFactory
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry
import reactor.core.scheduler.Scheduler
import reactor.core.scheduler.Schedulers
import java.time.Duration
class RestConnectorFactory(
private val httpFactory: HttpFactory?,
private val forkChoice: ForkChoice,
private val blockValidator: BlockValidator,
private val headScheduler: Scheduler,
private val headLivenessScheduler: Scheduler,
private val expectedBlockTime: Duration,
) : ConnectorFactory {
override fun create(upstream: DefaultUpstream, chain: Chain): GenericConnector {
val specific = ChainSpecificRegistry.resolve(chain)
if (httpFactory == null) {
throw IllegalArgumentException("No http endpoint")
}
return GenericRpcConnector(
GenericConnectorFactory.ConnectorMode.RPC_ONLY,
httpFactory.create(upstream.getId(), chain),
null,
upstream,
forkChoice,
blockValidator,
Schedulers.single(),
headScheduler,
headLivenessScheduler,
expectedBlockTime,
specific,
)
}
override fun isValid(): Boolean {
return httpFactory != null
}
}

View File

@@ -24,9 +24,11 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.BuildInfo import io.emeraldpay.dshackle.upstream.BuildInfo
import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
@@ -37,8 +39,6 @@ import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.reactivestreams.Publisher import org.reactivestreams.Publisher
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
@@ -70,7 +70,7 @@ class BitcoinGrpcUpstream(
Lifecycle { Lifecycle {
private val extractBlock = ExtractBlock() private val extractBlock = ExtractBlock()
private val defaultReader: JsonRpcReader = client.getReader() private val defaultReader: ChainReader = client.getReader()
private val blockConverter: Function<BlockchainOuterClass.ChainHead, BlockContainer> = Function { value -> private val blockConverter: Function<BlockchainOuterClass.ChainHead, BlockContainer> = Function { value ->
val parentHash = val parentHash =
if (value.parentBlockId.isBlank()) { if (value.parentBlockId.isBlank()) {
@@ -94,8 +94,8 @@ class BitcoinGrpcUpstream(
private val reloadBlock: Function<BlockContainer, Publisher<BlockContainer>> = Function { existingBlock -> private val reloadBlock: Function<BlockContainer, Publisher<BlockContainer>> = Function { existingBlock ->
// head comes without transaction data // head comes without transaction data
// need to download transactions for the block // need to download transactions for the block
defaultReader.read(JsonRpcRequest("getblock", ListParams(existingBlock.hash.toHex()))) defaultReader.read(ChainRequest("getblock", ListParams(existingBlock.hash.toHex())))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map(extractBlock::extract) .map(extractBlock::extract)
.timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream")))
.doOnError { t -> .doOnError { t ->
@@ -134,7 +134,7 @@ class BitcoinGrpcUpstream(
return grpcHead return grpcHead
} }
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
return defaultReader return defaultReader
} }

View File

@@ -24,7 +24,7 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.BuildInfo import io.emeraldpay.dshackle.upstream.BuildInfo
import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.Capability
@@ -101,7 +101,7 @@ open class GenericGrpcUpstream(
private var capabilities: Set<Capability> = emptySet() private var capabilities: Set<Capability> = emptySet()
private val buildInfo: BuildInfo = BuildInfo() private val buildInfo: BuildInfo = BuildInfo()
private val defaultReader: JsonRpcReader = client.getReader() private val defaultReader: ChainReader = client.getReader()
override fun start() { override fun start() {
} }
@@ -159,7 +159,7 @@ open class GenericGrpcUpstream(
return grpcHead return grpcHead
} }
override fun getIngressReader(): JsonRpcReader { override fun getIngressReader(): ChainReader {
return defaultReader return defaultReader
} }

View File

@@ -34,13 +34,13 @@ import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.RequestMetrics
import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.grpc.auth.AuthException import io.emeraldpay.dshackle.upstream.grpc.auth.AuthException
import io.emeraldpay.dshackle.upstream.grpc.auth.ClientAuthenticationInterceptor import io.emeraldpay.dshackle.upstream.grpc.auth.ClientAuthenticationInterceptor
import io.emeraldpay.dshackle.upstream.grpc.auth.GrpcAuthContext import io.emeraldpay.dshackle.upstream.grpc.auth.GrpcAuthContext
import io.emeraldpay.dshackle.upstream.grpc.auth.GrpcUpstreamsAuth import io.emeraldpay.dshackle.upstream.grpc.auth.GrpcUpstreamsAuth
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.grpc.ClientInterceptor import io.grpc.ClientInterceptor
import io.grpc.Codec import io.grpc.Codec
import io.grpc.Status import io.grpc.Status
@@ -279,13 +279,13 @@ class GrpcUpstreams(
return getOrCreate(chain, metrics, creator) return getOrCreate(chain, metrics, creator)
} }
private fun makeMetrics(chain: Chain): RpcMetrics { private fun makeMetrics(chain: Chain): RequestMetrics {
val metricsTags = listOf( val metricsTags = listOf(
Tag.of("upstream", id), Tag.of("upstream", id),
Tag.of("chain", chain.chainCode), Tag.of("chain", chain.chainCode),
) )
return RpcMetrics( return RequestMetrics(
Timer.builder("upstream.grpc.conn") Timer.builder("upstream.grpc.conn")
.description("Request time through a Dshackle/gRPC connection") .description("Request time through a Dshackle/gRPC connection")
.tags(metricsTags) .tags(metricsTags)
@@ -300,7 +300,7 @@ class GrpcUpstreams(
private fun getOrCreate( private fun getOrCreate(
chain: Chain, chain: Chain,
metrics: RpcMetrics, metrics: RequestMetrics,
creator: (chain: Chain, client: JsonRpcGrpcClient) -> DefaultUpstream, creator: (chain: Chain, client: JsonRpcGrpcClient) -> DefaultUpstream,
): UpstreamChangeEvent { ): UpstreamChangeEvent {
lock.withLock { lock.withLock {

View File

@@ -8,6 +8,7 @@ import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions.Options import io.emeraldpay.dshackle.foundation.ChainOptions.Options
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.SingleCallValidator import io.emeraldpay.dshackle.upstream.SingleCallValidator
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
@@ -15,7 +16,6 @@ import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.UpstreamValidator import io.emeraldpay.dshackle.upstream.UpstreamValidator
import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific
import io.emeraldpay.dshackle.upstream.generic.GenericUpstreamValidator import io.emeraldpay.dshackle.upstream.generic.GenericUpstreamValidator
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import io.emeraldpay.dshackle.upstream.rpcclient.ObjectParams import io.emeraldpay.dshackle.upstream.rpcclient.ObjectParams
import java.math.BigInteger import java.math.BigInteger
@@ -44,11 +44,11 @@ object NearChainSpecific : AbstractPollChainSpecific() {
throw NotImplementedError() throw NotImplementedError()
} }
override fun listenNewHeadsRequest(): JsonRpcRequest { override fun listenNewHeadsRequest(): ChainRequest {
throw NotImplementedError() throw NotImplementedError()
} }
override fun unsubscribeNewHeadsRequest(subId: String): JsonRpcRequest { override fun unsubscribeNewHeadsRequest(subId: String): ChainRequest {
throw NotImplementedError() throw NotImplementedError()
} }
@@ -62,7 +62,7 @@ object NearChainSpecific : AbstractPollChainSpecific() {
upstream, upstream,
options, options,
SingleCallValidator( SingleCallValidator(
JsonRpcRequest("status", ListParams()), ChainRequest("status", ListParams()),
) { data -> ) { data ->
validate(data) validate(data)
}, },
@@ -82,8 +82,8 @@ object NearChainSpecific : AbstractPollChainSpecific() {
} }
} }
override fun latestBlockRequest(): JsonRpcRequest = // {...} override fun latestBlockRequest(): ChainRequest = // {...}
JsonRpcRequest("block", ObjectParams("finality" to "optimistic")) ChainRequest("block", ObjectParams("finality" to "optimistic"))
} }
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)

View File

@@ -2,9 +2,9 @@ package io.emeraldpay.dshackle.upstream.near
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector import io.emeraldpay.dshackle.upstream.LowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -14,7 +14,7 @@ class NearLowerBoundBlockDetector(
) : LowerBoundBlockDetector(chain, upstream) { ) : LowerBoundBlockDetector(chain, upstream) {
override fun lowerBlockDetect(): Mono<LowerBlockData> { override fun lowerBlockDetect(): Mono<LowerBlockData> {
return upstream.getIngressReader().read(JsonRpcRequest("status", ListParams())).map { return upstream.getIngressReader().read(ChainRequest("status", ListParams())).map {
val resp = Global.objectMapper.readValue(it.getResult(), NearStatus::class.java) val resp = Global.objectMapper.readValue(it.getResult(), NearStatus::class.java)
LowerBlockData(resp.syncInfo.earliestHeight, null, resp.syncInfo.earliestBlockTime.toEpochMilli()) LowerBlockData(resp.syncInfo.earliestHeight, null, resp.syncInfo.earliestBlockTime.toEpochMilli())
} }

View File

@@ -8,8 +8,9 @@ import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig
import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.foundation.ChainOptions.Options import io.emeraldpay.dshackle.foundation.ChainOptions.Options
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.CachingReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.EgressSubscription import io.emeraldpay.dshackle.upstream.EgressSubscription
import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.IngressSubscription import io.emeraldpay.dshackle.upstream.IngressSubscription
@@ -28,7 +29,6 @@ import io.emeraldpay.dshackle.upstream.generic.GenericEgressSubscription
import io.emeraldpay.dshackle.upstream.generic.GenericIngressSubscription import io.emeraldpay.dshackle.upstream.generic.GenericIngressSubscription
import io.emeraldpay.dshackle.upstream.generic.GenericUpstreamValidator import io.emeraldpay.dshackle.upstream.generic.GenericUpstreamValidator
import io.emeraldpay.dshackle.upstream.generic.LocalReader import io.emeraldpay.dshackle.upstream.generic.LocalReader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -66,21 +66,21 @@ object PolkadotChainSpecific : AbstractPollChainSpecific() {
) )
} }
override fun latestBlockRequest(): JsonRpcRequest = override fun latestBlockRequest(): ChainRequest =
JsonRpcRequest("chain_getBlock", ListParams()) ChainRequest("chain_getBlock", ListParams())
override fun listenNewHeadsRequest(): JsonRpcRequest = override fun listenNewHeadsRequest(): ChainRequest =
JsonRpcRequest("chain_subscribeNewHeads", ListParams()) ChainRequest("chain_subscribeNewHeads", ListParams())
override fun unsubscribeNewHeadsRequest(subId: String): JsonRpcRequest = override fun unsubscribeNewHeadsRequest(subId: String): ChainRequest =
JsonRpcRequest("chain_unsubscribeNewHeads", ListParams(subId)) ChainRequest("chain_unsubscribeNewHeads", ListParams(subId))
override fun localReaderBuilder( override fun localReaderBuilder(
cachingReader: CachingReader, cachingReader: CachingReader,
methods: CallMethods, methods: CallMethods,
head: Head, head: Head,
logsOracle: LogsOracle?, logsOracle: LogsOracle?,
): Mono<JsonRpcReader> { ): Mono<ChainReader> {
return Mono.just(LocalReader(methods)) return Mono.just(LocalReader(methods))
} }
@@ -98,7 +98,7 @@ object PolkadotChainSpecific : AbstractPollChainSpecific() {
upstream, upstream,
options, options,
SingleCallValidator( SingleCallValidator(
JsonRpcRequest("system_health", ListParams()), ChainRequest("system_health", ListParams()),
) { data -> ) { data ->
validate(data, options.minPeers, upstream.getId()) validate(data, options.minPeers, upstream.getId())
}, },

View File

@@ -1,10 +1,10 @@
package io.emeraldpay.dshackle.upstream.polkadot package io.emeraldpay.dshackle.upstream.polkadot
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.RecursiveLowerBoundBlockDetector import io.emeraldpay.dshackle.upstream.RecursiveLowerBoundBlockDetector
import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.rpcclient.ListParams
import io.emeraldpay.dshackle.upstream.toHex import io.emeraldpay.dshackle.upstream.toHex
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
@@ -22,25 +22,25 @@ class PolkadotLowerBoundBlockDetector(
override fun hasState(blockNumber: Long): Mono<Boolean> { override fun hasState(blockNumber: Long): Mono<Boolean> {
return upstream.getIngressReader().read( return upstream.getIngressReader().read(
JsonRpcRequest( ChainRequest(
"chain_getBlockHash", "chain_getBlockHash",
ListParams(blockNumber.toHex()), // in polkadot state methods work only with hash ListParams(blockNumber.toHex()), // in polkadot state methods work only with hash
), ),
) )
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { .map {
String(it, 1, it.size - 2) String(it, 1, it.size - 2)
} }
.flatMap { .flatMap {
upstream.getIngressReader().read( upstream.getIngressReader().read(
JsonRpcRequest( ChainRequest(
"state_getMetadata", "state_getMetadata",
ListParams(it), ListParams(it),
), ),
) )
} }
.retryWhen(retrySpec(nonRetryableErrors)) .retryWhen(retrySpec(nonRetryableErrors))
.flatMap(JsonRpcResponse::requireResult) .flatMap(ChainResponse::requireResult)
.map { true } .map { true }
.onErrorReturn(false) .onErrorReturn(false)
} }

View File

@@ -0,0 +1,112 @@
package io.emeraldpay.dshackle.upstream.restclient
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.HttpReader
import io.emeraldpay.dshackle.upstream.RequestMetrics
import io.emeraldpay.dshackle.upstream.rpcclient.ResponseRpcParser
import io.emeraldpay.dshackle.upstream.rpcclient.RestParams
import io.emeraldpay.dshackle.upstream.stream.AggregateResponse
import io.emeraldpay.dshackle.upstream.stream.Chunk
import io.emeraldpay.dshackle.upstream.stream.Response
import io.emeraldpay.dshackle.upstream.stream.StreamResponse
import io.netty.buffer.Unpooled
import io.netty.handler.codec.http.HttpMethod
import org.apache.commons.lang3.time.StopWatch
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.kotlin.core.publisher.switchIfEmpty
import java.util.concurrent.TimeUnit
class RestHttpReader(
target: String,
metrics: RequestMetrics,
basicAuth: AuthConfig.ClientBasicAuth? = null,
tlsCAAuth: ByteArray? = null,
) : HttpReader(target, metrics, basicAuth, tlsCAAuth) {
private val parser = ResponseRpcParser()
private val requestParser = RestRequestParser
override fun internalRead(key: ChainRequest): Mono<ChainResponse> {
val startTime = StopWatch()
return Mono.just(key)
.doOnNext {
if (!startTime.isStarted) {
startTime.start()
}
}
.flatMap(this::execute)
.doOnNext {
if (startTime.isStarted) {
metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
}
}
.handle { it, sink ->
when (it) {
is StreamResponse -> sink.next(ChainResponse(it.stream, key.id))
is AggregateResponse -> {
if (it.code != 200) {
val error = parser.readError(Global.objectMapper.createParser(it.response))
sink.next(ChainResponse(null, error))
} else {
sink.next(ChainResponse(it.response, null))
}
}
else -> sink.error(IllegalStateException("Wrong response type"))
}
}
}
private fun execute(key: ChainRequest): Mono<out Response> {
val restParams = key.params as RestParams
val methodParams = key.method.split("#")
val restMethod = methodParams[0]
val path = methodParams[1]
val url = target
.plus(requestParser.transformPathParams(path, restParams.pathParams))
.plus(requestParser.transformQueryParams(restParams.queryParams))
val response = httpClient.headers { headers ->
restParams.headers.forEach {
headers.add(it.key, it.value)
}
}
.request(HttpMethod.valueOf(restMethod))
.uri(url)
.send(Mono.just(Unpooled.wrappedBuffer(key.toJson())))
return if (!key.isStreamed) {
response.response { header, bytes ->
val statusCode = header.status().code()
bytes.aggregate().asByteArray().map {
AggregateResponse(it, statusCode)
}.switchIfEmpty {
Mono.just(AggregateResponse(ByteArray(0), statusCode))
}
}.single()
} else {
response.responseConnection { t, u ->
if (t.status().code() != 200) {
u.inbound().receive().aggregate().asByteArray()
.map { AggregateResponse(it, t.status().code()) }
} else {
Mono.just(
StreamResponse(
Flux.concat(
u.inbound().receive().asByteArray()
.map { Chunk(it, false) },
Mono.just(Chunk(ByteArray(0), true)),
),
),
)
}
}.single()
}
}
}

View File

@@ -0,0 +1,39 @@
package io.emeraldpay.dshackle.upstream.restclient
import java.util.ArrayDeque
import java.util.Queue
object RestRequestParser {
fun transformPathParams(path: String, pathParams: List<String>): String {
if (pathParams.isEmpty()) {
return path
}
val params: Queue<String> = ArrayDeque(pathParams)
val paramPlaceholder = '*'
var occurrenceIndex = path.indexOf(paramPlaceholder)
if (occurrenceIndex < 0) {
return path
}
val builder = StringBuilder()
var i = 0
while (occurrenceIndex >= 0 && params.isNotEmpty()) {
val param = params.poll()
builder.append(path, i, occurrenceIndex).append(param)
i = occurrenceIndex + 1
occurrenceIndex = path.indexOf(paramPlaceholder, occurrenceIndex + 1)
}
return builder.append(path, i, path.length).toString()
}
fun transformQueryParams(queryParams: Map<String, String>): String {
if (queryParams.isEmpty()) {
return ""
}
return "?".plus(queryParams.entries.joinToString("&") { "${it.key}=${it.value}" })
}
}

View File

@@ -1,11 +1,76 @@
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream.rpcclient
sealed interface CallParams import io.emeraldpay.dshackle.Global
data class ListParams(val list: List<Any>) : CallParams { sealed interface CallParams {
fun toJson(id: Int, method: String): ByteArray
}
abstract class JsonRpcParams : CallParams {
override fun toJson(id: Int, method: String): ByteArray {
val json = mapOf(
"jsonrpc" to "2.0",
"id" to id,
"method" to method,
"params" to params(),
)
return Global.objectMapper.writeValueAsBytes(json)
}
protected abstract fun params(): Any
}
data class ListParams(val list: List<Any>) : JsonRpcParams() {
constructor(vararg elements: Any) : this(listOf(*elements)) constructor(vararg elements: Any) : this(listOf(*elements))
constructor() : this(listOf()) constructor() : this(listOf())
override fun params(): Any {
return list
}
} }
data class ObjectParams(val obj: Map<Any, Any>) : CallParams {
data class ObjectParams(val obj: Map<Any, Any>) : JsonRpcParams() {
constructor(vararg pairs: Pair<Any, Any>) : this(mapOf(*pairs)) constructor(vararg pairs: Pair<Any, Any>) : this(mapOf(*pairs))
override fun params(): Any {
return obj
}
}
data class RestParams(
val headers: Map<String, String>,
val queryParams: Map<String, String>,
val pathParams: List<String>,
val payload: ByteArray,
) : CallParams {
companion object {
fun emptyParams() = RestParams(emptyMap(), emptyMap(), emptyList(), ByteArray(0))
}
override fun toJson(id: Int, method: String): ByteArray {
return payload
}
// we need equals and hashCode because of having ByteArray
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is RestParams) return false
if (headers != other.headers) return false
if (queryParams != other.queryParams) return false
if (pathParams != other.pathParams) return false
if (!payload.contentEquals(other.payload)) return false
return true
}
override fun hashCode(): Int {
var result = headers.hashCode()
result = 31 * result + queryParams.hashCode()
result = 31 * result + pathParams.hashCode()
result = 31 * result + payload.contentHashCode()
return result
}
} }

View File

@@ -21,37 +21,35 @@ import io.emeraldpay.api.proto.BlockchainOuterClass.NativeCallReplySignature
import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.RequestMetrics
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.grpc.StatusRuntimeException import io.grpc.StatusRuntimeException
import org.apache.commons.lang3.time.StopWatch import org.apache.commons.lang3.time.StopWatch
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
class JsonRpcGrpcClient( class JsonRpcGrpcClient(
private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub, private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val chain: Chain, private val chain: Chain,
private val metrics: RpcMetrics?, private val metrics: RequestMetrics?,
) { ) {
companion object { fun getReader(): ChainReader {
private val log = LoggerFactory.getLogger(JsonRpcGrpcClient::class.java)
}
fun getReader(): JsonRpcReader {
return Executor(stub, chain, metrics) return Executor(stub, chain, metrics)
} }
class Executor( class Executor(
private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub, private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub,
private val chain: Chain, private val chain: Chain,
private val metrics: RpcMetrics?, private val metrics: RequestMetrics?,
) : JsonRpcReader { ) : ChainReader {
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> { override fun read(key: ChainRequest): Mono<ChainResponse> {
val timer = StopWatch() val timer = StopWatch()
val req = BlockchainOuterClass.NativeCallRequest.newBuilder() val req = BlockchainOuterClass.NativeCallRequest.newBuilder()
.setChainValue(chain.id) .setChainValue(chain.id)
@@ -61,16 +59,28 @@ class JsonRpcGrpcClient(
val reqItem = BlockchainOuterClass.NativeCallItem.newBuilder() val reqItem = BlockchainOuterClass.NativeCallItem.newBuilder()
.setId(1) .setId(1)
.setMethod(key.method) .setMethod(key.method)
.setPayload( if (key.params is RestParams) {
reqItem.setRestData(
BlockchainOuterClass.RestData.newBuilder()
.addAllHeaders(mapKeyValue(key.params.headers))
.addAllQueryParams(mapKeyValue(key.params.queryParams))
.setPayload(ByteString.copyFrom(Global.objectMapper.writeValueAsBytes(key.params.payload)))
.addAllPathParams(key.params.pathParams)
.build(),
)
} else {
reqItem.setPayload(
ByteString.copyFrom( ByteString.copyFrom(
Global.objectMapper.writeValueAsBytes( Global.objectMapper.writeValueAsBytes(
when (key.params) { when (key.params) {
is ListParams -> key.params.list is ListParams -> key.params.list
is ObjectParams -> key.params.obj is ObjectParams -> key.params.obj
else -> throw IllegalStateException("Wrong param types ${key.params.javaClass}")
}, },
), ),
), ),
) )
}
if (key.nonce != null) { if (key.nonce != null) {
reqItem.nonce = key.nonce reqItem.nonce = key.nonce
} }
@@ -91,7 +101,7 @@ class JsonRpcGrpcClient(
} }
} }
fun handleResponse(resp: BlockchainOuterClass.NativeCallReplyItem): Mono<JsonRpcResponse> = fun handleResponse(resp: BlockchainOuterClass.NativeCallReplyItem): Mono<ChainResponse> =
if (resp.succeed) { if (resp.succeed) {
val bytes = resp.payload.toByteArray() val bytes = resp.payload.toByteArray()
val signature = if (resp.hasSignature()) { val signature = if (resp.hasSignature()) {
@@ -99,7 +109,7 @@ class JsonRpcGrpcClient(
} else { } else {
null null
} }
Mono.just(JsonRpcResponse(bytes, null, JsonRpcResponse.NumberId(0), null, signature, resp.upstreamId)) Mono.just(ChainResponse(bytes, null, ChainResponse.NumberId(0), null, signature, resp.upstreamId))
} else { } else {
metrics?.fails?.increment() metrics?.fails?.increment()
Mono.error( Mono.error(
@@ -140,5 +150,14 @@ class JsonRpcGrpcClient(
resp.keyId, resp.keyId,
) )
} }
private fun mapKeyValue(entries: Map<String, String>) =
entries.map {
BlockchainOuterClass.KeyValue
.newBuilder()
.setKey(it.key)
.setValue(it.value)
.build()
}
} }
} }

View File

@@ -1,223 +0,0 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.rpcclient
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.reader.JsonRpcHttpReader
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.stream.AggregateResponse
import io.emeraldpay.dshackle.upstream.rpcclient.stream.JsonRpcStreamParser
import io.emeraldpay.dshackle.upstream.rpcclient.stream.Response
import io.emeraldpay.dshackle.upstream.rpcclient.stream.SingleResponse
import io.emeraldpay.dshackle.upstream.rpcclient.stream.StreamResponse
import io.micrometer.core.instrument.Metrics
import io.netty.buffer.Unpooled
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpHeaders
import io.netty.handler.ssl.SslContextBuilder
import io.netty.resolver.DefaultAddressResolverGroup
import org.apache.commons.lang3.time.StopWatch
import reactor.core.publisher.Mono
import reactor.netty.http.client.HttpClient
import reactor.netty.resources.ConnectionProvider
import java.io.ByteArrayInputStream
import java.security.KeyStore
import java.security.cert.CertificateFactory
import java.security.cert.X509Certificate
import java.util.Base64
import java.util.concurrent.TimeUnit
import java.util.function.Consumer
import java.util.function.Function
/**
* JSON RPC client
*/
class JsonRpcHttpClient(
private val target: String,
private val metrics: RpcMetrics,
basicAuth: AuthConfig.ClientBasicAuth? = null,
tlsCAAuth: ByteArray? = null,
) : JsonRpcHttpReader {
private val parser = ResponseRpcParser()
private val streamParser = JsonRpcStreamParser()
private val httpClient: HttpClient
init {
val connectionProvider = ConnectionProvider.builder("dshackleConnectionPool")
.maxConnections(1500)
.pendingAcquireMaxCount(10000)
.build()
var build = HttpClient.create(connectionProvider)
.compress(true)
.resolver(DefaultAddressResolverGroup.INSTANCE)
build = build.headers { h ->
h.add(HttpHeaderNames.CONTENT_TYPE, "application/json")
}
basicAuth?.let { auth ->
val authString: String = auth.username + ":" + auth.password
val authBase64 = Base64.getEncoder().encodeToString(authString.toByteArray())
val encodedAuth = "Basic $authBase64"
val headers = Consumer { h: HttpHeaders -> h.add(HttpHeaderNames.AUTHORIZATION, encodedAuth) }
build = build.headers(headers)
}
tlsCAAuth?.let { auth ->
val cf = CertificateFactory.getInstance("X.509")
val cert = cf.generateCertificate(ByteArrayInputStream(auth)) as X509Certificate
val ks = KeyStore.getInstance(KeyStore.getDefaultType())
ks.load(null, "".toCharArray())
ks.setCertificateEntry("server", cert)
val sslContext = SslContextBuilder.forClient().trustManager(cert).build()
build.secure { spec ->
spec.sslContext(sslContext)
}
}
this.httpClient = build
}
private fun execute(request: JsonRpcRequest): Mono<out Response> {
val bytesRequest = request.toJson()
val response = httpClient
.post()
.uri(target)
.send(Mono.just(Unpooled.wrappedBuffer(bytesRequest)))
return if (!request.isStreamed) {
response.response { header, bytes ->
val statusCode = header.status().code()
bytes.aggregate().asByteArray().map {
AggregateResponse(it, statusCode)
}
}.single()
} else {
response.responseConnection { t, u ->
streamParser.streamParse(
t.status().code(),
u.inbound().receive().asByteArray(),
)
}.single()
}
}
override fun onStop() {
Metrics.globalRegistry.remove(metrics.timer)
Metrics.globalRegistry.remove(metrics.fails)
}
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> {
val startTime = StopWatch()
return Mono.just(key)
.doOnNext {
if (!startTime.isStarted) {
startTime.start()
}
}
.flatMap(this@JsonRpcHttpClient::execute)
.doOnNext {
if (startTime.isStarted) {
metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
}
}
.transform(asJsonRpcResponse(key))
.transform(convertErrors(key))
.transform(throwIfError())
}
/**
* The subscribers expect to catch an exception if the response contains JSON RPC Error. Convert it here to JsonRpcException
*/
private fun throwIfError(): Function<Mono<JsonRpcResponse>, Mono<JsonRpcResponse>> {
return Function { resp ->
resp.flatMap {
if (it.hasError()) {
Mono.error(JsonRpcUpstreamException(it.id, it.error!!))
} else {
Mono.just(it)
}
}
}
}
/**
* Convert internal exceptions to standard JsonRpcException
*/
private fun convertErrors(key: JsonRpcRequest): Function<Mono<JsonRpcResponse>, Mono<JsonRpcResponse>> {
return Function { resp ->
resp.onErrorResume { t ->
val err = when (t) {
is RpcException -> JsonRpcException.from(t)
is JsonRpcException -> t
else -> JsonRpcException(key.id, t.message ?: t.javaClass.name, cause = t)
}
// here we're measure the internal errors, not upstream errors
metrics.fails.increment()
Mono.error(err)
}
}
}
/**
* Process response from the upstream and convert it to JsonRpcResponse.
* The input is a pair of (Http Status Code, Http Response Body)
*/
private fun asJsonRpcResponse(key: JsonRpcRequest): Function<Mono<out Response>, Mono<JsonRpcResponse>> {
return Function { resp ->
resp.map {
when (it) {
is AggregateResponse -> {
val parsed = parser.parse(it.response)
val statusCode = it.code
if (statusCode != 200) {
if (parsed.hasError() && parsed.error!!.code != RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE) {
// extracted the error details from the HTTP Body
parsed
} else {
// here we got a valid response with ERROR as HTTP Status Code. We assume that HTTP Status has
// a higher priority so return an error here anyway
JsonRpcResponse.error(
RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE,
"HTTP Code: $statusCode",
JsonRpcResponse.NumberId(key.id),
)
}
} else {
parsed
}
}
is StreamResponse -> {
JsonRpcResponse(it.stream, key.id)
}
is SingleResponse -> {
if (it.hasError()) {
JsonRpcResponse(null, it.error)
} else {
JsonRpcResponse(it.result, null)
}
}
}
}
}
}
}

View File

@@ -0,0 +1,133 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.rpcclient
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.HttpReader
import io.emeraldpay.dshackle.upstream.RequestMetrics
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import io.emeraldpay.dshackle.upstream.rpcclient.stream.JsonRpcStreamParser
import io.emeraldpay.dshackle.upstream.stream.AggregateResponse
import io.emeraldpay.dshackle.upstream.stream.Response
import io.emeraldpay.dshackle.upstream.stream.SingleResponse
import io.emeraldpay.dshackle.upstream.stream.StreamResponse
import io.netty.buffer.Unpooled
import org.apache.commons.lang3.time.StopWatch
import reactor.core.publisher.Mono
import java.util.concurrent.TimeUnit
import java.util.function.Function
/**
* JSON RPC client
*/
class JsonRpcHttpReader(
target: String,
metrics: RequestMetrics,
basicAuth: AuthConfig.ClientBasicAuth? = null,
tlsCAAuth: ByteArray? = null,
) : HttpReader(target, metrics, basicAuth, tlsCAAuth) {
private val parser = ResponseRpcParser()
private val streamParser = JsonRpcStreamParser()
private fun execute(request: ChainRequest): Mono<out Response> {
val bytesRequest = request.toJson()
val response = httpClient
.post()
.uri(target)
.send(Mono.just(Unpooled.wrappedBuffer(bytesRequest)))
return if (!request.isStreamed) {
response.response { header, bytes ->
val statusCode = header.status().code()
bytes.aggregate().asByteArray().map {
AggregateResponse(it, statusCode)
}
}.single()
} else {
response.responseConnection { t, u ->
streamParser.streamParse(
t.status().code(),
u.inbound().receive().asByteArray(),
)
}.single()
}
}
override fun internalRead(key: ChainRequest): Mono<ChainResponse> {
val startTime = StopWatch()
return Mono.just(key)
.doOnNext {
if (!startTime.isStarted) {
startTime.start()
}
}
.flatMap(this@JsonRpcHttpReader::execute)
.doOnNext {
if (startTime.isStarted) {
metrics.timer.record(startTime.nanoTime, TimeUnit.NANOSECONDS)
}
}
.transform(asJsonRpcResponse(key))
}
/**
* Process response from the upstream and convert it to JsonRpcResponse.
* The input is a pair of (Http Status Code, Http Response Body)
*/
private fun asJsonRpcResponse(key: ChainRequest): Function<Mono<out Response>, Mono<ChainResponse>> {
return Function { resp ->
resp.map {
when (it) {
is AggregateResponse -> {
val parsed = parser.parse(it.response)
val statusCode = it.code
if (statusCode != 200) {
if (parsed.hasError() && parsed.error!!.code != RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE) {
// extracted the error details from the HTTP Body
parsed
} else {
// here we got a valid response with ERROR as HTTP Status Code. We assume that HTTP Status has
// a higher priority so return an error here anyway
ChainResponse.error(
RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE,
"HTTP Code: $statusCode",
ChainResponse.NumberId(key.id),
)
}
} else {
parsed
}
}
is StreamResponse -> {
ChainResponse(it.stream, key.id)
}
is SingleResponse -> {
if (it.hasError()) {
ChainResponse(null, it.error)
} else {
ChainResponse(it.result, null)
}
}
}
}
}
}
}

View File

@@ -1,6 +0,0 @@
package io.emeraldpay.dshackle.upstream.rpcclient
class JsonRpcUpstreamException(
id: JsonRpcResponse.Id,
error: JsonRpcError,
) : JsonRpcException(id, error, null, false)

View File

@@ -15,22 +15,26 @@
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream.rpcclient
import io.emeraldpay.dshackle.reader.JsonRpcReader import io.emeraldpay.dshackle.reader.ChainReader
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainException
import io.emeraldpay.dshackle.upstream.ChainRequest
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPool import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPool
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import reactor.core.publisher.Mono import reactor.core.publisher.Mono
class JsonRpcWsClient( class JsonRpcWsClient(
private val wsPool: WsConnectionPool, private val wsPool: WsConnectionPool,
) : JsonRpcReader { ) : ChainReader {
override fun read(key: JsonRpcRequest): Mono<JsonRpcResponse> { override fun read(key: ChainRequest): Mono<ChainResponse> {
val conn = wsPool.getConnection() val conn = wsPool.getConnection()
if (!conn.isConnected) { if (!conn.isConnected) {
return Mono.error( return Mono.error(
JsonRpcException( ChainException(
JsonRpcResponse.NumberId(key.id), ChainResponse.NumberId(key.id),
JsonRpcError( ChainCallError(
RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR, RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR,
"WebSocket is not connected", "WebSocket is not connected",
), ),

View File

@@ -15,8 +15,10 @@
*/ */
package io.emeraldpay.dshackle.upstream.rpcclient package io.emeraldpay.dshackle.upstream.rpcclient
import io.emeraldpay.dshackle.upstream.ChainCallError
class JsonRpcWsMessage( class JsonRpcWsMessage(
val result: ByteArray?, val result: ByteArray?,
val error: JsonRpcError?, val error: ChainCallError?,
val subscriptionId: String, val subscriptionId: String,
) )

View File

@@ -20,6 +20,8 @@ import com.fasterxml.jackson.core.JsonParseException
import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonToken import com.fasterxml.jackson.core.JsonToken
import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.ChainCallError
import io.emeraldpay.dshackle.upstream.ChainResponse
import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.io.IOException import java.io.IOException
@@ -45,7 +47,7 @@ abstract class ResponseParser<T> {
parser.nextToken() parser.nextToken()
if (parser.currentToken != JsonToken.START_OBJECT) { if (parser.currentToken != JsonToken.START_OBJECT) {
return Preparsed( return Preparsed(
error = JsonRpcError( error = ChainCallError(
RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE,
"Invalid JSON: not an Object", "Invalid JSON: not an Object",
), ),
@@ -59,13 +61,13 @@ abstract class ResponseParser<T> {
log.warn("Failed to parse JSON from upstream: ${e.message}") log.warn("Failed to parse JSON from upstream: ${e.message}")
} }
if (state.error != null && state.id == null) { if (state.error != null && state.id == null) {
state = state.copy(id = JsonRpcResponse.NumberId(0)) state = state.copy(id = ChainResponse.NumberId(0))
} }
if (state.isReady) { if (state.isReady) {
return state return state
} }
return Preparsed( return Preparsed(
error = JsonRpcError( error = ChainCallError(
RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE,
"Invalid JSON structure: never finalized", "Invalid JSON structure: never finalized",
), ),
@@ -76,7 +78,7 @@ abstract class ResponseParser<T> {
if (field == "jsonrpc") { if (field == "jsonrpc") {
if (!parser.nextToken().isScalarValue) { if (!parser.nextToken().isScalarValue) {
return state.copy( return state.copy(
error = JsonRpcError( error = ChainCallError(
RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE,
"Invalid JSON (jsonrpc value)", "Invalid JSON (jsonrpc value)",
), ),
@@ -103,17 +105,17 @@ abstract class ResponseParser<T> {
return state return state
} }
private fun readId(parser: JsonParser): JsonRpcResponse.Id { private fun readId(parser: JsonParser): ChainResponse.Id {
if (parser.currentToken() == JsonToken.FIELD_NAME) { if (parser.currentToken() == JsonToken.FIELD_NAME) {
parser.nextToken() parser.nextToken()
} }
return if (parser.currentToken() == JsonToken.VALUE_NUMBER_INT) { return if (parser.currentToken() == JsonToken.VALUE_NUMBER_INT) {
JsonRpcResponse.NumberId(parser.intValue) ChainResponse.NumberId(parser.intValue)
} else if (parser.currentToken() == JsonToken.VALUE_STRING) { } else if (parser.currentToken() == JsonToken.VALUE_STRING) {
JsonRpcResponse.StringId(parser.text) ChainResponse.StringId(parser.text)
} else { } else {
log.warn("Invalid id type: ${parser.currentToken()}") log.warn("Invalid id type: ${parser.currentToken()}")
return JsonRpcResponse.NumberId(0) return ChainResponse.NumberId(0)
} }
} }
@@ -151,7 +153,7 @@ abstract class ResponseParser<T> {
} }
} }
fun readError(parser: JsonParser): JsonRpcError? { fun readError(parser: JsonParser): ChainCallError? {
var code = 0 var code = 0
var message = "" var message = ""
var details: Any? = null var details: Any? = null
@@ -202,14 +204,14 @@ abstract class ResponseParser<T> {
} }
} }
} }
return JsonRpcError(code, message, details) return ChainCallError(code, message, details)
} }
data class Preparsed( data class Preparsed(
val id: JsonRpcResponse.Id? = null, val id: ChainResponse.Id? = null,
val result: ByteArray? = null, val result: ByteArray? = null,
val nullResult: Boolean = false, val nullResult: Boolean = false,
val error: JsonRpcError? = null, val error: ChainCallError? = null,
val subMethod: String? = null, val subMethod: String? = null,
val subId: String? = null, val subId: String? = null,
) { ) {

Some files were not shown because too many files have changed in this diff Show More