eth_filter methods support

This commit is contained in:
Maksim Fomenkov
2022-10-26 05:10:16 +03:00
parent 032caabf16
commit 4ff3965b79
27 changed files with 251 additions and 37 deletions

View File

@@ -21,6 +21,7 @@ import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import java.util.concurrent.ConcurrentLinkedQueue
open class AlwaysQuorum : CallQuorum { open class AlwaysQuorum : CallQuorum {
@@ -28,6 +29,7 @@ open class AlwaysQuorum : CallQuorum {
private var result: ByteArray? = null private var result: ByteArray? = null
private var rpcError: JsonRpcError? = null private var rpcError: JsonRpcError? = null
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
private val resolvers: MutableCollection<Upstream> = ConcurrentLinkedQueue()
override fun init(head: Head) { override fun init(head: Head) {
} }
@@ -48,6 +50,7 @@ open class AlwaysQuorum : CallQuorum {
result = response result = response
resolved = true resolved = true
sig = signature sig = signature
resolvers.add(upstream)
return true return true
} }
@@ -64,6 +67,9 @@ open class AlwaysQuorum : CallQuorum {
return rpcError return rpcError
} }
override fun getResolvedBy(): List<Upstream> =
resolvers.toList()
override fun toString(): String { override fun toString(): String {
return "Quorum: Accept Any" return "Quorum: Accept Any"
} }

View File

@@ -34,4 +34,5 @@ interface CallQuorum {
fun getSignature(): ResponseSigner.Signature? fun getSignature(): ResponseSigner.Signature?
fun getResult(): ByteArray? fun getResult(): ByteArray?
fun getError(): JsonRpcError? fun getError(): JsonRpcError?
fun getResolvedBy(): Collection<Upstream>
} }

View File

@@ -21,6 +21,7 @@ import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
/** /**
@@ -34,6 +35,7 @@ class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
private val failed = AtomicReference(false) private val failed = AtomicReference(false)
private var rpcError: JsonRpcError? = null private var rpcError: JsonRpcError? = null
private var sig: ResponseSigner.Signature? = null private var sig: ResponseSigner.Signature? = null
private val resolvers: MutableCollection<Upstream> = ConcurrentLinkedQueue()
override fun init(head: Head) { override fun init(head: Head) {
} }
@@ -51,6 +53,7 @@ class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
if (!lagging) { if (!lagging) {
result.set(response) result.set(response)
sig = signature sig = signature
resolvers.add(upstream)
return true return true
} }
return false return false
@@ -75,6 +78,8 @@ class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum {
return rpcError return rpcError
} }
override fun getResolvedBy(): Collection<Upstream> =
resolvers.toList()
override fun toString(): String { override fun toString(): String {
return "Quorum: late <= $maxLag blocks" return "Quorum: late <= $maxLag blocks"
} }

View File

@@ -117,9 +117,9 @@ class QuorumRpcReader(
return Function { quorumResult -> return Function { quorumResult ->
quorumResult quorumResult
.filter { it.isResolved() } // return nothing if not resolved .filter { it.isResolved() } // return nothing if not resolved
.map { .map { quorum ->
// TODO find actual quorum number // TODO find actual quorum number
QuorumRpcReader.Result(it.getResult()!!, it.getSignature(), 1) Result(quorum.getResult()!!, quorum.getSignature(), 1, quorum.getResolvedBy().map { it.hash() })
} }
.switchIfEmpty(defaultResult) .switchIfEmpty(defaultResult)
} }
@@ -197,6 +197,7 @@ class QuorumRpcReader(
class Result( class Result(
val value: ByteArray, val value: ByteArray,
val signature: ResponseSigner.Signature?, val signature: ResponseSigner.Signature?,
val quorum: Int val quorum: Int,
val resolvers: Collection<Byte>
) )
} }

View File

@@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.etherjar.rpc.RpcException import io.emeraldpay.etherjar.rpc.RpcException
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.util.concurrent.ConcurrentLinkedQueue
abstract class ValueAwareQuorum<T>( abstract class ValueAwareQuorum<T>(
val clazz: Class<T> val clazz: Class<T>
@@ -30,6 +31,7 @@ abstract class ValueAwareQuorum<T>(
private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java) private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java)
private var rpcError: JsonRpcError? = null private var rpcError: JsonRpcError? = null
private val resolvers: MutableCollection<Upstream> = ConcurrentLinkedQueue()
fun extractValue(response: ByteArray, clazz: Class<T>): T? { fun extractValue(response: ByteArray, clazz: Class<T>): T? {
return Global.objectMapper.readValue(response.inputStream(), clazz) return Global.objectMapper.readValue(response.inputStream(), clazz)
@@ -39,6 +41,7 @@ abstract class ValueAwareQuorum<T>(
try { try {
val value = extractValue(response, clazz) val value = extractValue(response, clazz)
recordValue(response, value, signature, upstream) recordValue(response, value, signature, upstream)
resolvers.add(upstream)
} catch (e: RpcException) { } catch (e: RpcException) {
recordError(response, e.rpcMessage, signature, upstream) recordError(response, e.rpcMessage, signature, upstream)
} catch (e: Exception) { } catch (e: Exception) {
@@ -59,4 +62,7 @@ abstract class ValueAwareQuorum<T>(
override fun getError(): JsonRpcError? { override fun getError(): JsonRpcError? {
return rpcError return rpcError
} }
override fun getResolvedBy(): Collection<Upstream> =
resolvers.toList()
} }

View File

@@ -102,7 +102,8 @@ open class NativeCall(
} }
fun parseParams(it: ValidCallContext<RawCallDetails>): ValidCallContext<ParsedCallDetails> { fun parseParams(it: ValidCallContext<RawCallDetails>): ValidCallContext<ParsedCallDetails> {
val params = extractParams(it.payload.params) val rawParams = extractParams(it.payload.params)
val params = it.requestDecorator.processRequest(rawParams)
return it.withPayload(ParsedCallDetails(it.payload.method, params)) return it.withPayload(ParsedCallDetails(it.payload.method, params))
} }
@@ -234,17 +235,28 @@ open class NativeCall(
matcher.withMatcher(heightMatcher) matcher.withMatcher(heightMatcher)
} }
val nonce = requestItem.nonce.let { if (it == 0L) null else it } val nonce = requestItem.nonce.let { if (it == 0L) null else it }
val requestDecorator = getRequestDecorator(requestItem.method)
val resultDecorator = getResultDecorator(requestItem.method)
ValidCallContext( ValidCallContext(
requestItem.id, requestItem.id,
nonce, nonce,
upstream, upstream,
matcher.build(), matcher.build(),
callQuorum, callQuorum,
RawCallDetails(method, params) RawCallDetails(method, params),
requestDecorator,
resultDecorator
) )
} }
} }
private fun getRequestDecorator(method: String): RequestDecorator =
if (method == "eth_getFilterChanges") GetFilterUpdatesDecorator() else NoneRequestDecorator()
private fun getResultDecorator(method: String): ResultDecorator =
if (CreateFilterDecorator.createFilterMethods.contains(method)) CreateFilterDecorator() else NoneResultDecorator()
fun fetch(ctx: ValidCallContext<ParsedCallDetails>): Mono<CallResult> { fun fetch(ctx: ValidCallContext<ParsedCallDetails>): Mono<CallResult> {
return ctx.upstream.getRoutedApi(ctx.matcher) return ctx.upstream.getRoutedApi(ctx.matcher)
.flatMap { api -> .flatMap { api ->
@@ -283,7 +295,8 @@ open class NativeCall(
return reader return reader
.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params, ctx.nonce)) .read(JsonRpcRequest(ctx.payload.method, ctx.payload.params, ctx.nonce))
.map { .map {
CallResult(ctx.id, ctx.nonce, it.value, null, it.signature) val bytes = ctx.resultDecorator.processResult(it)
CallResult(ctx.id, ctx.nonce, bytes, null, it.signature)
} }
.onErrorResume { t -> .onErrorResume { t ->
val failure = when (t) { val failure = when (t) {
@@ -350,14 +363,67 @@ open class NativeCall(
fun getError(): CallError fun getError(): CallError
} }
interface ResultDecorator {
fun processResult(result: QuorumRpcReader.Result): ByteArray
}
open class NoneResultDecorator : ResultDecorator {
override fun processResult(result: QuorumRpcReader.Result): ByteArray = result.value
}
open class CreateFilterDecorator : ResultDecorator {
companion object {
const val quoteCode = '"'.code.toByte()
val createFilterMethods = listOf("eth_getFilterChanges", "eth_newFilter", "eth_newBlockFilter")
}
override fun processResult(result: QuorumRpcReader.Result): ByteArray {
val bytes = result.value
if (bytes.last() == quoteCode) {
val suffix = result.resolvers.first().toUByte().toString(16).padStart(2, padChar = '0').toByteArray()
bytes[bytes.lastIndex] = suffix.first()
return bytes + suffix.last() + quoteCode
}
return bytes
}
}
interface RequestDecorator {
fun processRequest(request: List<Any>): List<Any>
}
open class NoneRequestDecorator : RequestDecorator {
override fun processRequest(request: List<Any>): List<Any> = request
}
open class GetFilterUpdatesDecorator : RequestDecorator {
override fun processRequest(request: List<Any>): List<Any> {
val filterId = request.first().toString()
val sanitized = filterId.substring(0, filterId.lastIndex - 1)
return listOf(sanitized)
}
}
open class ValidCallContext<T>( open class ValidCallContext<T>(
val id: Int, val id: Int,
val nonce: Long?, val nonce: Long?,
val upstream: Multistream, val upstream: Multistream,
val matcher: Selector.Matcher, val matcher: Selector.Matcher,
val callQuorum: CallQuorum, val callQuorum: CallQuorum,
val payload: T val payload: T,
val requestDecorator: RequestDecorator,
val resultDecorator: ResultDecorator
) : CallContext { ) : CallContext {
constructor(
id: Int,
nonce: Long?,
upstream: Multistream,
matcher: Selector.Matcher,
callQuorum: CallQuorum,
payload: T
) : this(id, nonce, upstream, matcher, callQuorum, payload, NoneRequestDecorator(), NoneResultDecorator())
override fun isValid(): Boolean { override fun isValid(): Boolean {
return true return true
} }
@@ -371,7 +437,7 @@ open class NativeCall(
} }
fun <X> withPayload(payload: X): ValidCallContext<X> { fun <X> withPayload(payload: X): ValidCallContext<X> {
return ValidCallContext(id, nonce, upstream, matcher, callQuorum, payload) return ValidCallContext(id, nonce, upstream, matcher, callQuorum, payload, requestDecorator, resultDecorator)
} }
fun getApis(): ApiSource { fun getApis(): ApiSource {

View File

@@ -53,7 +53,9 @@ import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository import org.springframework.stereotype.Repository
import java.net.URI import java.net.URI
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
import java.util.function.Function
import javax.annotation.PostConstruct import javax.annotation.PostConstruct
import kotlin.math.abs
@Repository @Repository
open class ConfiguredUpstreams( open class ConfiguredUpstreams(
@@ -65,6 +67,8 @@ open class ConfiguredUpstreams(
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
private var seq = AtomicInteger(0) private var seq = AtomicInteger(0)
private val hashes: MutableMap<Byte, Boolean> = HashMap()
@PostConstruct @PostConstruct
fun start() { fun start() {
log.debug("Starting upstreams") log.debug("Starting upstreams")
@@ -90,12 +94,19 @@ open class ConfiguredUpstreams(
BlockchainType.ETHEREUM -> { BlockchainType.ETHEREUM -> {
buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options) buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
} }
BlockchainType.BITCOIN -> { BlockchainType.BITCOIN -> {
buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options) buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options)
} }
BlockchainType.ETHEREUM_POS -> { BlockchainType.ETHEREUM_POS -> {
buildEthereumPosUpstream(up.cast(UpstreamsConfig.EthereumPosConnection::class.java), chain, options) buildEthereumPosUpstream(
up.cast(UpstreamsConfig.EthereumPosConnection::class.java),
chain,
options
)
} }
else -> { else -> {
log.error("Chain is unsupported: ${up.chain}") log.error("Chain is unsupported: ${up.chain}")
return@forEach return@forEach
@@ -167,13 +178,26 @@ open class ConfiguredUpstreams(
return null return null
} }
val urls = ArrayList<URI>() val urls = ArrayList<URI>()
val connectorFactory = buildEthereumConnectorFactory(config.id!!, execution, chain, urls, NoChoiceWithPriorityForkChoice(conn.upstreamRating), BlockValidator.ALWAYS_VALID) val connectorFactory = buildEthereumConnectorFactory(
config.id!!,
execution,
chain,
urls,
NoChoiceWithPriorityForkChoice(conn.upstreamRating),
BlockValidator.ALWAYS_VALID
)
val methods = buildMethods(config, chain) val methods = buildMethods(config, chain)
if (connectorFactory == null) { if (connectorFactory == null) {
return null return null
} }
val hashUrl = conn.execution!!.let {
if (it.preferHttp) it.rpc?.url ?: it.ws?.url else it.ws?.url ?: it.rpc?.url
}
val hash = getHash(hashUrl!!)
val upstream = EthereumPosRpcUpstream( val upstream = EthereumPosRpcUpstream(
config.id!!, config.id!!,
hash,
chain, chain,
options, config.role, options, config.role,
methods, methods,
@@ -236,12 +260,22 @@ open class ConfiguredUpstreams(
val urls = ArrayList<URI>() val urls = ArrayList<URI>()
val methods = buildMethods(config, chain) val methods = buildMethods(config, chain)
val connectorFactory = buildEthereumConnectorFactory(config.id!!, conn, chain, urls, MostWorkForkChoice(), EthereumBlockValidator()) val connectorFactory = buildEthereumConnectorFactory(
config.id!!,
conn,
chain,
urls,
MostWorkForkChoice(),
EthereumBlockValidator()
)
if (connectorFactory == null) { if (connectorFactory == null) {
return null return null
} }
val hashUrl = if (conn.preferHttp) conn.rpc?.url ?: conn.ws?.url else conn.ws?.url ?: conn.rpc?.url
val upstream = EthereumRpcUpstream( val upstream = EthereumRpcUpstream(
config.id!!, config.id!!,
getHash(hashUrl!!),
chain, chain,
options, config.role, options, config.role,
methods, methods,
@@ -257,8 +291,10 @@ open class ConfiguredUpstreams(
options: UpstreamsConfig.Options options: UpstreamsConfig.Options
) { ) {
val endpoint = config.connection!! val endpoint = config.connection!!
val hash = getHash("${endpoint.host}:${endpoint.port}")
val ds = GrpcUpstreams( val ds = GrpcUpstreams(
config.id!!, config.id!!,
hash,
config.role, config.role,
endpoint.host!!, endpoint.host!!,
endpoint.port, endpoint.port,
@@ -289,7 +325,12 @@ open class ConfiguredUpstreams(
} }
} }
private fun buildWsFactory(id: String, chain: Chain, conn: UpstreamsConfig.EthereumConnection, urls: ArrayList<URI>? = null): EthereumWsFactory? { private fun buildWsFactory(
id: String,
chain: Chain,
conn: UpstreamsConfig.EthereumConnection,
urls: ArrayList<URI>? = null
): EthereumWsFactory? {
return conn.ws?.let { endpoint -> return conn.ws?.let { endpoint ->
val wsApi = EthereumWsFactory( val wsApi = EthereumWsFactory(
id, chain, id, chain,
@@ -305,15 +346,43 @@ open class ConfiguredUpstreams(
} }
} }
private fun buildEthereumConnectorFactory(id: String, conn: UpstreamsConfig.EthereumConnection, chain: Chain, urls: ArrayList<URI>, forkChoice: ForkChoice, blockValidator: BlockValidator): EthereumConnectorFactory? { private fun buildEthereumConnectorFactory(
id: String,
conn: UpstreamsConfig.EthereumConnection,
chain: Chain,
urls: ArrayList<URI>,
forkChoice: ForkChoice,
blockValidator: BlockValidator
): EthereumConnectorFactory? {
val wsFactoryApi = buildWsFactory(id, chain, conn, urls) val wsFactoryApi = buildWsFactory(id, chain, conn, urls)
val httpFactory = buildHttpFactory(conn, urls) val httpFactory = buildHttpFactory(conn, urls)
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}") log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
val connectorFactory = EthereumConnectorFactory(conn.preferHttp, wsFactoryApi, httpFactory, forkChoice, blockValidator) val connectorFactory =
EthereumConnectorFactory(conn.preferHttp, wsFactoryApi, httpFactory, forkChoice, blockValidator)
if (!connectorFactory.isValid()) { if (!connectorFactory.isValid()) {
log.warn("Upstream configuration is invalid (probably no http endpoint)") log.warn("Upstream configuration is invalid (probably no http endpoint)")
return null return null
} }
return connectorFactory return connectorFactory
} }
private fun getHash(obj: Any): Byte {
val hashCode = (obj.hashCode() % 255)
val modifiers: List<Function<Int, Number>> = listOf(
Function { i -> i },
Function { i -> (-i) },
Function { i -> 127 - abs(i) },
Function { i -> abs(i) - 128 },
)
return modifiers.map {
it.apply(hashCode).toByte()
}.firstOrNull {
hashes[it] != true
}?.let {
hashes[it] = true
it
} ?: (Byte.MIN_VALUE..Byte.MAX_VALUE).first {
hashes[it.toByte()] != true
}.toByte()
}
} }

View File

@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference
abstract class DefaultUpstream( abstract class DefaultUpstream(
private val id: String, private val id: String,
private val hash: Byte,
defaultLag: Long, defaultLag: Long,
defaultAvail: UpstreamAvailability, defaultAvail: UpstreamAvailability,
private val options: UpstreamsConfig.Options, private val options: UpstreamsConfig.Options,
@@ -36,12 +37,14 @@ abstract class DefaultUpstream(
constructor( constructor(
id: String, id: String,
hash: Byte,
options: UpstreamsConfig.Options, options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
targets: CallMethods? targets: CallMethods?
) : ) :
this( this(
id, id,
hash,
Long.MAX_VALUE, Long.MAX_VALUE,
UpstreamAvailability.UNAVAILABLE, UpstreamAvailability.UNAVAILABLE,
options, options,
@@ -52,12 +55,13 @@ abstract class DefaultUpstream(
constructor( constructor(
id: String, id: String,
hash: Byte,
options: UpstreamsConfig.Options, options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?, targets: CallMethods?,
node: QuorumForLabels.QuorumItem? node: QuorumForLabels.QuorumItem?
) : ) :
this(id, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node) this(id, hash, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node)
private val status = AtomicReference(Status(defaultLag, defaultAvail, statusByLag(defaultLag, defaultAvail))) private val status = AtomicReference(Status(defaultLag, defaultAvail, statusByLag(defaultLag, defaultAvail)))
private val statusStream = Sinks.many() private val statusStream = Sinks.many()
@@ -139,6 +143,8 @@ abstract class DefaultUpstream(
return targets ?: throw IllegalStateException("Methods are not set") return targets ?: throw IllegalStateException("Methods are not set")
} }
override fun hash(): Byte = hash
private val quorumByLabel = node?.let { QuorumForLabels(it) } private val quorumByLabel = node?.let { QuorumForLabels(it) }
?: QuorumForLabels(QuorumForLabels.QuorumItem.empty()) ?: QuorumForLabels(QuorumForLabels.QuorumItem.empty())

View File

@@ -282,6 +282,8 @@ abstract class Multistream(
return false return false
} }
override fun hash(): Byte = 0
fun printStatus() { fun printStatus() {
var height: Long? = null var height: Long? = null
try { try {

View File

@@ -396,4 +396,16 @@ class Selector {
return "Matcher: ${describeInternal()}" return "Matcher: ${describeInternal()}"
} }
} }
class SameUpstreamMatcher(private val upstreamHash: Byte) : Matcher {
override fun matches(up: Upstream): Boolean =
up.hash() == upstreamHash
override fun describeInternal(): String =
"upstream hash=$upstreamHash"
override fun toString(): String {
return "Matcher: ${describeInternal()}"
}
}
} }

View File

@@ -40,4 +40,6 @@ interface Upstream {
fun isGrpc(): Boolean fun isGrpc(): Boolean
fun <T : Upstream> cast(selfType: Class<T>): T fun <T : Upstream> cast(selfType: Class<T>): T
fun hash(): Byte
} }

View File

@@ -31,7 +31,7 @@ abstract class BitcoinUpstream(
callMethods: CallMethods, callMethods: CallMethods,
node: QuorumForLabels.QuorumItem, node: QuorumForLabels.QuorumItem,
val esploraClient: EsploraClient? = null val esploraClient: EsploraClient? = null
) : DefaultUpstream(id, options, role, callMethods, node) { ) : DefaultUpstream(id, 0.toByte(), options, role, callMethods, node) {
constructor( constructor(
id: String, id: String,

View File

@@ -70,7 +70,14 @@ class DefaultEthereumMethods(
"eth_feeHistory" "eth_feeHistory"
) )
private val allowedMethods = anyResponseMethods + firstValueMethods + specialMethods + headVerifiedMethods private val filterMethods = listOf(
"eth_getFilterChanges",
"eth_newFilter",
"eth_newBlockFilter",
"eth_newPendingTransactionFilter",
)
private val allowedMethods = anyResponseMethods + firstValueMethods + specialMethods + headVerifiedMethods + filterMethods
private val hardcodedMethods = listOf( private val hardcodedMethods = listOf(
"net_version", "net_version",
@@ -88,6 +95,7 @@ class DefaultEthereumMethods(
override fun getQuorumFor(method: String): CallQuorum { override fun getQuorumFor(method: String): CallQuorum {
return when { return when {
filterMethods.contains(method) -> AlwaysQuorum()
hardcodedMethods.contains(method) -> AlwaysQuorum() hardcodedMethods.contains(method) -> AlwaysQuorum()
firstValueMethods.contains(method) -> AlwaysQuorum() firstValueMethods.contains(method) -> AlwaysQuorum()
anyResponseMethods.contains(method) -> NotLaggingQuorum(4) anyResponseMethods.contains(method) -> NotLaggingQuorum(4)

View File

@@ -57,10 +57,28 @@ class EthereumCallSelector(
return blockTagSelector(params, 1, head) return blockTagSelector(params, 1, head)
} else if (method == "eth_getStorageAt") { } else if (method == "eth_getStorageAt") {
return blockTagSelector(params, 2, head) return blockTagSelector(params, 2, head)
} else if (method == "eth_getFilterChanges") {
return sameUpstreamMatcher(params)
} }
return Mono.empty() return Mono.empty()
} }
private fun sameUpstreamMatcher(params: String): Mono<Selector.Matcher> {
val list = objectMapper.readerFor(Any::class.java).readValues<Any>(params).readAll()
if (list.isEmpty()) {
return Mono.empty()
}
val filterId = list[0].toString()
val hashHex = filterId.substring(filterId.length - 2)
val hash = hashHex.toInt(16)
if (hash < 0 || hash > 255) {
return Mono.empty()
}
return Mono.just(Selector.SameUpstreamMatcher(hash.toByte()))
}
private fun blockTagSelector(params: String, pos: Int, head: Head): Mono<Selector.Matcher> { private fun blockTagSelector(params: String, pos: Int, head: Head): Mono<Selector.Matcher> {
val list = objectMapper.readerFor(Any::class.java).readValues<Any>(params).readAll() val list = objectMapper.readerFor(Any::class.java).readValues<Any>(params).readAll()
if (list.size < pos + 1) { if (list.size < pos + 1) {

View File

@@ -36,13 +36,14 @@ import reactor.core.Disposable
open class EthereumRpcUpstream( open class EthereumRpcUpstream(
id: String, id: String,
hash: Byte,
val chain: Chain, val chain: Chain,
options: UpstreamsConfig.Options, options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?, targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?, private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory connectorFactory: ConnectorFactory
) : EthereumUpstream(id, options, role, targets, node), Lifecycle, Upstream, CachesEnabled { ) : EthereumUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java) private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java)
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions()) private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain) private val connector: EthereumConnector = connectorFactory.create(this, validator, chain)

View File

@@ -24,11 +24,12 @@ import io.emeraldpay.dshackle.upstream.calls.CallMethods
abstract class EthereumUpstream( abstract class EthereumUpstream(
id: String, id: String,
hash: Byte,
options: UpstreamsConfig.Options, options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?, targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem? private val node: QuorumForLabels.QuorumItem?
) : DefaultUpstream(id, options, role, targets, node) { ) : DefaultUpstream(id, hash, options, role, targets, node) {
private val capabilities = if (options.providesBalance != false) { private val capabilities = if (options.providesBalance != false) {
setOf(Capability.RPC, Capability.BALANCE) setOf(Capability.RPC, Capability.BALANCE)

View File

@@ -36,13 +36,14 @@ import reactor.core.Disposable
open class EthereumPosRpcUpstream( open class EthereumPosRpcUpstream(
id: String, id: String,
hash: Byte,
val chain: Chain, val chain: Chain,
options: UpstreamsConfig.Options, options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?, targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?, private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory connectorFactory: ConnectorFactory
) : EthereumPosUpstream(id, options, role, targets, node), Lifecycle, Upstream, CachesEnabled { ) : EthereumPosUpstream(id, hash, options, role, targets, node), Lifecycle, Upstream, CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumPosRpcUpstream::class.java) private val log = LoggerFactory.getLogger(EthereumPosRpcUpstream::class.java)
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions()) private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain) private val connector: EthereumConnector = connectorFactory.create(this, validator, chain)

View File

@@ -24,11 +24,12 @@ import io.emeraldpay.dshackle.upstream.calls.CallMethods
abstract class EthereumPosUpstream( abstract class EthereumPosUpstream(
id: String, id: String,
hash: Byte,
options: UpstreamsConfig.Options, options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?, targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem? private val node: QuorumForLabels.QuorumItem?
) : DefaultUpstream(id, options, role, targets, node) { ) : DefaultUpstream(id, hash, options, role, targets, node) {
private val capabilities = if (options.providesBalance != false) { private val capabilities = if (options.providesBalance != false) {
setOf(Capability.RPC, Capability.BALANCE) setOf(Capability.RPC, Capability.BALANCE)

View File

@@ -51,6 +51,7 @@ import java.util.function.Function
open class EthereumGrpcUpstream( open class EthereumGrpcUpstream(
private val parentId: String, private val parentId: String,
hash: Byte,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
private val chain: Chain, private val chain: Chain,
private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
@@ -58,6 +59,7 @@ open class EthereumGrpcUpstream(
overrideLabels: UpstreamsConfig.Labels? overrideLabels: UpstreamsConfig.Labels?
) : EthereumUpstream( ) : EthereumUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
hash,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.Options.getDefaults(),
role, role,
null, null,

View File

@@ -51,6 +51,7 @@ import java.util.function.Function
open class EthereumPosGrpcUpstream( open class EthereumPosGrpcUpstream(
private val parentId: String, private val parentId: String,
hash: Byte,
role: UpstreamsConfig.UpstreamRole, role: UpstreamsConfig.UpstreamRole,
private val chain: Chain, private val chain: Chain,
private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
@@ -59,6 +60,7 @@ open class EthereumPosGrpcUpstream(
overrideLabels: UpstreamsConfig.Labels? overrideLabels: UpstreamsConfig.Labels?
) : EthereumPosUpstream( ) : EthereumPosUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}", "${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
hash,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.Options.getDefaults(),
role, role,
null, null null, null

View File

@@ -52,6 +52,7 @@ import kotlin.concurrent.withLock
class GrpcUpstreams( class GrpcUpstreams(
private val id: String, private val id: String,
private val hash: Byte,
private val role: UpstreamsConfig.UpstreamRole, private val role: UpstreamsConfig.UpstreamRole,
private val host: String, private val host: String,
private val port: Int, private val port: Int,
@@ -206,7 +207,7 @@ class GrpcUpstreams(
val current = known[chain] val current = known[chain]
return if (current == null) { return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics) val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = EthereumGrpcUpstream(id, role, chain, client!!, rpcClient, labels) val created = EthereumGrpcUpstream(id, hash, role, chain, client!!, rpcClient, labels)
created.timeout = this.timeout created.timeout = this.timeout
known[chain] = created known[chain] = created
created.start() created.start()
@@ -222,7 +223,7 @@ class GrpcUpstreams(
val current = known[chain] val current = known[chain]
return if (current == null) { return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics) val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = EthereumPosGrpcUpstream(id, role, chain, client!!, rpcClient, nodeRating, labels) val created = EthereumPosGrpcUpstream(id, hash, role, chain, client!!, rpcClient, nodeRating, labels)
created.timeout = this.timeout created.timeout = this.timeout
known[chain] = created known[chain] = created
created.start() created.start()

View File

@@ -118,7 +118,7 @@ class NativeCallSpec extends Specification {
def nativeCall = nativeCall() def nativeCall = nativeCall()
nativeCall.quorumReaderFactory = Mock(QuorumReaderFactory) { nativeCall.quorumReaderFactory = Mock(QuorumReaderFactory) {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(_) >> Mono.just(new QuorumRpcReader.Result("\"foo\"".bytes, null, 1)) 1 * read(_) >> Mono.just(new QuorumRpcReader.Result("\"foo\"".bytes, null, 1, Collections.singletonList((byte) 1)))
} }
} }
def call = new NativeCall.ValidCallContext(1, 10, TestingCommons.multistream(TestingCommons.api()), Selector.empty, quorum, def call = new NativeCall.ValidCallContext(1, 10, TestingCommons.multistream(TestingCommons.api()), Selector.empty, quorum,

View File

@@ -69,7 +69,7 @@ class EthereumPosRpcUpstreamMock extends EthereumPosRpcUpstream {
} }
EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods, Map<String, String> labels) { EthereumPosRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods, Map<String, String> labels) {
super(id, chain, super(id, (byte)id.hashCode(), chain,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.Options.getDefaults(),
UpstreamsConfig.UpstreamRole.PRIMARY, UpstreamsConfig.UpstreamRole.PRIMARY,
methods, methods,

View File

@@ -60,7 +60,7 @@ class EthereumRpcUpstreamMock extends EthereumRpcUpstream {
} }
EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) { EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
super(id, chain, super(id, id.hashCode().byteValue(), chain,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.Options.getDefaults(),
UpstreamsConfig.UpstreamRole.PRIMARY, UpstreamsConfig.UpstreamRole.PRIMARY,
methods, methods,

View File

@@ -51,6 +51,7 @@ class FilteredApisSpec extends Specification {
def connectorFactory = new EthereumConnectorFactory(false, null, httpFactory, new MostWorkForkChoice(), BlockValidator.@Companion.ALWAYS_VALID) def connectorFactory = new EthereumConnectorFactory(false, null, httpFactory, new MostWorkForkChoice(), BlockValidator.@Companion.ALWAYS_VALID)
new EthereumRpcUpstream( new EthereumRpcUpstream(
"test", "test",
(byte)123,
Chain.ETHEREUM, Chain.ETHEREUM,
new UpstreamsConfig.Options(), new UpstreamsConfig.Options(),
UpstreamsConfig.UpstreamRole.PRIMARY, UpstreamsConfig.UpstreamRole.PRIMARY,

View File

@@ -30,6 +30,7 @@ class EthereumDirectReaderSpec extends Specification {
String hash1 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5" String hash1 = "0x40d15edaff9acdabd2a1c96fd5f683b3300aad34e7015f34def3c56ba8a7ffb5"
String address1 = "0xe0aadb0a012dbcdc529c4c743d3e0385a0b54d3d" String address1 = "0xe0aadb0a012dbcdc529c4c743d3e0385a0b54d3d"
List<Byte> resolvers = Collections.singletonList((byte)1)
def "Reads block by hash"() { def "Reads block by hash"() {
setup: setup:
@@ -53,8 +54,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getBlockByHash", [hash1, false])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getBlockByHash", [hash1, false])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes(json), null, 1 Global.objectMapper.writeValueAsBytes(json), null, 1, resolvers)
)
) )
} }
} }
@@ -84,7 +84,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getBlockByHash", [hash1, false])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getBlockByHash", [hash1, false])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes(null), null, 1 Global.objectMapper.writeValueAsBytes(null), null, 1, resolvers
) )
) )
} }
@@ -119,7 +119,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getBlockByNumber", ["0x64", false])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getBlockByNumber", ["0x64", false])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes(json), null, 1 Global.objectMapper.writeValueAsBytes(json), null, 1, resolvers
) )
) )
} }
@@ -155,7 +155,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getTransactionByHash", [hash1])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getTransactionByHash", [hash1])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes(json), null, 1 Global.objectMapper.writeValueAsBytes(json), null, 1, resolvers
) )
) )
} }
@@ -186,7 +186,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getTransactionByHash", [hash1])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getTransactionByHash", [hash1])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes(null), null, 1 Global.objectMapper.writeValueAsBytes(null), null, 1, resolvers
) )
) )
} }
@@ -217,7 +217,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getBalance", [address1, "latest"])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getBalance", [address1, "latest"])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes("0x100"), null, 1 Global.objectMapper.writeValueAsBytes("0x100"), null, 1, resolvers
) )
) )
} }
@@ -249,7 +249,7 @@ class EthereumDirectReaderSpec extends Specification {
1 * create(_, _, _) >> Mock(Reader) { 1 * create(_, _, _) >> Mock(Reader) {
1 * read(new JsonRpcRequest("eth_getBalance", [address1, "0xa8c9bb"])) >> Mono.just( 1 * read(new JsonRpcRequest("eth_getBalance", [address1, "0xa8c9bb"])) >> Mono.just(
new QuorumRpcReader.Result( new QuorumRpcReader.Result(
Global.objectMapper.writeValueAsBytes("0x100"), null, 1 Global.objectMapper.writeValueAsBytes("0x100"), null, 1, resolvers
) )
) )
} }

View File

@@ -50,6 +50,8 @@ class EthereumGrpcUpstreamSpec extends Specification {
Counter.builder("test2").register(TestingCommons.meterRegistry) Counter.builder("test2").register(TestingCommons.meterRegistry)
) )
def hash = (byte)123
def "Subscribe to head"() { def "Subscribe to head"() {
setup: setup:
def callData = [:] def callData = [:]
@@ -81,7 +83,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
) )
} }
}) })
def upstream = new EthereumGrpcUpstream("test", UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null) def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null)
upstream.setLag(0) upstream.setLag(0)
upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() upstream.update(BlockchainOuterClass.DescribeChain.newBuilder()
.setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId))
@@ -139,7 +141,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
) )
} }
}) })
def upstream = new EthereumGrpcUpstream("test", UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics), null) def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, Chain.ETHEREUM, client, new JsonRpcGrpcClient(client, Chain.ETHEREUM, metrics), null)
upstream.setLag(0) upstream.setLag(0)
upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() upstream.update(BlockchainOuterClass.DescribeChain.newBuilder()
.setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId))
@@ -201,7 +203,7 @@ class EthereumGrpcUpstreamSpec extends Specification {
finished.complete(true) finished.complete(true)
} }
}) })
def upstream = new EthereumGrpcUpstream("test", UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null) def upstream = new EthereumGrpcUpstream("test", hash, UpstreamsConfig.UpstreamRole.PRIMARY, chain, client, new JsonRpcGrpcClient(client, chain, metrics), null)
upstream.setLag(0) upstream.setLag(0)
upstream.update(BlockchainOuterClass.DescribeChain.newBuilder() upstream.update(BlockchainOuterClass.DescribeChain.newBuilder()
.setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId)) .setStatus(BlockchainOuterClass.ChainStatus.newBuilder().setQuorum(1).setAvailabilityValue(UpstreamAvailability.OK.grpcId))