StakeSquid/dshackle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from Termina1/ethereum-pos-initial-support

Browse Source 
Ethereum PoS initial support
This commit is contained in:
Vyacheslav Shebanov
2022-08-12 14:33:01 +03:00
committed by GitHub
parent 4bcaa70e2c 3369eb35e2
commit 936049f6e5
89 changed files with 1711 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "emerald-java-client"]
path = emerald-java-client
url = git@github.com:p2p-org/emerald-java-client.git

16
docs/04-upstream-config.adoc
View File

@@ -17,6 +17,13 @@ Those protocols can be configures with additional security, TLS and authenticati
- Most of Ethereum nodes support WebSocket connection, in addition to the JSON RPC.
If it's available on your node, it's suggested to configure both JSON RPC and WebSocket connection
==== Ethereum PoS
- The support for PoS Ethereum isn't fully implemented yet. However, we support it in a simplified way.
As we can no longer rely on the difficulty parameter of a block for fork choice algorithm we implement upstream rating.
In short terms, we just consider the upstream with the highest rating to be always correct when reporting its head.
Unless it's down then we will fallback on the upstream with the second highest rating, etc.
==== Bitcoin
- Bitcoind needs to be configured to index/track addresses that you're going to request.
@@ -72,6 +79,15 @@ cluster:
basic-auth:
username: ${INFURA_USER}
password: ${INFURA_PASSWD}
- id: ethereum-pos
chain: ropsten
connection:
ethereum-pos:
execution:
rpc:
url: ${ROPSTEN_NODE_RPC_URL}
ws:
url: ${ROPSTEN_NODE_WS_URL}
----
There are two main segments for upstreams configuration:

16
docs/reference-configuration.adoc
View File

@@ -769,6 +769,22 @@ Default is 15Mb
|===
==== PoS Ethereum Connection Options
.Connection Config for PoS Ethereum Upstream
[cols="2a,5"]
|===
| Option | Description
| `execution`
a| Here you can specify any option from plain ethereum connection options listed above +
This is your connection to an execution layer of PoS Ethereum
| `upstream-rating`
a| Rating for this upstream. We will always consider the head of the chain to be +
the latest block we saw from the upstream with the highest rating.
|===
==== Bitcoin Connection Options
.Connection Config for Bitcoin Upstream

1
emerald-java-client Submodule

Submodule emerald-java-client added at 934a7d0fd0

6
settings.gradle
View File

@@ -1,2 +1,8 @@
enableFeaturePreview("VERSION_CATALOGS")
enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
includeBuild('./emerald-java-client') {
dependencySubstitution {
substitute module('io.emeraldpay:emerald-api:0.12-alpha.1') using project(':')
}
}

6
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt
View File

@@ -102,6 +102,7 @@ open class UpstreamsConfig {
var host: String? = null
var port: Int = 0
var auth: AuthConfig.ClientTlsAuth? = null
var upstreamRating: Int = 0
}
class EthereumConnection : RpcConnection() {
@@ -114,6 +115,11 @@ open class UpstreamsConfig {
var zeroMq: BitcoinZeroMq? = null
}
class EthereumPosConnection : UpstreamConnection() {
var execution: EthereumConnection? = null
var upstreamRating: Int = 0
}
data class BitcoinZeroMq(
val host: String = "127.0.0.1",
val port: Int

198
src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt
View File

@@ -86,104 +86,29 @@ class UpstreamsConfigReader(
getList<MappingNode>(input, "upstreams")?.value?.forEachIndexed { _, upNode ->
val connNode = getMapping(upNode, "connection")
if (hasAny(connNode, "ethereum")) {
val connConfigNode = getMapping(connNode, "ethereum")!!
val upstream = UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>()
readUpstreamCommon(upNode, upstream)
readUpstreamStandard(upNode, upstream)
if (isValid(upstream)) {
config.upstreams.add(upstream)
val connection = UpstreamsConfig.EthereumConnection()
upstream.connection = connection
getMapping(connConfigNode, "rpc")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val http = UpstreamsConfig.HttpEndpoint(URI(url))
connection.rpc = http
http.basicAuth = authConfigReader.readClientBasicAuth(node)
http.tls = authConfigReader.readClientTls(node)
}
}
getMapping(connConfigNode, "ws")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val ws = UpstreamsConfig.WsEndpoint(URI(url))
connection.ws = ws
getValueAsString(node, "origin")?.let { origin ->
ws.origin = URI(origin)
}
ws.basicAuth = authConfigReader.readClientBasicAuth(node)
getValueAsBytes(node, "frameSize")?.let {
if (it < 65_535) {
throw IllegalStateException("frameSize cannot be less than 64Kb")
}
ws.frameSize = it
}
getValueAsBytes(node, "msgSize")?.let {
if (it < 65_535) {
throw IllegalStateException("msgSize cannot be less than 64Kb")
}
ws.msgSize = it
}
}
}
} else {
log.error("Upstream at #0 has invalid configuration")
readUpstream(config, upNode) {
readEthereumConnection(getMapping(connNode, "ethereum")!!)
}
} else if (hasAny(connNode, "bitcoin")) {
val connConfigNode = getMapping(connNode, "bitcoin")!!
val upstream = UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>()
readUpstreamCommon(upNode, upstream)
readUpstreamStandard(upNode, upstream)
if (isValid(upstream)) {
config.upstreams.add(upstream)
val connection = UpstreamsConfig.BitcoinConnection()
upstream.connection = connection
getMapping(connConfigNode, "rpc")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val http = UpstreamsConfig.HttpEndpoint(URI(url))
connection.rpc = http
http.basicAuth = authConfigReader.readClientBasicAuth(node)
http.tls = authConfigReader.readClientTls(node)
}
}
getMapping(connConfigNode, "esplora")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val http = UpstreamsConfig.HttpEndpoint(URI(url))
http.basicAuth = authConfigReader.readClientBasicAuth(node)
http.tls = authConfigReader.readClientTls(node)
connection.esplora = http
}
}
getMapping(connConfigNode, "zeromq")?.let { node ->
getValueAsString(node, "address")?.let { address ->
val zmqConfig: Pair<String, Int>? = try {
if (address.contains(":")) {
address.split(":").let {
Pair(it[0], it[1].toInt())
}
} else {
Pair("127.0.0.1", address.toInt())
}
} catch (t: Throwable) {
log.warn("Invalid config for ZeroMQ: $address. Expected to be in format HOST:PORT")
null
}
zmqConfig?.let {
connection.zeroMq = UpstreamsConfig.BitcoinZeroMq(it.first, it.second)
}
}
}
} else {
log.error("Upstream at #0 has invalid configuration")
readUpstream(config, upNode) {
readBitcoinConnection(getMapping(connNode, "bitcoin")!!)
}
} else if (hasAny(connNode, "ethereum-pos")) {
readUpstream(config, upNode) {
readEthereumPosConnection(getMapping(connNode, "ethereum-pos")!!)
}
} else if (hasAny(connNode, "grpc")) {
val connConfigNode = getMapping(connNode, "grpc")!!
val upstream = UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>()
readUpstreamCommon(upNode, upstream)
readUpstreamGrpc(upNode, upstream)
readUpstreamGrpc(upNode)
if (isValid(upstream)) {
config.upstreams.add(upstream)
val connection = UpstreamsConfig.GrpcConnection()
upstream.connection = connection
getValueAsInt(connConfigNode, "upstream-rating")?.let {
connection.upstreamRating = it
}
getValueAsString(connConfigNode, "host")?.let {
connection.host = it
}
@@ -200,6 +125,104 @@ class UpstreamsConfigReader(
return config
}
private fun readBitcoinConnection(connConfigNode: MappingNode): UpstreamsConfig.BitcoinConnection {
val connection = UpstreamsConfig.BitcoinConnection()
getMapping(connConfigNode, "rpc")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val http = UpstreamsConfig.HttpEndpoint(URI(url))
connection.rpc = http
http.basicAuth = authConfigReader.readClientBasicAuth(node)
http.tls = authConfigReader.readClientTls(node)
}
}
getMapping(connConfigNode, "esplora")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val http = UpstreamsConfig.HttpEndpoint(URI(url))
http.basicAuth = authConfigReader.readClientBasicAuth(node)
http.tls = authConfigReader.readClientTls(node)
connection.esplora = http
}
}
getMapping(connConfigNode, "zeromq")?.let { node ->
getValueAsString(node, "address")?.let { address ->
val zmqConfig: Pair<String, Int>? = try {
if (address.contains(":")) {
address.split(":").let {
Pair(it[0], it[1].toInt())
}
} else {
Pair("127.0.0.1", address.toInt())
}
} catch (t: Throwable) {
log.warn("Invalid config for ZeroMQ: $address. Expected to be in format HOST:PORT")
null
}
zmqConfig?.let {
connection.zeroMq = UpstreamsConfig.BitcoinZeroMq(it.first, it.second)
}
}
}
return connection
}
private fun readEthereumPosConnection(connConfigNode: MappingNode): UpstreamsConfig.EthereumPosConnection {
val connection = UpstreamsConfig.EthereumPosConnection()
getMapping(connConfigNode, "execution")?.let {
connection.execution = readEthereumConnection(it)
}
getValueAsInt(connConfigNode, "upstream-rating")?.let {
connection.upstreamRating = it
}
return connection
}
private fun readEthereumConnection(connConfigNode: MappingNode): UpstreamsConfig.EthereumConnection {
val connection = UpstreamsConfig.EthereumConnection()
getMapping(connConfigNode, "rpc")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val http = UpstreamsConfig.HttpEndpoint(URI(url))
connection.rpc = http
http.basicAuth = authConfigReader.readClientBasicAuth(node)
http.tls = authConfigReader.readClientTls(node)
}
}
getMapping(connConfigNode, "ws")?.let { node ->
getValueAsString(node, "url")?.let { url ->
val ws = UpstreamsConfig.WsEndpoint(URI(url))
connection.ws = ws
getValueAsString(node, "origin")?.let { origin ->
ws.origin = URI(origin)
}
ws.basicAuth = authConfigReader.readClientBasicAuth(node)
getValueAsBytes(node, "frameSize")?.let {
if (it < 65_535) {
throw IllegalStateException("frameSize cannot be less than 64Kb")
}
ws.frameSize = it
}
getValueAsBytes(node, "msgSize")?.let {
if (it < 65_535) {
throw IllegalStateException("msgSize cannot be less than 64Kb")
}
ws.msgSize = it
}
}
}
return connection
}
private fun <T : UpstreamsConfig.UpstreamConnection> readUpstream(config: UpstreamsConfig, upNode: MappingNode, connFactory: () -> T) {
val upstream = UpstreamsConfig.Upstream<T>()
readUpstreamCommon(upNode, upstream)
readUpstreamStandard(upNode, upstream)
if (isValid(upstream)) {
config.upstreams.add(upstream)
upstream.connection = connFactory()
} else {
log.error("Upstream at #0 has invalid configuration")
}
}
fun isValid(upstream: UpstreamsConfig.Upstream<*>): Boolean {
val id = upstream.id
// In general, we just check that id is suitable for urls and references,
@@ -222,7 +245,6 @@ class UpstreamsConfigReader(
internal fun readUpstreamGrpc(
upNode: MappingNode,
upstream: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>
) {
// Dshackle gRPC connection dispatches requests to different upstreams, which may
// be on different blockchains, and each may have different set of labels.

7
src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt
View File

@@ -30,7 +30,8 @@ class BlockContainer(
val full: Boolean,
json: ByteArray?,
val parsed: Any?,
val transactions: List<TxId> = emptyList()
val transactions: List<TxId> = emptyList(),
val nodeRating: Int = 0
) : SourceContainer(json, parsed) {
companion object {
@@ -82,6 +83,10 @@ class BlockContainer(
return true
}
fun copyWithRating(nodeRating: Int): BlockContainer {
return BlockContainer(height, hash, difficulty, timestamp, full, json, parsed, transactions, nodeRating)
}
override fun hashCode(): Int {
var result = super.hashCode()
result = 31 * result + height.hashCode()

40
src/main/kotlin/io/emeraldpay/dshackle/data/RingSet.kt Normal file
View File

@@ -0,0 +1,40 @@
package io.emeraldpay.dshackle.data
import java.util.concurrent.atomic.AtomicReference
class RingSet<T>(
private val maxSize: Int
) : Set<T> {
private var setRef: AtomicReference<LinkedHashSet<T>> = AtomicReference(LinkedHashSet<T>())
override val size: Int
get() = setRef.get().size
fun add(element: T) {
setRef.getAndUpdate { set ->
if (!set.contains(element)) {
val copyset = LinkedHashSet<T>(set)
copyset.add(element)
if (copyset.size > maxSize) {
copyset.remove(set.elementAt(0))
}
copyset
} else {
set
}
}
}
override fun isEmpty(): Boolean {
return setRef.get().isEmpty()
}
override fun contains(element: @UnsafeVariance T): Boolean {
return setRef.get().contains(element)
}
override fun iterator(): Iterator<T> {
return setRef.get().iterator()
}
override fun containsAll(elements: Collection<@UnsafeVariance T>): Boolean {
return setRef.get().containsAll(elements)
}
}

3
src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt
View File

@@ -25,6 +25,7 @@ import io.grpc.ServerInterceptor
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import java.time.Instant
@Service
class AccessHandlerGrpc(
@@ -119,7 +120,7 @@ class AccessHandlerGrpc(
): ServerCall.Listener<ReqT> {
return process(
call, headers, next,
EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT>
EventsBuilder.NativeCall(Instant.now()) as EventsBuilder.RequestReply<*, ReqT, RespT>
)
}

6
src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt
View File

@@ -127,11 +127,13 @@ class AccessHandlerHttp(
private val accessLogWriter: AccessLogWriter,
private val channel: Events.Channel
) : RequestHandler {
protected var startTs: Instant? = null
protected var request: BlockchainOuterClass.NativeCallRequest? = null
protected val responses = ArrayList<NativeCall.CallResult>()
protected val updateLock = ReentrantLock()
override fun onRequest(request: BlockchainOuterClass.NativeCallRequest) {
this.startTs = Instant.now()
this.request = request
}
@@ -164,7 +166,7 @@ class AccessHandlerHttp(
if (request == null) {
return
}
val builder = EventsBuilder.NativeCall()
val builder = EventsBuilder.NativeCall(startTs!!)
builder.withChain(blockchain.id)
builder.start(httpRequest)
builder.onRequest(request!!)
@@ -182,7 +184,7 @@ class AccessHandlerHttp(
if (request == null) {
return
}
val builder = EventsBuilder.NativeCall()
val builder = EventsBuilder.NativeCall(startTs!!)
builder.withChain(blockchain.id)
builder.start(wsRequest)
builder.onRequest(request!!)

2
src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/Events.kt
View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.monitoring.accesslog
import com.fasterxml.jackson.annotation.JsonInclude
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import java.time.Duration
import java.time.Instant
import java.util.UUID
@@ -104,6 +105,7 @@ class Events {
val selector: String? = null,
val quorum: Long? = null,
val minAvailability: String? = null,
val latency: Long,
val succeed: Boolean,
val rpcError: Int? = null,

5
src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/EventsBuilder.kt
View File

@@ -30,6 +30,7 @@ import reactor.netty.http.server.HttpServerRequest
import reactor.netty.http.websocket.WebsocketInbound
import java.net.InetAddress
import java.net.InetSocketAddress
import java.time.Duration
import java.time.Instant
import java.util.Locale
import java.util.UUID
@@ -294,7 +295,7 @@ class EventsBuilder {
}
}
class NativeCall :
class NativeCall(private val startTs : Instant) :
Base<NativeCall>(),
RequestReply<Events.NativeCall, BlockchainOuterClass.NativeCallRequest, BlockchainOuterClass.NativeCallReplyItem> {
val items = ArrayList<Events.NativeCallItemDetails>()
@@ -330,6 +331,7 @@ class EventsBuilder {
index = index++,
succeed = msg.succeed,
blockchain = chain,
latency = Duration.between(Instant.now(), startTs).toMillis(),
nativeCall = item,
payloadSizeBytes = item.payloadSizeBytes,
id = UUID.randomUUID(),
@@ -354,6 +356,7 @@ class EventsBuilder {
index = index++,
succeed = !reply.isError(),
blockchain = chain,
latency = Duration.between(startTs, Instant.now()).toMillis(),
nativeCall = item,
payloadSizeBytes = item.payloadSizeBytes,
id = UUID.randomUUID(),

4
src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt
View File

@@ -20,7 +20,7 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.grpc.Status
@@ -83,7 +83,7 @@ open class NativeSubscribe(
open fun subscribe(chain: Chain, method: String, params: Any?): Flux<out Any> {
val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain))
return (up as EthereumMultistream)
return (up as EthereumLikeMultistream)
.getSubscribe()
.subscribe(method, params)
}

189
src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt
View File

@@ -22,7 +22,9 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.CurrentMultistreamHolder
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HttpRpcFactory
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcHead
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinZMQHead
@@ -31,20 +33,18 @@ import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
import io.emeraldpay.dshackle.upstream.bitcoin.ZMQServer
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosRpcUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsUpstream
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstreams
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
@@ -83,18 +83,24 @@ open class ConfiguredUpstreams(
}
val options = (up.options ?: UpstreamsConfig.Options())
.merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults())
when (BlockchainType.from(chain)) {
val upstream = when (BlockchainType.from(chain)) {
BlockchainType.ETHEREUM -> {
buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options)
}
BlockchainType.BITCOIN -> {
buildBitcoinUpstream(up.cast(UpstreamsConfig.BitcoinConnection::class.java), chain, options)
}
BlockchainType.ETHEREUM_POS -> {
buildEthereumPosUpstream(up.cast(UpstreamsConfig.EthereumPosConnection::class.java), chain, options)
}
else -> {
log.error("Chain is unsupported: ${up.chain}")
return@forEach
}
}
upstream?.let {
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
}
}
}
}
@@ -141,19 +147,47 @@ open class ConfiguredUpstreams(
}
}
private fun buildEthereumPosUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumPosConnection>,
chain: Chain,
options: UpstreamsConfig.Options
): Upstream? {
val conn = config.connection!!
val execution = conn.execution
if (execution == null) {
log.warn("Upstream doesn't have execution layer configuration")
return null
}
val urls = ArrayList<URI>()
val connectorFactory = buildEthereumConnectorFactory(config.id!!, execution, chain, urls, NoChoiceWithPriorityForkChoice(conn.upstreamRating))
val methods = buildMethods(config, chain)
if (connectorFactory == null) {
return null
}
val upstream = EthereumPosRpcUpstream(
config.id!!,
chain,
options, config.role,
methods,
QuorumForLabels.QuorumItem(1, config.labels),
connectorFactory
)
upstream.start()
return upstream
}
private fun buildBitcoinUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.BitcoinConnection>,
chain: Chain,
options: UpstreamsConfig.Options
) {
): Upstream? {
val conn = config.connection!!
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
if (directApi == null) {
val httpFactory = buildHttpFactory(conn)
if (httpFactory == null) {
log.warn("Upstream doesn't have API configuration")
return
return null
}
val directApi: Reader<JsonRpcRequest, JsonRpcResponse> = httpFactory.create(config.id, chain)
val esplora = conn.esplora?.let { endpoint ->
val tls = endpoint.tls?.let { tls ->
tls.ca?.let { ca ->
@@ -168,7 +202,7 @@ open class ConfiguredUpstreams(
val head: Head = conn.zeroMq?.let { zeroMq ->
val server = ZMQServer(zeroMq.host, zeroMq.port, "hashblock")
val zeroMqHead = BitcoinZMQHead(server, directApi, extractBlock)
MergedHead(listOf(rpcHead, zeroMqHead))
MergedHead(listOf(rpcHead, zeroMqHead), MostWorkForkChoice())
} ?: rpcHead
val methods = buildMethods(config, chain)
@@ -180,66 +214,34 @@ open class ConfiguredUpstreams(
QuorumForLabels.QuorumItem(1, config.labels),
methods, esplora
)
upstream.start()
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
return upstream
}
private fun buildEthereumUpstream(
config: UpstreamsConfig.Upstream<UpstreamsConfig.EthereumConnection>,
chain: Chain,
options: UpstreamsConfig.Options
) {
): EthereumRpcUpstream? {
val conn = config.connection!!
val urls = ArrayList<URI>()
val methods = buildMethods(config, chain)
conn.rpc?.let { endpoint ->
urls.add(endpoint.url)
val connectorFactory = buildEthereumConnectorFactory(config.id!!, conn, chain, urls, MostWorkForkChoice())
if (connectorFactory == null) {
return null
}
val wsFactoryApi: EthereumWsFactory? = conn.ws?.let { endpoint ->
val wsApi = EthereumWsFactory(
config.id!!, chain,
endpoint.url,
endpoint.origin ?: URI("http://localhost"),
)
wsApi.config = endpoint
endpoint.basicAuth?.let { auth ->
wsApi.basicAuth = auth
}
urls.add(endpoint.url)
wsApi
}
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
val directApi: Reader<JsonRpcRequest, JsonRpcResponse>? = buildHttpClient(config)
if (directApi == null) {
log.warn("Upstream doesn't have API configuration")
return
}
val ethereumUpstream = if (wsFactoryApi != null && !conn.preferHttp) {
EthereumWsUpstream(
config.id!!,
chain, directApi, wsFactoryApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods
)
} else {
EthereumRpcUpstream(
config.id!!,
chain, directApi, wsFactoryApi,
options, config.role,
QuorumForLabels.QuorumItem(1, config.labels),
methods
)
}
ethereumUpstream.start()
currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED))
val upstream = EthereumRpcUpstream(
config.id!!,
chain,
options, config.role,
methods,
QuorumForLabels.QuorumItem(1, config.labels),
connectorFactory
)
upstream.start()
return upstream
}
private fun buildGrpcUpstream(
@@ -253,7 +255,8 @@ open class ConfiguredUpstreams(
endpoint.host!!,
endpoint.port,
endpoint.auth,
fileResolver
fileResolver,
endpoint.upstreamRating
).apply {
timeout = options.timeout
}
@@ -265,39 +268,43 @@ open class ConfiguredUpstreams(
.subscribe(currentUpstreams::update)
}
private fun buildHttpClient(config: UpstreamsConfig.Upstream<out UpstreamsConfig.RpcConnection>): JsonRpcHttpClient? {
val conn = config.connection!!
val urls = ArrayList<URI>()
private fun buildHttpFactory(conn: UpstreamsConfig.RpcConnection, urls: ArrayList<URI>? = null): HttpRpcFactory? {
return conn.rpc?.let { endpoint ->
val tls = conn.rpc?.tls?.let { tls ->
tls.ca?.let { ca ->
fileResolver.resolve(ca).readBytes()
}
}
val metricsTags = listOf(
// "unknown" is not supposed to happen
Tag.of("upstream", config.id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", (Global.chainById(config.chain).chainCode))
)
val metrics = RpcMetrics(
Timer.builder("upstream.rpc.conn")
.description("Request time through a HTTP JSON RPC connection")
.tags(metricsTags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry),
Counter.builder("upstream.rpc.fail")
.description("Number of failures of HTTP JSON RPC requests")
.tags(metricsTags)
.register(Metrics.globalRegistry)
)
urls.add(endpoint.url)
JsonRpcHttpClient(
endpoint.url.toString(),
metrics,
conn.rpc?.basicAuth,
tls
)
urls?.add(endpoint.url)
HttpRpcFactory(endpoint.url.toString(), conn.rpc?.basicAuth, tls)
}
}
private fun buildWsFactory(id: String, chain: Chain, conn: UpstreamsConfig.EthereumConnection, urls: ArrayList<URI>? = null): EthereumWsFactory? {
return conn.ws?.let { endpoint ->
val wsApi = EthereumWsFactory(
id, chain,
endpoint.url,
endpoint.origin ?: URI("http://localhost"),
)
wsApi.config = endpoint
endpoint.basicAuth?.let { auth ->
wsApi.basicAuth = auth
}
urls?.add(endpoint.url)
wsApi
}
}
private fun buildEthereumConnectorFactory(id: String, conn: UpstreamsConfig.EthereumConnection, chain: Chain, urls: ArrayList<URI>, forkChoice: ForkChoice): EthereumConnectorFactory? {
val wsFactoryApi = buildWsFactory(id, chain, conn, urls)
val httpFactory = buildHttpFactory(conn, urls)
log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}")
val connectorFactory = EthereumConnectorFactory(conn.preferHttp, wsFactoryApi, httpFactory, forkChoice)
if (!connectorFactory.isValid()) {
log.warn("Upstream configuration is invalid (probably no http endpoint)")
return null
}
return connectorFactory
}
}

39
src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt
View File

@@ -16,21 +16,22 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import org.slf4j.LoggerFactory
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.core.publisher.Sinks
import reactor.core.scheduler.Schedulers
import java.util.concurrent.atomic.AtomicReference
import reactor.kotlin.core.publisher.toMono
abstract class AbstractHead : Head {
abstract class AbstractHead(
private val forkChoice: ForkChoice
) : Head {
companion object {
private val log = LoggerFactory.getLogger(AbstractHead::class.java)
}
private val head = AtomicReference<BlockContainer>(null)
private var stream = Sinks.many().multicast().directBestEffort<BlockContainer>()
private var completed = false
private val beforeBlockHandlers = ArrayList<Runnable>()
@@ -44,10 +45,8 @@ abstract class AbstractHead : Head {
return source
.distinctUntilChanged {
it.hash
}.filter { block ->
val curr = head.get()
curr == null || curr.difficulty < block.difficulty
}
.filter { forkChoice.filter(it) }
.doFinally {
// close internal stream if upstream is finished, otherwise it gets stuck,
// but technically it should never happen during normal work, only when the Head
@@ -58,19 +57,16 @@ abstract class AbstractHead : Head {
.subscribeOn(Schedulers.boundedElastic())
.subscribe { block ->
notifyBeforeBlock()
val prev = head.getAndUpdate { curr ->
if (curr == null || curr.difficulty < block.difficulty) {
block
} else {
curr
}
}
if (prev == null || prev.hash != block.hash) {
log.debug("New block ${block.height} ${block.hash}")
val result = stream.tryEmitNext(block)
if (result.isFailure && result != Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) {
log.warn("Failed to dispatch block: $result as ${this.javaClass}")
when (val choiceResult = forkChoice.choose(block)) {
is ForkChoice.ChoiceResult.Updated -> {
val newHead = choiceResult.nwhead
log.debug("New block ${newHead.height} ${newHead.hash}")
val result = stream.tryEmitNext(newHead)
if (result.isFailure && result != Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) {
log.warn("Failed to dispatch block: $result as ${this.javaClass}")
}
}
is ForkChoice.ChoiceResult.Same -> {}
}
}
}
@@ -90,14 +86,15 @@ abstract class AbstractHead : Head {
}
override fun getFlux(): Flux<BlockContainer> {
val curHead = forkChoice.getHead()
return Flux.concat(
Mono.justOrEmpty(head.get()),
forkChoice.getHead().toMono(),
stream.asFlux()
).onBackpressureLatest()
}
fun getCurrent(): BlockContainer? {
return head.get()
return forkChoice.getHead()
}
override fun getCurrentHeight(): Long? {

12
src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt
View File

@@ -24,8 +24,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.ethereum.*
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
@@ -68,6 +67,14 @@ open class CurrentMultistreamHolder(
}
processUpdate(change, up, current, factory)
}
BlockchainType.ETHEREUM_POS -> {
val up = change.upstream.cast(EthereumPosUpstream::class.java)
val current = chainMapping[chain]
val factory = Callable<Multistream> {
EthereumPosMultistream(chain, ArrayList(), cachesFactory.getCaches(chain))
}
processUpdate(change, up, current, factory)
}
BlockchainType.BITCOIN -> {
val up = change.upstream.cast(BitcoinUpstream::class.java)
val current = chainMapping[chain]
@@ -137,6 +144,7 @@ open class CurrentMultistreamHolder(
val created = when (BlockchainType.from(chain)) {
BlockchainType.ETHEREUM -> DefaultEthereumMethods(chain)
BlockchainType.BITCOIN -> DefaultBitcoinMethods()
BlockchainType.ETHEREUM_POS -> DefaultEthereumMethods(chain)
else -> throw IllegalStateException("Unsupported chain: $chain")
}
callTargets[chain] = created

28
src/main/kotlin/io/emeraldpay/dshackle/upstream/DistanceExtractor.kt Normal file
View File

@@ -0,0 +1,28 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.data.BlockContainer
class DistanceExtractor {
sealed class ChainDistance {
data class Distance(val dist: Long) : ChainDistance()
object Fork : ChainDistance()
}
companion object {
fun extractPowDistance(top: BlockContainer, curr: BlockContainer): ChainDistance {
return when {
curr.height > top.height -> if (curr.difficulty >= top.difficulty) ChainDistance.Distance(0) else ChainDistance.Fork
curr.height == top.height -> if (curr.difficulty == top.difficulty) ChainDistance.Distance(0) else ChainDistance.Fork
else -> ChainDistance.Distance(top.height - curr.height)
}
}
fun extractPriorityDistance(top: BlockContainer, curr: BlockContainer): ChainDistance {
return when {
curr.height > top.height -> ChainDistance.Fork
curr.height == top.height -> if (curr.hash == top.hash) ChainDistance.Distance(0) else ChainDistance.Fork
else -> ChainDistance.Distance(top.height - curr.height)
}
}
}
}

11
src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt
View File

@@ -30,9 +30,11 @@ import java.time.Duration
* Observer group of upstreams and defined a distance in blocks (lag) between a leader (best height/difficulty) and
* other upstreams.
*/
typealias Extractor = (top: BlockContainer, curr: BlockContainer) -> DistanceExtractor.ChainDistance
abstract class HeadLagObserver(
private val master: Head,
private val followers: Collection<Upstream>
private val followers: Collection<Upstream>,
private val distanceExtractor: Extractor
) : Lifecycle {
private val log = LoggerFactory.getLogger(HeadLagObserver::class.java)
@@ -85,10 +87,9 @@ abstract class HeadLagObserver(
}
open fun extractDistance(top: BlockContainer, curr: BlockContainer): Long {
return when {
curr.height > top.height -> if (curr.difficulty >= top.difficulty) 0 else forkDistance(top, curr)
curr.height == top.height -> if (curr.difficulty == top.difficulty) 0 else forkDistance(top, curr)
else -> top.height - curr.height
return when (val distance = distanceExtractor(top, curr)) {
is DistanceExtractor.ChainDistance.Distance -> distance.dist
is DistanceExtractor.ChainDistance.Fork -> forkDistance(top, curr)
}
}

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpFactory.kt Normal file
View File

@@ -0,0 +1,10 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
interface HttpFactory {
fun create(id: String?, chain: Chain): Reader<JsonRpcRequest, JsonRpcResponse>
}

45
src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpRpcFactory.kt Normal file
View File

@@ -0,0 +1,45 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer
open class HttpRpcFactory(
private val url: String,
private val basicAuth: AuthConfig.ClientBasicAuth?,
private val tls: ByteArray?
) : HttpFactory {
override fun create(id: String?, chain: Chain): Reader<JsonRpcRequest, JsonRpcResponse> {
val metricsTags = listOf(
// "unknown" is not supposed to happen
Tag.of("upstream", id ?: "unknown"),
// UNSPECIFIED shouldn't happen too
Tag.of("chain", chain.chainCode)
)
val metrics = RpcMetrics(
Timer.builder("upstream.rpc.conn")
.description("Request time through a HTTP JSON RPC connection")
.tags(metricsTags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry),
Counter.builder("upstream.rpc.fail")
.description("Number of failures of HTTP JSON RPC requests")
.tags(metricsTags)
.register(Metrics.globalRegistry)
)
return JsonRpcHttpClient(
url,
metrics,
basicAuth,
tls
)
}
}

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt
View File

@@ -18,13 +18,15 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
class MergedHead(
private val sources: Iterable<Head>
) : AbstractHead(), Lifecycle, CachesEnabled {
private val sources: Iterable<Head>,
forkChoice: ForkChoice
) : AbstractHead(forkChoice), Lifecycle, CachesEnabled {
private var subscription: Disposable? = null

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinHeadLagObserver.kt
View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.DistanceExtractor
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HeadLagObserver
import io.emeraldpay.dshackle.upstream.Upstream
@@ -24,7 +25,7 @@ import org.slf4j.LoggerFactory
class BitcoinHeadLagObserver(
master: Head,
followers: Collection<Upstream>
) : HeadLagObserver(master, followers) {
) : HeadLagObserver(master, followers, DistanceExtractor::extractPowDistance) {
companion object {
private val log = LoggerFactory.getLogger(BitcoinHeadLagObserver::class.java)

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt
View File

@@ -26,6 +26,8 @@ import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.RequestPostprocessor
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.bitcoin.LocalCallRouter
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -90,7 +92,7 @@ open class BitcoinMultistream(
}
}
} else {
val newHead = MergedHead(sourceUpstreams.map { it.getHead() }).apply {
val newHead = MergedHead(sourceUpstreams.map { it.getHead() }, MostWorkForkChoice()).apply {
this.start()
}
val lagObserver = BitcoinHeadLagObserver(newHead, sourceUpstreams)

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt
View File

@@ -19,6 +19,7 @@ import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
@@ -35,7 +36,7 @@ class BitcoinRpcHead(
private val api: Reader<JsonRpcRequest, JsonRpcResponse>,
private val extractBlock: ExtractBlock,
private val interval: Duration = Duration.ofSeconds(15)
) : Head, AbstractHead(), Lifecycle {
) : Head, AbstractHead(MostWorkForkChoice()), Lifecycle {
companion object {
private val log = LoggerFactory.getLogger(BitcoinRpcHead::class.java)

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinZMQHead.kt
View File

@@ -5,6 +5,7 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.apache.commons.codec.binary.Hex
@@ -20,7 +21,7 @@ class BitcoinZMQHead(
private val server: ZMQServer,
private val api: Reader<JsonRpcRequest, JsonRpcResponse>,
private val extractBlock: ExtractBlock,
) : Head, AbstractHead(), Lifecycle {
) : Head, AbstractHead(MostWorkForkChoice()), Lifecycle {
companion object {
private val log = LoggerFactory.getLogger(BitcoinZMQHead::class.java)

5
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt
View File

@@ -20,13 +20,16 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.hex.HexQuantity
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
open class DefaultEthereumHead : Head, AbstractHead() {
open class DefaultEthereumHead(
forkChoice: ForkChoice
) : Head, AbstractHead(forkChoice) {
companion object {
private val log = LoggerFactory.getLogger(DefaultEthereumHead::class.java)

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/ERC20Balance.kt
View File

@@ -49,7 +49,7 @@ open class ERC20Balance {
apis.request(1)
return Flux.from(apis)
.flatMap {
getBalance(it.cast(EthereumUpstream::class.java), token, address)
getBalance(it.cast(EthereumRpcUpstream::class.java), token, address)
}
.doOnNext {
apis.resolve()
@@ -57,7 +57,7 @@ open class ERC20Balance {
.next()
}
open fun getBalance(upstream: EthereumUpstream, token: ERC20Token, address: Address): Mono<BigInteger> {
open fun getBalance(upstream: EthereumRpcUpstream, token: ERC20Token, address: Address): Mono<BigInteger> {
return upstream
.getApi()
.read(prepareEthCall(token, address, upstream.getHead()))

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFees.kt
View File

@@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.AbstractChainFees
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.etherjar.domain.Wei
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
@@ -28,7 +29,7 @@ import reactor.util.function.Tuples
import java.util.function.Function
abstract class EthereumFees(
upstreams: EthereumMultistream,
upstreams: Multistream,
private val reader: EthereumReader,
heightLimit: Int,
) : AbstractChainFees<EthereumFees.EthereumFee, BlockJson<TransactionRefJson>, TransactionRefJson, TransactionJson>(heightLimit, upstreams, extractTx), ChainFees {

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt
View File

@@ -17,6 +17,7 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.DistanceExtractor
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HeadLagObserver
import io.emeraldpay.dshackle.upstream.Upstream
@@ -25,7 +26,7 @@ import org.slf4j.LoggerFactory
class EthereumHeadLagObserver(
master: Head,
followers: Collection<Upstream>
) : HeadLagObserver(master, followers) {
) : HeadLagObserver(master, followers, DistanceExtractor::extractPowDistance) {
companion object {
private val log = LoggerFactory.getLogger(EthereumHeadLagObserver::class.java)

8
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLikeMultistream.kt Normal file
View File

@@ -0,0 +1,8 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.Upstream
interface EthereumLikeMultistream : Upstream {
fun getReader(): EthereumReader
fun getSubscribe(): EthereumSubscribe
}

9
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt
View File

@@ -25,6 +25,7 @@ import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
@@ -37,7 +38,7 @@ open class EthereumMultistream(
chain: Chain,
val upstreams: MutableList<EthereumUpstream>,
caches: Caches
) : Multistream(chain, upstreams as MutableList<Upstream>, caches, CacheRequested(caches)) {
) : Multistream(chain, upstreams as MutableList<Upstream>, caches, CacheRequested(caches)), EthereumLikeMultistream {
companion object {
private val log = LoggerFactory.getLogger(EthereumMultistream::class.java)
@@ -79,7 +80,7 @@ open class EthereumMultistream(
return super.isRunning() || reader.isRunning
}
open fun getReader(): EthereumReader {
override fun getReader(): EthereumReader {
return reader
}
@@ -109,7 +110,7 @@ open class EthereumMultistream(
}
} else {
val heads = upstreams.map { it.getHead() }
val newHead = MergedHead(heads).apply {
val newHead = MergedHead(heads, MostWorkForkChoice()).apply {
this.start()
}
val lagObserver = EthereumHeadLagObserver(newHead, upstreams as Collection<Upstream>)
@@ -137,7 +138,7 @@ open class EthereumMultistream(
return Mono.just(LocalCallRouter(reader, getMethods(), getHead()))
}
open fun getSubscribe(): EthereumSubscribe {
override fun getSubscribe(): EthereumSubscribe {
return subscribe
}

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumPriorityFees.kt
View File

@@ -16,6 +16,7 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.etherjar.domain.Wei
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
@@ -23,7 +24,7 @@ import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import org.slf4j.LoggerFactory
import java.util.function.Function
class EthereumPriorityFees(upstreams: EthereumMultistream, reader: EthereumReader, heightLimit: Int) :
class EthereumPriorityFees(upstreams: Multistream, reader: EthereumReader, heightLimit: Int) :
EthereumFees(upstreams, reader, heightLimit) {
companion object {

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt
View File

@@ -17,6 +17,7 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
@@ -30,8 +31,9 @@ import java.util.concurrent.Executors
class EthereumRpcHead(
private val api: Reader<JsonRpcRequest, JsonRpcResponse>,
private val interval: Duration = Duration.ofSeconds(10)
) : DefaultEthereumHead(), Lifecycle {
forkChoice: ForkChoice,
private val interval: Duration = Duration.ofSeconds(10),
) : DefaultEthereumHead(forkChoice), Lifecycle {
companion object {
val scheduler =

88
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt
View File

@@ -1,3 +1,19 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
* Copyright (c) 2019 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.cache.Caches
@@ -6,106 +22,68 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import java.time.Duration
open class EthereumRpcUpstream(
id: String,
val chain: Chain,
private val directReader: Reader<JsonRpcRequest, JsonRpcResponse>,
private val ethereumWsFactory: EthereumWsFactory? = null,
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
private val node: QuorumForLabels.QuorumItem,
targets: CallMethods
) : EthereumUpstream(id, options, role, targets, node), Upstream, CachesEnabled, Lifecycle {
constructor(id: String, chain: Chain, api: Reader<JsonRpcRequest, JsonRpcResponse>) :
this(
id, chain, api, null,
UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.UpstreamRole.PRIMARY,
QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()),
DirectCallMethods()
)
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory
) : EthereumUpstream(id, options, role, targets, node), Lifecycle, Upstream, CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java)
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain)
private val head: Head = this.createHead()
private var validatorSubscription: Disposable? = null
override fun setCaches(caches: Caches) {
if (head is CachesEnabled) {
head.setCaches(caches)
if (connector is CachesEnabled) {
connector.setCaches(caches)
}
}
override fun start() {
log.info("Configured for ${chain.chainName}")
connector.start()
if (getOptions().disableValidation != null && getOptions().disableValidation!!) {
log.warn("Disable validation for upstream ${this.getId()}")
this.setLag(0)
this.setStatus(UpstreamAvailability.OK)
} else {
log.debug("Start validation for upstream ${this.getId()}")
val validator = EthereumUpstreamValidator(this, getOptions())
validatorSubscription = validator.start()
.subscribe(this::setStatus)
}
}
override fun isRunning(): Boolean {
return true
override fun getHead(): Head {
return connector.getHead()
}
override fun stop() {
validatorSubscription?.dispose()
validatorSubscription = null
if (head is Lifecycle) {
head.stop()
}
connector.stop()
}
open fun createHead(): Head {
return if (ethereumWsFactory != null) {
// do not set upstream to the WS, since it doesn't control the RPC upstream
val ws = ethereumWsFactory.create(null, null).apply {
connect()
}
val wsHead = EthereumWsHead(ws).apply {
start()
}
// receive bew blocks through WebSockets, but also periodically verify with RPC in case if WS failed
val rpcHead = EthereumRpcHead(getApi(), Duration.ofSeconds(60)).apply {
start()
}
MergedHead(listOf(rpcHead, wsHead)).apply {
start()
}
} else {
log.warn("Setting up upstream ${this.getId()} with RPC-only access, less effective than WS+RPC")
EthereumRpcHead(getApi()).apply {
start()
}
}
}
override fun getHead(): Head {
return head
override fun isRunning(): Boolean {
return connector.isRunning
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {
return directReader
return connector.getApi()
}
override fun isGrpc(): Boolean {

2
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt
View File

@@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
open class EthereumSubscribe(
val upstream: EthereumMultistream
val upstream: EthereumLikeMultistream
) {
companion object {

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt
View File

@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -34,7 +35,7 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeoutException
open class EthereumUpstreamValidator(
private val upstream: EthereumUpstream,
private val upstream: Upstream,
private val options: UpstreamsConfig.Options
) {
companion object {

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt
View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
@@ -23,8 +24,9 @@ import reactor.core.Disposable
import reactor.core.publisher.Flux
class EthereumWsHead(
private val ws: WsConnection
) : DefaultEthereumHead(), Lifecycle {
private val ws: WsConnection,
forkChoice: ForkChoice
) : DefaultEthereumHead(forkChoice), Lifecycle {
private val log = LoggerFactory.getLogger(EthereumWsHead::class.java)

10
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/ConnectorFactory.kt Normal file
View File

@@ -0,0 +1,10 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.grpc.Chain
interface ConnectorFactory {
fun create(upstream: DefaultUpstream, validator: EthereumUpstreamValidator, chain: Chain): EthereumConnector
fun isValid(): Boolean
}

13
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnector.kt Normal file
View File

@@ -0,0 +1,13 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.springframework.context.Lifecycle
interface EthereumConnector : Lifecycle {
fun getHead(): Head
fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse>
}

35
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumConnectorFactory.kt Normal file
View File

@@ -0,0 +1,35 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.HttpFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
open class EthereumConnectorFactory(
private val preferHttp: Boolean,
private val wsFactory: EthereumWsFactory?,
private val httpFactory: HttpFactory?,
private val forkChoice: ForkChoice
) : ConnectorFactory {
private val log = LoggerFactory.getLogger(EthereumConnectorFactory::class.java)
override fun isValid(): Boolean {
if (preferHttp && httpFactory == null) {
return false
}
return true
}
override fun create(upstream: DefaultUpstream, validator: EthereumUpstreamValidator, chain: Chain): EthereumConnector {
if (wsFactory != null && !preferHttp) {
return EthereumWsConnector(wsFactory, upstream, validator, chain, forkChoice)
}
if (httpFactory == null) {
throw java.lang.IllegalArgumentException("Can't create rpc connector if no http factory set")
}
return EthereumRpcConnector(httpFactory.create(upstream.getId(), chain), wsFactory, upstream.getId(), forkChoice)
}
}

81
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumRpcConnector.kt Normal file
View File

@@ -0,0 +1,81 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead
import io.emeraldpay.dshackle.upstream.ethereum.WsConnection
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import java.time.Duration
class EthereumRpcConnector(
private val directReader: Reader<JsonRpcRequest, JsonRpcResponse>,
wsFactory: EthereumWsFactory?,
id: String,
forkChoice: ForkChoice
) : EthereumConnector, CachesEnabled {
private val conn: WsConnection?
private val head: Head
companion object {
private val log = LoggerFactory.getLogger(EthereumRpcConnector::class.java)
}
init {
if (wsFactory != null) {
// do not set upstream to the WS, since it doesn't control the RPC upstream
conn = wsFactory.create(null, null)
val wsHead = EthereumWsHead(conn, forkChoice)
// receive bew blocks through WebSockets, but also periodically verify with RPC in case if WS failed
val rpcHead = EthereumRpcHead(directReader, forkChoice, Duration.ofSeconds(60))
head = MergedHead(listOf(rpcHead, wsHead), forkChoice)
} else {
conn = null
log.warn("Setting up connector for $id upstream with RPC-only access, less effective than WS+RPC")
head = EthereumRpcHead(directReader, forkChoice)
}
}
override fun setCaches(caches: Caches) {
if (head is CachesEnabled) {
head.setCaches(caches)
}
}
override fun start() {
if (head is Lifecycle) {
head.start()
}
conn?.connect()
}
override fun isRunning(): Boolean {
if (head is Lifecycle) {
return head.isRunning
}
return true
}
override fun stop() {
if (head is Lifecycle) {
head.stop()
}
conn?.close()
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {
return directReader
}
override fun getHead(): Head {
return head
}
}

59
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/connectors/EthereumWsConnector.kt Normal file
View File

@@ -0,0 +1,59 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsHead
import io.emeraldpay.dshackle.upstream.ethereum.WsConnection
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.Timer
class EthereumWsConnector(
wsFactory: EthereumWsFactory,
upstream: DefaultUpstream,
validator: EthereumUpstreamValidator,
chain: Chain,
forkChoice: ForkChoice
) : EthereumConnector {
private val conn: WsConnection
private val api: Reader<JsonRpcRequest, JsonRpcResponse>
private val head: EthereumWsHead
init {
conn = wsFactory.create(upstream, validator)
head = EthereumWsHead(conn, forkChoice)
api = JsonRpcWsClient(conn)
}
override fun start() {
conn.connect()
head.start()
}
override fun isRunning(): Boolean {
return head.isRunning
}
override fun stop() {
conn.close()
head.stop()
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {
return api
}
override fun getHead(): Head {
return head
}
}

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt
View File

@@ -19,7 +19,7 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.scheduler.Schedulers
@@ -32,7 +32,7 @@ import kotlin.concurrent.withLock
import kotlin.concurrent.write
class ConnectBlockUpdates(
private val upstream: EthereumMultistream
private val upstream: EthereumLikeMultistream
) {
companion object {

6
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt
View File

@@ -15,7 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.hex.Hex32
@@ -25,7 +25,7 @@ import reactor.core.publisher.Flux
import java.util.function.Function
open class ConnectLogs(
upstream: EthereumMultistream,
upstream: EthereumLikeMultistream,
private val connectBlockUpdates: ConnectBlockUpdates,
) {
@@ -36,7 +36,7 @@ open class ConnectLogs(
private val TOPIC_COMPARATOR = HexDataComparator()
}
constructor(upstream: EthereumMultistream) : this(upstream, ConnectBlockUpdates(upstream))
constructor(upstream: EthereumLikeMultistream) : this(upstream, ConnectBlockUpdates(upstream))
private val produceLogs = ProduceLogs(upstream)

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt
View File

@@ -15,7 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.NewHeadMessage
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
@@ -28,7 +28,7 @@ import kotlin.concurrent.withLock
* Connects/reconnects to the upstream to produce NewHeads messages
*/
class ConnectNewHeads(
private val upstream: EthereumMultistream
private val upstream: EthereumLikeMultistream
) {
companion object {

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectSyncing.kt
View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.upstream.ethereum.subscribe
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import java.time.Duration
@@ -24,7 +24,7 @@ import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
class ConnectSyncing(
private val upstream: EthereumMultistream
private val upstream: EthereumLikeMultistream
) {
companion object {

4
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt
View File

@@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage
import io.emeraldpay.etherjar.hex.HexData
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
@@ -38,7 +38,7 @@ class ProduceLogs(
private val log = LoggerFactory.getLogger(ProduceLogs::class.java)
}
constructor(upstream: EthereumMultistream) : this(upstream.getReader().receipts())
constructor(upstream: EthereumLikeMultistream) : this(upstream.getReader().receipts())
private val objectMapper = Global.objectMapper

22
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosHeadLagObserver.kt Normal file
View File

@@ -0,0 +1,22 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.DistanceExtractor
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HeadLagObserver
import io.emeraldpay.dshackle.upstream.Upstream
import org.slf4j.LoggerFactory
class EthereumPostHeadLagObserver(
master: Head,
followers: Collection<Upstream>
) : HeadLagObserver(master, followers, DistanceExtractor::extractPriorityDistance) {
companion object {
private val log = LoggerFactory.getLogger(EthereumPostHeadLagObserver::class.java)
}
override fun forkDistance(top: BlockContainer, curr: BlockContainer): Long {
return 6
}
}

143
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosMultiStream.kt Normal file
View File

@@ -0,0 +1,143 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
* Copyright (c) 2020 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Mono
@Suppress("UNCHECKED_CAST")
open class EthereumPosMultistream(
chain: Chain,
val upstreams: MutableList<EthereumPosUpstream>,
caches: Caches
) : Multistream(chain, upstreams as MutableList<Upstream>, caches, CacheRequested(caches)), EthereumLikeMultistream {
companion object {
private val log = LoggerFactory.getLogger(EthereumPosMultistream::class.java)
}
private var head: Head? = null
private val reader: EthereumReader = EthereumReader(this, this.caches, getMethodsFactory())
private val feeEstimation = EthereumPriorityFees(this, reader, 256)
private val subscribe = EthereumSubscribe(this)
init {
this.init()
}
override fun init() {
if (upstreams.size > 0) {
head = updateHead()
}
super.init()
}
override fun start() {
super.start()
reader.start()
}
override fun stop() {
super.stop()
reader.stop()
}
override fun isRunning(): Boolean {
return super.isRunning() || reader.isRunning
}
override fun getReader(): EthereumReader {
return reader
}
override fun getHead(): Head {
return head!!
}
override fun setHead(head: Head) {
this.head = head
}
override fun updateHead(): Head {
head?.let {
if (it is Lifecycle) {
it.stop()
}
}
lagObserver?.stop()
lagObserver = null
val head = if (upstreams.size == 1) {
val upstream = upstreams.first()
upstream.setLag(0)
upstream.getHead().apply {
if (this is Lifecycle) {
this.start()
}
}
} else {
val heads = upstreams.map { it.getHead() }
val newHead = MergedHead(heads, PriorityForkChoice()).apply {
this.start()
}
val lagObserver = EthereumPostHeadLagObserver(newHead, upstreams as Collection<Upstream>)
this.lagObserver = lagObserver
lagObserver.start()
newHead
}
onHeadUpdated(head)
return head
}
override fun getLabels(): Collection<UpstreamsConfig.Labels> {
return upstreams.flatMap { it.getLabels() }
}
@Suppress("UNCHECKED_CAST")
override fun <T : Upstream> cast(selfType: Class<T>): T {
if (!selfType.isAssignableFrom(this.javaClass)) {
throw ClassCastException("Cannot cast ${this.javaClass} to $selfType")
}
return this as T
}
override fun getRoutedApi(matcher: Selector.Matcher): Mono<Reader<JsonRpcRequest, JsonRpcResponse>> {
return Mono.just(LocalCallRouter(reader, getMethods(), getHead()))
}
override fun getSubscribe(): EthereumSubscribe {
return subscribe
}
override fun getFeeEstimation(): ChainFees {
return feeEstimation
}
}

100
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsUpstream.kt → src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosRpcUpstream.kt
View File

@@ -1,5 +1,6 @@
/**
* Copyright (c) 2021 EmeraldPay, Inc
* Copyright (c) 2020 EmeraldPay, Inc
* Copyright (c) 2019 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,6 +16,8 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
@@ -22,55 +25,65 @@ import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcSwitchClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
class EthereumWsUpstream(
open class EthereumPosRpcUpstream(
id: String,
val chain: Chain,
httpConnection: Reader<JsonRpcRequest, JsonRpcResponse>,
ethereumWsFactory: EthereumWsFactory,
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
node: QuorumForLabels.QuorumItem,
targets: CallMethods
) : EthereumUpstream(id, options, role, targets, node), Upstream, Lifecycle {
companion object {
private val log = LoggerFactory.getLogger(EthereumWsUpstream::class.java)
}
private val head: EthereumWsHead
private val connection: WsConnection
private val api: Reader<JsonRpcRequest, JsonRpcResponse>
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?,
connectorFactory: ConnectorFactory
) : EthereumPosUpstream(id, options, role, targets, node), Lifecycle, Upstream, CachesEnabled {
private val log = LoggerFactory.getLogger(EthereumPosRpcUpstream::class.java)
private val validator: EthereumUpstreamValidator = EthereumUpstreamValidator(this, getOptions())
private val connector: EthereumConnector = connectorFactory.create(this, validator, chain)
private var validatorSubscription: Disposable? = null
private val validator: EthereumUpstreamValidator
init {
validator = EthereumUpstreamValidator(this, getOptions())
connection = ethereumWsFactory.create(this, validator)
head = EthereumWsHead(connection)
// Sometimes the server may close the WebSocket connection during the execution of a call, for example if the response
// is too large for WebSockets Frame (and Geth is unable to split messages into separate frames)
// In this case the failed request must be rerouted to the HTTP connection, because otherwise it would always fail
api = JsonRpcSwitchClient(
JsonRpcWsClient(connection), httpConnection
)
override fun setCaches(caches: Caches) {
if (connector is CachesEnabled) {
connector.setCaches(caches)
}
}
override fun start() {
log.info("Configured for ${chain.chainName}")
connector.start()
if (getOptions().disableValidation != null && getOptions().disableValidation!!) {
log.warn("Disable validation for upstream ${this.getId()}")
this.setLag(0)
this.setStatus(UpstreamAvailability.OK)
} else {
log.debug("Start validation for upstream ${this.getId()}")
validatorSubscription = validator.start()
.subscribe(this::setStatus)
}
}
override fun getHead(): Head {
return head
return connector.getHead()
}
override fun stop() {
validatorSubscription?.dispose()
validatorSubscription = null
connector.stop()
}
override fun isRunning(): Boolean {
return connector.isRunning
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {
return api
return connector.getApi()
}
override fun isGrpc(): Boolean {
@@ -84,31 +97,4 @@ class EthereumWsUpstream(
}
return this as T
}
override fun start() {
connection.connect()
head.start()
if (getOptions().disableValidation != null && getOptions().disableValidation!!) {
log.warn("Disable validation for upstream ${this.getId()}")
this.setLag(0)
this.setStatus(UpstreamAvailability.OK)
} else {
log.debug("Start validation for upstream ${this.getId()}")
val validator = EthereumUpstreamValidator(this, getOptions())
validatorSubscription = validator.start()
.subscribe(this::setStatus)
}
}
override fun stop() {
validatorSubscription?.dispose()
validatorSubscription = null
head.stop()
connection.close()
}
override fun isRunning(): Boolean {
return head.isRunning
}
}

46
src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum_pos/EthereumPosUpstream.kt Normal file
View File

@@ -0,0 +1,46 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
* Copyright (c) 2019 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
abstract class EthereumPosUpstream(
id: String,
options: UpstreamsConfig.Options,
role: UpstreamsConfig.UpstreamRole,
targets: CallMethods?,
private val node: QuorumForLabels.QuorumItem?
) : DefaultUpstream(id, options, role, targets, node) {
private val capabilities = if (options.providesBalance != false) {
setOf(Capability.RPC, Capability.BALANCE)
} else {
setOf(Capability.RPC)
}
override fun getCapabilities(): Set<Capability> {
return capabilities
}
override fun getLabels(): Collection<UpstreamsConfig.Labels> {
return node?.let { listOf(it.labels) } ?: emptyList()
}
}

17
src/main/kotlin/io/emeraldpay/dshackle/upstream/forkchoice/ForkChoice.kt Normal file
View File

@@ -0,0 +1,17 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
interface ForkChoice {
sealed class ChoiceResult {
data class Updated(val nwhead: BlockContainer) : ChoiceResult()
data class Same(val head: BlockContainer?) : ChoiceResult()
}
fun getHead(): BlockContainer?
fun filter(block: BlockContainer): Boolean
fun choose(block: BlockContainer): ChoiceResult
}

31
src/main/kotlin/io/emeraldpay/dshackle/upstream/forkchoice/MostWorkForkChoice.kt Normal file
View File

@@ -0,0 +1,31 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
import java.util.concurrent.atomic.AtomicReference
class MostWorkForkChoice : ForkChoice {
private val head = AtomicReference<BlockContainer>(null)
override fun getHead(): BlockContainer? {
return head.get()
}
override fun filter(block: BlockContainer): Boolean {
val curr = head.get()
return curr == null || curr.difficulty < block.difficulty
}
override fun choose(block: BlockContainer): ForkChoice.ChoiceResult {
val nwhead = head.updateAndGet { curr ->
if (filter(block)) {
block
} else {
curr
}
}
if (nwhead.hash == block.hash) {
return ForkChoice.ChoiceResult.Updated(nwhead)
}
return ForkChoice.ChoiceResult.Same(nwhead)
}
}

36
src/main/kotlin/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoice.kt Normal file
View File

@@ -0,0 +1,36 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.RingSet
import java.util.concurrent.atomic.AtomicReference
class NoChoiceWithPriorityForkChoice(
private val nodeRating: Int
) : ForkChoice {
private val head = AtomicReference<BlockContainer>(null)
private val seenBlocks = RingSet<BlockId>(100)
override fun getHead(): BlockContainer? {
return head.get()
}
override fun filter(block: BlockContainer): Boolean {
return !seenBlocks.contains(block.hash)
}
override fun choose(block: BlockContainer): ForkChoice.ChoiceResult {
val nwhead = head.updateAndGet { curr ->
if (!filter(block)) {
curr
} else {
seenBlocks.add(block.hash)
block.copyWithRating(nodeRating)
}
}
if (nwhead.hash == block.hash) {
return ForkChoice.ChoiceResult.Updated(nwhead)
}
return ForkChoice.ChoiceResult.Same(nwhead)
}
}

35
src/main/kotlin/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoice.kt Normal file
View File

@@ -0,0 +1,35 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.RingSet
import java.util.concurrent.atomic.AtomicReference
class PriorityForkChoice : ForkChoice {
private val head = AtomicReference<BlockContainer>(null)
private val seenBlocks = RingSet<BlockId>(10)
override fun getHead(): BlockContainer? {
return head.get()
}
override fun filter(block: BlockContainer): Boolean {
val curr = head.get()
return (curr == null || curr.nodeRating <= block.nodeRating) && !seenBlocks.contains(block.hash)
}
override fun choose(block: BlockContainer): ForkChoice.ChoiceResult {
val nwhead = head.updateAndGet { curr ->
if (!filter(block)) {
curr
} else {
seenBlocks.add(block.hash)
block
}
}
if (nwhead.hash == block.hash) {
return ForkChoice.ChoiceResult.Updated(nwhead)
}
return ForkChoice.ChoiceResult.Same(nwhead)
}
}

3
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt
View File

@@ -29,6 +29,7 @@ import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream
import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -95,7 +96,7 @@ class BitcoinGrpcUpstream(
}
}
private val upstreamStatus = GrpcUpstreamStatus()
private val grpcHead = GrpcHead(chain, this, remote, blockConverter, reloadBlock)
private val grpcHead = GrpcHead(chain, this, remote, blockConverter, reloadBlock, MostWorkForkChoice())
var timeout = Defaults.timeout
private var capabilities: Set<Capability> = emptySet()

9
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt
View File

@@ -24,13 +24,10 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -97,7 +94,7 @@ open class EthereumGrpcUpstream(
private val log = LoggerFactory.getLogger(EthereumGrpcUpstream::class.java)
private val upstreamStatus = GrpcUpstreamStatus()
private val grpcHead = GrpcHead(chain, this, remote, blockConverter, reloadBlock)
private val grpcHead = GrpcHead(chain, this, remote, blockConverter, reloadBlock, MostWorkForkChoice())
private var capabilities: Set<Capability> = emptySet()
private val defaultReader: Reader<JsonRpcRequest, JsonRpcResponse> = client.forSelector(Selector.empty)

167
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumPosGrpcUpstream.kt Normal file
View File

@@ -0,0 +1,167 @@
/**
* Copyright (c) 2020 EmeraldPay, Inc
* Copyright (c) 2019 ETCDEV GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosUpstream
import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Mono
import java.math.BigInteger
import java.time.Instant
import java.util.Locale
import java.util.concurrent.TimeoutException
import java.util.function.Function
open class EthereumPosGrpcUpstream(
private val parentId: String,
role: UpstreamsConfig.UpstreamRole,
private val chain: Chain,
private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub,
client: JsonRpcGrpcClient,
nodeRating: Int
) : EthereumPosUpstream(
"${parentId}_${chain.chainCode.lowercase(Locale.getDefault())}",
UpstreamsConfig.Options.getDefaults(),
role,
null, null
),
GrpcUpstream,
Lifecycle {
private val blockConverter: Function<BlockchainOuterClass.ChainHead, BlockContainer> = Function { value ->
val block = BlockContainer(
value.height,
BlockId.from(BlockHash.from("0x" + value.blockId)),
BigInteger(1, value.weight.toByteArray()),
Instant.ofEpochMilli(value.timestamp),
false,
null,
null
)
block
}
private val reloadBlock: Function<BlockContainer, Publisher<BlockContainer>> = Function { existingBlock ->
// head comes without transaction data
// need to download transactions for the block
defaultReader.read(JsonRpcRequest("eth_getBlockByHash", listOf(existingBlock.hash.toHexWithPrefix(), false)))
.flatMap(JsonRpcResponse::requireResult)
.map {
BlockContainer.fromEthereumJson(it)
}
.timeout(timeout, Mono.error(TimeoutException("Timeout from upstream")))
.doOnError { t ->
setStatus(UpstreamAvailability.UNAVAILABLE)
val msg = "Failed to download block data for chain $chain on $parentId"
if (t is RpcException || t is TimeoutException) {
log.warn("$msg. Message: ${t.message}")
} else {
log.error(msg, t)
}
}
}
private val log = LoggerFactory.getLogger(EthereumGrpcUpstream::class.java)
private val upstreamStatus = GrpcUpstreamStatus()
private val grpcHead = GrpcHead(chain, this, remote, blockConverter, reloadBlock, NoChoiceWithPriorityForkChoice(nodeRating))
private var capabilities: Set<Capability> = emptySet()
private val defaultReader: Reader<JsonRpcRequest, JsonRpcResponse> = client.forSelector(Selector.empty)
var timeout = Defaults.timeout
override fun start() {
}
override fun isRunning(): Boolean {
return true
}
override fun stop() {
}
override fun update(conf: BlockchainOuterClass.DescribeChain) {
upstreamStatus.update(conf)
capabilities = RemoteCapabilities.extract(conf)
conf.status?.let { status -> onStatus(status) }
}
override fun getQuorumByLabel(): QuorumForLabels {
return upstreamStatus.getNodes()
}
override fun getBlockchainApi(): ReactorBlockchainGrpc.ReactorBlockchainStub {
return remote
}
// ------------------------------------------------------------------------------------------
override fun getLabels(): Collection<UpstreamsConfig.Labels> {
return upstreamStatus.getLabels()
}
override fun getMethods(): CallMethods {
return upstreamStatus.getCallMethods()
}
override fun isAvailable(): Boolean {
return super.isAvailable() && grpcHead.getCurrent() != null && getQuorumByLabel().getAll().any {
it.quorum > 0
}
}
override fun getHead(): Head {
return grpcHead
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {
return defaultReader
}
@Suppress("UNCHECKED_CAST")
override fun <T : Upstream> cast(selfType: Class<T>): T {
if (!selfType.isAssignableFrom(this.javaClass)) {
throw ClassCastException("Cannot cast ${this.javaClass} to $selfType")
}
return this as T
}
override fun getCapabilities(): Set<Capability> {
return capabilities
}
override fun isGrpc(): Boolean {
return true
}
}

11
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt
View File

@@ -23,6 +23,7 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
@@ -45,8 +46,9 @@ class GrpcHead(
/**
* Populate block data with all missing details, of any
*/
private val enhancer: Function<BlockContainer, Publisher<BlockContainer>>?
) : AbstractHead(), Lifecycle {
private val enhancer: Function<BlockContainer, Publisher<BlockContainer>>?,
private val forkChoice: ForkChoice
) : AbstractHead(forkChoice), Lifecycle {
companion object {
private val log = LoggerFactory.getLogger(GrpcHead::class.java)
@@ -94,10 +96,7 @@ class GrpcHead(
var blocks = source.map(converter)
.distinctUntilChanged {
it.hash
}.filter { block ->
val curr = this.getCurrent()
curr == null || curr.difficulty < block.difficulty
}
}.filter { forkChoice.filter(it) }
if (enhancer != null) {
blocks = blocks.flatMap(enhancer)
}

21
src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt
View File

@@ -56,7 +56,8 @@ class GrpcUpstreams(
private val host: String,
private val port: Int,
private val auth: AuthConfig.ClientTlsAuth? = null,
private val fileResolver: FileResolver
private val fileResolver: FileResolver,
private val nodeRating: Int
) {
private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java)
@@ -192,6 +193,8 @@ class GrpcUpstreams(
return getOrCreateEthereum(chain, metrics)
} else if (blockchainType == BlockchainType.BITCOIN) {
return getOrCreateBitcoin(chain, metrics)
} else if (blockchainType == BlockchainType.ETHEREUM_POS) {
return getOrCreateEthereumPos(chain, metrics)
} else {
throw IllegalArgumentException("Unsupported blockchain: $chain")
}
@@ -213,6 +216,22 @@ class GrpcUpstreams(
}
}
fun getOrCreateEthereumPos(chain: Chain, metrics: RpcMetrics): UpstreamChange {
lock.withLock {
val current = known[chain]
return if (current == null) {
val rpcClient = JsonRpcGrpcClient(client!!, chain, metrics)
val created = EthereumPosGrpcUpstream(id, role, chain, client!!, rpcClient, nodeRating)
created.timeout = this.timeout
known[chain] = created
created.start()
UpstreamChange(chain, created, UpstreamChange.ChangeType.ADDED)
} else {
UpstreamChange(chain, current, UpstreamChange.ChangeType.REVALIDATED)
}
}
}
fun getOrCreateBitcoin(chain: Chain, metrics: RpcMetrics): UpstreamChange {
lock.withLock {
val current = known[chain]

2
src/test/groovy/io/emeraldpay/dshackle/cache/HeightByHashAddingSpec.groovy vendored
View File

@@ -27,7 +27,7 @@ class HeightByHashAddingSpec extends Specification {
def block = new BlockContainer(
12079192L, BlockId.from("0xa6af163aab691919c595e2a466f0a7b01f1dff8cfd9631dee811df57064c2d32"),
BigInteger.ONE, Instant.now(), false, "".bytes, null, []
BigInteger.ONE, Instant.now(), false, "".bytes, null, [], 0
)
def "use memory if available"() {

3
src/test/groovy/io/emeraldpay/dshackle/cache/ReceiptMemCacheSpec.groovy vendored
View File

@@ -86,7 +86,8 @@ class ReceiptMemCacheSpec extends Specification {
false,
"{}".bytes,
null,
[TxId.from(receipt.transactionHash)]
[TxId.from(receipt.transactionHash)],
0
)
when:

23
src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy
View File

@@ -17,9 +17,6 @@
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.rpc.RpcClient
import spock.lang.Specification
class UpstreamsConfigReaderSpec extends Specification {
@@ -154,6 +151,26 @@ class UpstreamsConfigReaderSpec extends Specification {
}
}
def "Parse ethereum pos upstreams"() {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-ethereum-pos.yaml")
when:
def act = reader.read(config)
then:
act != null
act.upstreams.size() == 1
with(act.upstreams.get(0)) {
id == "eth2-1"
chain == "ropsten"
connection instanceof UpstreamsConfig.EthereumPosConnection
with((UpstreamsConfig.EthereumPosConnection) connection) {
execution.rpc != null
execution.rpc.url == new URI("http://34.106.60.110:8545")
upstreamRating == 100
}
}
}
def "Parse bitcoin upstreams with esplora"() {
setup:
def config = this.class.getClassLoader().getResourceAsStream("upstreams-bitcoin-esplora.yaml")

12
src/test/groovy/io/emeraldpay/dshackle/rpc/StreamHeadSpec.groovy
View File

@@ -22,10 +22,10 @@ import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.test.MultistreamHolderMock
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.json.BlockJson
@@ -43,7 +43,7 @@ class StreamHeadSpec extends Specification {
def "Errors on unavailable chain"() {
setup:
def upstreams = new MultistreamHolderMock(Chain.ETHEREUM, Stub(EthereumUpstream))
def upstreams = new MultistreamHolderMock(Chain.ETHEREUM, Stub(EthereumRpcUpstream))
def streamHead = new StreamHead(upstreams)
when:
def flux = streamHead.add(
@@ -78,7 +78,7 @@ class StreamHeadSpec extends Specification {
.build()
}
def upstream = new EthereumUpstreamMock(Chain.ETHEREUM, TestingCommons.api())
def upstream = new EthereumRpcUpstreamMock(Chain.ETHEREUM, TestingCommons.api())
def upstreams = new MultistreamHolderMock(Chain.ETHEREUM, upstream)
def streamHead = new StreamHead(upstreams)
when:
@@ -87,7 +87,9 @@ class StreamHeadSpec extends Specification {
)
then:
StepVerifier.create(flux.take(2))
.then { upstream.nextBlock(BlockContainer.from(blocks[0])) }
.then {
upstream.nextBlock(BlockContainer.from(blocks[0]))
}
.expectNext(heads[0])
.then { upstream.nextBlock(BlockContainer.from(blocks[1])) }
.expectNext(heads[1])

4
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinAddressSpec.groovy
View File

@@ -271,7 +271,7 @@ class TrackBitcoinAddressSpec extends Specification {
Head head = Mock(Head) {
1 * getFlux() >> Flux.concat(
Flux.just(
new BlockContainer(0L, BlockId.from(hash1), BigInteger.ZERO, Instant.now(), false, null, null, [])
new BlockContainer(0L, BlockId.from(hash1), BigInteger.ZERO, Instant.now(), false, null, null, [], 0)
),
blocks.asFlux()
)
@@ -312,7 +312,7 @@ class TrackBitcoinAddressSpec extends Specification {
StepVerifier.create(resp)
.expectNext("0")
.then {
blocks.tryEmitNext(new BlockContainer(1L, BlockId.from(hash1), BigInteger.ONE, Instant.now(), false, null, null, []))
blocks.tryEmitNext(new BlockContainer(1L, BlockId.from(hash1), BigInteger.ONE, Instant.now(), false, null, null, [], 0))
}
.expectNext("1230000")
.then {

6
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackBitcoinTxSpec.groovy
View File

@@ -142,7 +142,7 @@ class TrackBitcoinTxSpec extends Specification {
def txid = "69cd44d7c641db82e69824523c7ac0c5c1e5628f025474529cf5ffe64527efc9"
// start with the current block
def next = Flux.fromIterable([10, 12, 13, 14, 15]).map { h ->
new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [])
new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0)
}
Head head = Mock(Head) {
1 * getFlux() >> next
@@ -173,7 +173,7 @@ class TrackBitcoinTxSpec extends Specification {
def txid = "69cd44d7c641db82e69824523c7ac0c5c1e5628f025474529cf5ffe64527efc9"
// start with the current block
def next = Flux.fromIterable([10, 12, 13]).map { h ->
new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [])
new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0)
}
Head head = Mock(Head) {
1 * getFlux() >> next
@@ -268,7 +268,7 @@ class TrackBitcoinTxSpec extends Specification {
])
}
def next = Flux.fromIterable([10, 11, 12]).map { h ->
new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [])
new BlockContainer(h.longValue(), BlockId.from("0000000000000000000895d1b9d3898700e1deecc3b0e69f439aa77875e6042f"), BigInteger.ONE, Instant.now(), false, null, null, [], 0)
}
Head head = Mock(Head) {
_ * getFlux() >> next

8
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackERC20AddressSpec.groovy
View File

@@ -3,18 +3,12 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.config.TokensConfig
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.MultistreamHolderMock
import io.emeraldpay.dshackle.test.ReaderMock
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.ethereum.ERC20Balance
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumSubscribe
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectLogs
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.LogMessage
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.domain.TransactionId
import io.emeraldpay.etherjar.hex.Hex32
@@ -22,11 +16,9 @@ import io.emeraldpay.grpc.Chain
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.erc20.ERC20Token
import io.emeraldpay.etherjar.hex.HexData
import io.emeraldpay.etherjar.rpc.json.TransactionCallJson
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import reactor.test.StepVerifier
import spock.lang.Ignore
import spock.lang.Specification
import java.time.Duration

5
src/test/groovy/io/emeraldpay/dshackle/rpc/TrackEthereumTxSpec.groovy
View File

@@ -198,7 +198,7 @@ class TrackEthereumTxSpec extends Specification {
def tx = new TrackEthereumTx.TxDetails(Chain.ETHEREUM, Instant.now(), TransactionId.from(txId), 6)
def block = new BlockContainer(
100, BlockId.from(txId), BigInteger.ONE, Instant.now(), false, "".bytes, null,
[TxId.from(txId)]
[TxId.from(txId)], 0
)
when:
@@ -220,7 +220,8 @@ class TrackEthereumTxSpec extends Specification {
def tx = new TrackEthereumTx.TxDetails(Chain.ETHEREUM, Instant.now(), TransactionId.from(txId), 6)
def block = new BlockContainer(
100, BlockId.from(txId), BigInteger.ONE, Instant.now(), false, "".bytes, null,
[TxId.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22")]
[TxId.from("0xa0e65cbc1b52a8ca60562112c6060552d882f16f34a9dba2ccdc05c0a6a27c22")],
0
)
apiMock.answer("eth_getTransactionByHash", [txId], null)

29
src/test/groovy/io/emeraldpay/dshackle/test/ConnectorFactoryMock.groovy Normal file
View File

@@ -0,0 +1,29 @@
package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
class ConnectorFactoryMock implements ConnectorFactory {
Reader<JsonRpcRequest, JsonRpcResponse> api
Head head
ConnectorFactoryMock(Reader<JsonRpcRequest, JsonRpcResponse> api, Head head) {
this.api = api
this.head = head
}
boolean isValid() {
return true
}
EthereumConnector create(DefaultUpstream upstream, EthereumUpstreamValidator validator, Chain chain) {
return new EthereumConnectorMock(api, head)
}
}

37
src/test/groovy/io/emeraldpay/dshackle/test/EthereumConnectorMock.groovy Normal file
View File

@@ -0,0 +1,37 @@
package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
class EthereumConnectorMock implements EthereumConnector {
Reader<JsonRpcRequest, JsonRpcResponse> api
Head head
EthereumConnectorMock(Reader<JsonRpcRequest, JsonRpcResponse> api, Head head) {
this.api = api
this.head = head
}
@Override
Reader<JsonRpcRequest, JsonRpcResponse> getApi() {
return this.api
}
@Override
Head getHead() {
return this.head
}
@Override
void start() {}
@Override
void stop() {}
@Override
boolean isRunning() {
return true
}
}

35
src/test/groovy/io/emeraldpay/dshackle/test/EthereumUpstreamMock.groovy → src/test/groovy/io/emeraldpay/dshackle/test/EthereumRpcUpstreamMock.groovy
View File

@@ -19,7 +19,6 @@ package io.emeraldpay.dshackle.test
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.startup.QuorumForLabels
@@ -27,7 +26,6 @@ import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
@@ -36,9 +34,10 @@ import io.emeraldpay.grpc.Chain
import org.jetbrains.annotations.NotNull
import org.reactivestreams.Publisher
class EthereumUpstreamMock extends EthereumRpcUpstream {
EthereumHeadMock ethereumHeadMock = new EthereumHeadMock()
class EthereumRpcUpstreamMock extends EthereumRpcUpstream {
EthereumHeadMock ethereumHeadMock
static CallMethods allMethods() {
new AggregatedCallMethods([
@@ -48,45 +47,37 @@ class EthereumUpstreamMock extends EthereumRpcUpstream {
])
}
EthereumUpstreamMock(@NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api) {
EthereumRpcUpstreamMock(@NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api) {
this(chain, api, allMethods())
}
EthereumUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api) {
EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api) {
this(id, chain, api, allMethods())
}
EthereumUpstreamMock(@NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
EthereumRpcUpstreamMock(@NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
this("test", chain, api, methods)
}
EthereumUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
super(id, chain, api, null,
EthereumRpcUpstreamMock(@NotNull String id, @NotNull Chain chain, @NotNull Reader<JsonRpcRequest, JsonRpcResponse> api, CallMethods methods) {
super(id, chain,
UpstreamsConfig.Options.getDefaults(),
UpstreamsConfig.UpstreamRole.PRIMARY,
methods,
new QuorumForLabels.QuorumItem(1, new UpstreamsConfig.Labels()),
methods)
new ConnectorFactoryMock(api, new EthereumHeadMock()))
this.ethereumHeadMock = this.getHead() as EthereumHeadMock
setLag(0)
setStatus(UpstreamAvailability.OK)
start()
}
void nextBlock(BlockContainer block) {
ethereumHeadMock.nextBlock(block)
this.ethereumHeadMock.nextBlock(block)
}
void setBlocks(Publisher<BlockContainer> blocks) {
ethereumHeadMock.predefined = blocks
}
@Override
Head createHead() {
return ethereumHeadMock
}
@Override
Head getHead() {
return ethereumHeadMock
this.ethereumHeadMock.predefined = blocks
}
@Override

12
src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy
View File

@@ -28,7 +28,7 @@ import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumReader
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.jetbrains.annotations.NotNull
@@ -48,8 +48,8 @@ class MultistreamHolderMock implements MultistreamHolder {
if (BlockchainType.from(chain) == BlockchainType.ETHEREUM) {
if (up instanceof EthereumMultistream) {
upstreams[chain] = up
} else if (up instanceof EthereumUpstream) {
upstreams[chain] = new EthereumMultistreamMock(chain, [up as EthereumUpstream], Caches.default())
} else if (up instanceof EthereumRpcUpstream) {
upstreams[chain] = new EthereumMultistreamMock(chain, [up as EthereumRpcUpstream], Caches.default())
} else {
throw new IllegalArgumentException("Unsupported upstream type ${up.class}")
}
@@ -105,15 +105,15 @@ class MultistreamHolderMock implements MultistreamHolder {
CallMethods customMethods = null
Head customHead = null
EthereumMultistreamMock(@NotNull Chain chain, @NotNull List<EthereumUpstream> upstreams, @NotNull Caches caches) {
EthereumMultistreamMock(@NotNull Chain chain, @NotNull List<EthereumRpcUpstream> upstreams, @NotNull Caches caches) {
super(chain, upstreams, caches)
}
EthereumMultistreamMock(@NotNull Chain chain, @NotNull List<EthereumUpstream> upstreams) {
EthereumMultistreamMock(@NotNull Chain chain, @NotNull List<EthereumRpcUpstream> upstreams) {
this(chain, upstreams, Caches.default())
}
EthereumMultistreamMock(@NotNull Chain chain, @NotNull EthereumUpstream upstream) {
EthereumMultistreamMock(@NotNull Chain chain, @NotNull EthereumRpcUpstream upstream) {
this(chain, [upstream])
}

29
src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy
View File

@@ -28,7 +28,7 @@ import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
@@ -46,35 +46,35 @@ class TestingCommons {
return new ApiReaderMock()
}
static EthereumUpstreamMock upstream() {
return new EthereumUpstreamMock(Chain.ETHEREUM, api())
static EthereumRpcUpstreamMock upstream() {
return new EthereumRpcUpstreamMock(Chain.ETHEREUM, api())
}
static EthereumUpstreamMock upstream(String id) {
return new EthereumUpstreamMock(id, Chain.ETHEREUM, api())
static EthereumRpcUpstreamMock upstream(String id) {
return new EthereumRpcUpstreamMock(id, Chain.ETHEREUM, api())
}
static EthereumUpstreamMock upstream(String id, Reader<JsonRpcRequest, JsonRpcResponse> api) {
return new EthereumUpstreamMock(id, Chain.ETHEREUM, api)
static EthereumRpcUpstreamMock upstream(String id, Reader<JsonRpcRequest, JsonRpcResponse> api) {
return new EthereumRpcUpstreamMock(id, Chain.ETHEREUM, api)
}
static EthereumUpstreamMock upstream(Reader<JsonRpcRequest, JsonRpcResponse> api) {
return new EthereumUpstreamMock(Chain.ETHEREUM, api)
static EthereumRpcUpstreamMock upstream(Reader<JsonRpcRequest, JsonRpcResponse> api) {
return new EthereumRpcUpstreamMock(Chain.ETHEREUM, api)
}
static EthereumUpstreamMock upstream(Reader<JsonRpcRequest, JsonRpcResponse> api, String method) {
static EthereumRpcUpstreamMock upstream(Reader<JsonRpcRequest, JsonRpcResponse> api, String method) {
return upstream(api, [method])
}
static EthereumUpstreamMock upstream(Reader<JsonRpcRequest, JsonRpcResponse> api, List<String> methods) {
return new EthereumUpstreamMock(Chain.ETHEREUM, api, new DirectCallMethods(methods))
static EthereumRpcUpstreamMock upstream(Reader<JsonRpcRequest, JsonRpcResponse> api, List<String> methods) {
return new EthereumRpcUpstreamMock(Chain.ETHEREUM, api, new DirectCallMethods(methods))
}
static Multistream multistream(Reader<JsonRpcRequest, JsonRpcResponse> api) {
return multistream(upstream(api))
}
static Multistream multistream(EthereumUpstream up) {
static Multistream multistream(EthereumRpcUpstream up) {
return new EthereumMultistream(Chain.ETHEREUM, [up], Caches.default()).tap {
start()
}
@@ -111,7 +111,8 @@ class TestingCommons {
false,
null,
null,
[]
[],
0
)
}

26
src/test/groovy/io/emeraldpay/dshackle/upstream/AbstractHeadSpec.groovy
View File

@@ -17,6 +17,9 @@ package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import org.jetbrains.annotations.NotNull
import reactor.core.publisher.Flux
import reactor.core.publisher.Sinks
import reactor.test.StepVerifier
@@ -31,7 +34,7 @@ class AbstractHeadSpec extends Specification {
def blocks = [1L, 2, 3, 4].collect { i ->
byte[] hash = new byte[32]
hash[0] = i as byte
new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [])
new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0)
}
def "Calls beforeBlock on each block"() {
@@ -85,7 +88,7 @@ class AbstractHeadSpec extends Specification {
.verify(Duration.ofSeconds(1))
}
def "Ignores block will less difficulty"() {
def "Ignores block that is filtered by forkchoice"() {
setup:
Sinks.Many<BlockContainer> source = Sinks.many().unicast().onBackpressureBuffer()
def head = new TestHead()
@@ -93,7 +96,7 @@ class AbstractHeadSpec extends Specification {
blocks[1].height, BlockId.from(blocks[1].hash.value.clone().tap { it[1] = 0xff as byte }),
blocks[1].difficulty - 1,
Instant.now(),
false, null, null, []
false, null, null, [], 0
)
when:
head.follow(source.asFlux())
@@ -113,6 +116,23 @@ class AbstractHeadSpec extends Specification {
}
class TestHead extends AbstractHead {
TestHead() {
super(new ForkChoice() {
@Override
boolean filter(@NotNull BlockContainer block) {
return block.hash != BlockId.from("02ff000000000000000000000000000000000000000000000000000000000000")
}
@Override
ForkChoice.ChoiceResult choose(@NotNull BlockContainer block) {
return new ForkChoice.ChoiceResult.Updated(block)
}
@Override
BlockContainer getHead() {
return null
}
})
}
}
}

20
src/test/groovy/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolderSpec.groovy
View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.startup.UpstreamChange
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.grpc.Chain
import spock.lang.Specification
@@ -26,7 +26,7 @@ class CurrentMultistreamHolderSpec extends Specification {
def "add upstream"() {
setup:
def current = new CurrentMultistreamHolder(TestingCommons.emptyCaches())
def up = new EthereumUpstreamMock("test", Chain.ETHEREUM, TestingCommons.api())
def up = new EthereumRpcUpstreamMock("test", Chain.ETHEREUM, TestingCommons.api())
when:
current.update(new UpstreamChange(Chain.ETHEREUM, up, UpstreamChange.ChangeType.ADDED))
then:
@@ -37,9 +37,9 @@ class CurrentMultistreamHolderSpec extends Specification {
def "add multiple upstreams"() {
setup:
def current = new CurrentMultistreamHolder(TestingCommons.emptyCaches())
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
def up2 = new EthereumUpstreamMock("test2", Chain.ETHEREUM_CLASSIC, TestingCommons.api())
def up3 = new EthereumUpstreamMock("test3", Chain.ETHEREUM, TestingCommons.api())
def up1 = new EthereumRpcUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
def up2 = new EthereumRpcUpstreamMock("test2", Chain.ETHEREUM_CLASSIC, TestingCommons.api())
def up3 = new EthereumRpcUpstreamMock("test3", Chain.ETHEREUM, TestingCommons.api())
when:
current.update(new UpstreamChange(Chain.ETHEREUM, up1, UpstreamChange.ChangeType.ADDED))
current.update(new UpstreamChange(Chain.ETHEREUM_CLASSIC, up2, UpstreamChange.ChangeType.ADDED))
@@ -53,10 +53,10 @@ class CurrentMultistreamHolderSpec extends Specification {
def "remove upstream"() {
setup:
def current = new CurrentMultistreamHolder(TestingCommons.emptyCaches())
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
def up2 = new EthereumUpstreamMock("test2", Chain.ETHEREUM_CLASSIC, TestingCommons.api())
def up3 = new EthereumUpstreamMock("test3", Chain.ETHEREUM, TestingCommons.api())
def up1_del = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
def up1 = new EthereumRpcUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
def up2 = new EthereumRpcUpstreamMock("test2", Chain.ETHEREUM_CLASSIC, TestingCommons.api())
def up3 = new EthereumRpcUpstreamMock("test3", Chain.ETHEREUM, TestingCommons.api())
def up1_del = new EthereumRpcUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
when:
current.update(new UpstreamChange(Chain.ETHEREUM, up1, UpstreamChange.ChangeType.ADDED))
current.update(new UpstreamChange(Chain.ETHEREUM_CLASSIC, up2, UpstreamChange.ChangeType.ADDED))
@@ -71,7 +71,7 @@ class CurrentMultistreamHolderSpec extends Specification {
def "available after adding"() {
setup:
def current = new CurrentMultistreamHolder(TestingCommons.emptyCaches())
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
def up1 = new EthereumRpcUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api())
when:
def act = current.isAvailable(Chain.ETHEREUM)

74
src/test/groovy/io/emeraldpay/dshackle/upstream/DistanceExtractorSpec.groovy Normal file
View File

@@ -0,0 +1,74 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.json.BlockJson
import spock.lang.Specification
import java.time.Instant
class DistanceExtractorSpec extends Specification {
def "Correct distance for PoW"() {
expect:
def top = new BlockJson().with {
it.number = topHeight
it.totalDifficulty = topDiff
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123")
it.timestamp = Instant.now()
return it
}
def curr = new BlockJson().with {
it.number = currHeight
it.totalDifficulty = currDiff
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123")
it.timestamp = Instant.now()
return it
}
delta as DistanceExtractor.ChainDistance == DistanceExtractor.@Companion.extractPowDistance(BlockContainer.from(top), BlockContainer.from(curr))
where:
topHeight | topDiff | currHeight | currDiff | delta
100 | 1000 | 100 | 1000 | new DistanceExtractor.ChainDistance.Distance(0)
101 | 1010 | 100 | 1000 | new DistanceExtractor.ChainDistance.Distance(1)
102 | 1020 | 100 | 1000 | new DistanceExtractor.ChainDistance.Distance(2)
103 | 1030 | 100 | 1000 | new DistanceExtractor.ChainDistance.Distance(3)
150 | 1500 | 100 | 1000 | new DistanceExtractor.ChainDistance.Distance(50)
100 | 1000 | 101 | 1010 | new DistanceExtractor.ChainDistance.Distance(0)
100 | 1000 | 102 | 1020 | new DistanceExtractor.ChainDistance.Distance(0)
100 | 1000 | 100 | 1010 | DistanceExtractor.ChainDistance.Fork.INSTANCE
100 | 1100 | 100 | 1000 | DistanceExtractor.ChainDistance.Fork.INSTANCE
}
def "Correct distance for priority"() {
setup:
def hash1 = "0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123"
def hash2 = "0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915124"
expect:
def top = new BlockJson().with {
it.number = topHeight
it.totalDifficulty = 0
it.hash = BlockHash.from(hashA == 0 ? hash1 : hash2)
it.timestamp = Instant.now()
return it
}
def curr = new BlockJson().with {
it.number = currHeight
it.totalDifficulty = 0
it.hash = BlockHash.from(hashB == 0 ? hash1 : hash2)
it.timestamp = Instant.now()
return it
}
delta as DistanceExtractor.ChainDistance == DistanceExtractor.@Companion.extractPriorityDistance(BlockContainer.from(top), BlockContainer.from(curr))
where:
topHeight | hashA | currHeight | hashB || delta
100 | 0 | 100 | 0 || new DistanceExtractor.ChainDistance.Distance(0)
101 | 0 | 100 | 1 || new DistanceExtractor.ChainDistance.Distance(1)
102 | 0 | 100 | 1 || new DistanceExtractor.ChainDistance.Distance(2)
103 | 0 | 100 | 1 || new DistanceExtractor.ChainDistance.Distance(3)
150 | 0 | 100 | 1 || new DistanceExtractor.ChainDistance.Distance(50)
100 | 0 | 101 | 1 || DistanceExtractor.ChainDistance.Fork.INSTANCE
100 | 0 | 102 | 1 || DistanceExtractor.ChainDistance.Fork.INSTANCE
100 | 0 | 100 | 1 || DistanceExtractor.ChainDistance.Fork.INSTANCE
}
}

16
src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy
View File

@@ -22,10 +22,9 @@ import io.emeraldpay.dshackle.test.EthereumApiStub
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.grpc.Chain
import reactor.core.publisher.Flux
import reactor.test.StepVerifier
import spock.lang.Retry
import spock.lang.Specification
@@ -39,22 +38,25 @@ class FilteredApisSpec extends Specification {
def "Verifies labels"() {
setup:
def i = 0
List<EthereumUpstream> upstreams = [
List<EthereumRpcUpstream> upstreams = [
[test: "foo"],
[test: "bar"],
[test: "foo", test2: "baz"],
[test: "foo"],
[test: "baz"]
].collect {
def httpFactory = Mock(HttpFactory) {
create(_, _) >> TestingCommons.api().tap { it.id = "${i++}" }
}
def connectorFactory = new EthereumConnectorFactory(false, null, httpFactory, new MostWorkForkChoice())
new EthereumRpcUpstream(
"test",
Chain.ETHEREUM,
TestingCommons.api().tap { it.id = "${i++}" },
(EthereumWsFactory) null,
new UpstreamsConfig.Options(),
UpstreamsConfig.UpstreamRole.PRIMARY,
ethereumTargets,
new QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(it)),
ethereumTargets
connectorFactory
)
}
def matcher = new Selector.LabelMatcher("test", ["foo"])

37
src/test/groovy/io/emeraldpay/dshackle/upstream/HeadLagObserverSpec.groovy
View File

@@ -111,45 +111,10 @@ class HeadLagObserverSpec extends Specification {
.verifyComplete()
}
def "Correct distance"() {
setup:
Head master = Mock()
HeadLagObserver observer = new TestHeadLagObserver(master, [])
expect:
def top = new BlockJson().with {
it.number = topHeight
it.totalDifficulty = topDiff
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123")
it.timestamp = Instant.now()
return it
}
def curr = new BlockJson().with {
it.number = currHeight
it.totalDifficulty = currDiff
it.hash = BlockHash.from("0x3ec2ebf5d0ec474d0ac6bc50d2770d8409ad76e119968e7919f85d5ec8915123")
it.timestamp = Instant.now()
return it
}
delta as Long == observer.extractDistance(BlockContainer.from(top), BlockContainer.from(curr))
where:
topHeight | topDiff | currHeight | currDiff | delta
100 | 1000 | 100 | 1000 | 0
101 | 1010 | 100 | 1000 | 1
102 | 1020 | 100 | 1000 | 2
103 | 1030 | 100 | 1000 | 3
150 | 1500 | 100 | 1000 | 50
100 | 1000 | 101 | 1010 | 0
100 | 1000 | 102 | 1020 | 0
100 | 1000 | 100 | 1010 | 11
100 | 1100 | 100 | 1000 | 11
}
class TestHeadLagObserver extends HeadLagObserver {
TestHeadLagObserver(@NotNull Head master, @NotNull Collection<? extends Upstream> followers) {
super(master, followers)
super(master, followers, DistanceExtractor.@Companion::extractPowDistance)
}
@Override

11
src/test/groovy/io/emeraldpay/dshackle/upstream/MergedHeadSpec.groovy
View File

@@ -15,6 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import org.springframework.context.Lifecycle
import reactor.core.publisher.Flux
import spock.lang.Specification
@@ -36,7 +37,7 @@ class MergedHeadSpec extends Specification {
}
when:
def merged = new MergedHead([head1, head2, head3])
def merged = new MergedHead([head1, head2, head3], new MostWorkForkChoice())
merged.start()
then:
@@ -44,11 +45,17 @@ class MergedHeadSpec extends Specification {
}
class TestHead1 extends AbstractHead {
TestHead1() {
super(new MostWorkForkChoice())
}
}
class TestHead2 extends AbstractHead implements Lifecycle {
TestHead2() {
super(new MostWorkForkChoice())
}
@Override
void start() {

6
src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy
View File

@@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
@@ -38,8 +38,8 @@ class MultistreamSpec extends Specification {
def "Aggregates methods"() {
setup:
def up1 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"]))
def up2 = new EthereumUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(), new DirectCallMethods(["eth_test2", "eth_test3"]))
def up1 = new EthereumRpcUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"]))
def up2 = new EthereumRpcUpstreamMock("test1", Chain.ETHEREUM, TestingCommons.api(), new DirectCallMethods(["eth_test2", "eth_test3"]))
def aggr = new EthereumMultistream(Chain.ETHEREUM, [up1, up2], Caches.default())
when:
aggr.onUpstreamsUpdated()

3
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHeadSpec.groovy
View File

@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.json.BlockJson
import reactor.core.publisher.Flux
@@ -30,7 +31,7 @@ import java.time.Instant
class DefaultEthereumHeadSpec extends Specification {
DefaultEthereumHead head = new DefaultEthereumHead()
DefaultEthereumHead head = new DefaultEthereumHead(new MostWorkForkChoice())
ObjectMapper objectMapper = Global.objectMapper
def blocks = (10L..20L).collect { i ->

6
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/ERC20BalanceSpec.groovy
View File

@@ -15,7 +15,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock
import io.emeraldpay.dshackle.test.ReaderMock
import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.FilteredApis
@@ -47,7 +47,7 @@ class ERC20BalanceSpec extends Specification {
JsonRpcResponse.ok('"0x0000000000000000000000000000000000000000000000000000001f28d72868"')
)
EthereumUpstream upstream = new EthereumUpstreamMock(Chain.ETHEREUM, api)
EthereumRpcUpstream upstream = new EthereumRpcUpstreamMock(Chain.ETHEREUM, api)
ERC20Token token = new ERC20Token(Address.from("0x54EedeAC495271d0F6B175474E89094C44Da98b9"))
ERC20Balance query = new ERC20Balance()
@@ -73,7 +73,7 @@ class ERC20BalanceSpec extends Specification {
JsonRpcResponse.ok('"0x0000000000000000000000000000000000000000000000000000001f28d72868"')
)
EthereumUpstream upstream = new EthereumUpstreamMock(Chain.ETHEREUM, api)
EthereumRpcUpstream upstream = new EthereumRpcUpstreamMock(Chain.ETHEREUM, api)
ERC20Token token = new ERC20Token(Address.from("0x54EedeAC495271d0F6B175474E89094C44Da98b9"))
ERC20Balance query = new ERC20Balance()

8
src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumReaderSpec.groovy
View File

@@ -23,7 +23,7 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.test.EthereumUpstreamMock
import io.emeraldpay.dshackle.test.EthereumRpcUpstreamMock
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -203,7 +203,7 @@ class EthereumReaderSpec extends Specification {
api.answerOnce("eth_getBalance", ["0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c", "latest"], "0x10")
// height 101 + 1 => 102 => 0x66
api.answerOnce("eth_getBalance", ["0x70b91ff87a902b53dc6e2f6bda8bb9b330ccd30c", "0x66"], "0xff")
EthereumUpstreamMock upstream = new EthereumUpstreamMock(Chain.ETHEREUM, api)
EthereumRpcUpstreamMock upstream = new EthereumRpcUpstreamMock(Chain.ETHEREUM, api)
def upstreams = TestingCommons.multistream(upstream)
def reader = new EthereumReader(upstreams, Caches.default(), calls)
reader.start()
@@ -241,7 +241,7 @@ class EthereumReaderSpec extends Specification {
api.answerOnce("eth_getTransactionReceipt", ["0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2"], [
transactionHash: "0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2"
])
EthereumUpstreamMock upstream = new EthereumUpstreamMock(Chain.ETHEREUM, api)
EthereumRpcUpstreamMock upstream = new EthereumRpcUpstreamMock(Chain.ETHEREUM, api)
def upstreams = TestingCommons.multistream(upstream)
def reader = new EthereumReader(upstreams, Caches.default(), calls)
reader.start()
@@ -257,7 +257,7 @@ class EthereumReaderSpec extends Specification {
def "Read receipt from cache if available"() {
setup:
def api = TestingCommons.api()
EthereumUpstreamMock upstream = new EthereumUpstreamMock(Chain.ETHEREUM, api)
EthereumRpcUpstreamMock upstream = new EthereumRpcUpstreamMock(Chain.ETHEREUM, api)
def upstreams = TestingCommons.multistream(upstream)
def receiptCache = Mock(ReceiptRedisCache) {
1 * it.read(TxId.from("0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2")) >>

37
src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/MostWorkForkChoiceSpec.groovy Normal file
View File

@@ -0,0 +1,37 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import spock.lang.Specification
import java.time.Instant
class MostWorkForkChoiceSpec extends Specification {
def blocks = [1L, 2, 3, 4].collect { i ->
byte[] hash = new byte[32]
hash[0] = i as byte
new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0)
}
def "filters blocks"() {
def choice = new MostWorkForkChoice()
choice.choose(blocks[1])
expect:
!choice.filter(blocks[0])
choice.filter(blocks[2])
}
def "chooses correct block as head"() {
def choice = new MostWorkForkChoice()
choice.choose(blocks[1])
when:
choice.choose(blocks[0])
then:
choice.getHead() == blocks[1]
when:
choice.choose(blocks[2])
then:
choice.getHead() == blocks[2]
}
}

51
src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/NoChoiceWithPriorityForkChoiceSpec.groovy Normal file
View File

@@ -0,0 +1,51 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import spock.lang.Specification
import java.time.Instant
class NoChoiceWithPriorityForkChoiceSpec extends Specification {
def blocks = [1L, 2, 3, 4].collect { i ->
byte[] hash = new byte[32]
hash[0] = i as byte
new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], 0)
}
def "filters blocks"() {
def blockR0 = blocks[0].copyWithRating(10)
def blockR1 = blocks[1].copyWithRating(10)
def choice = new NoChoiceWithPriorityForkChoice(10)
when:
choice.choose(blocks[0])
then:
choice.getHead() == blockR0
when:
choice.choose(blocks[1])
then:
choice.getHead() == blockR1
when:
choice.choose(blocks[0])
then:
choice.getHead() == blocks[1]
}
def "chooses blocks and adds rating"() {
def blockR0 = blocks[0].copyWithRating(10)
def blockR1 = blocks[1].copyWithRating(10)
def choice = new NoChoiceWithPriorityForkChoice(10)
when:
choice.choose(blocks[0])
then:
choice.getHead() == blockR0
when:
choice.choose(blocks[1])
then:
choice.getHead() == blockR1
when:
choice.choose(blocks[0])
then:
choice.getHead() == blockR1
}
}

41
src/test/groovy/io/emeraldpay/dshackle/upstream/forkchoice/PriorityForkChoiceSpec.groovy Normal file
View File

@@ -0,0 +1,41 @@
package io.emeraldpay.dshackle.upstream.forkchoice
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import spock.lang.Specification
import java.time.Instant
class PriorityForkChoiceSpec extends Specification {
def blocks = [1L, 2, 3, 4].collect { i ->
byte[] hash = new byte[32]
hash[0] = i as byte
new BlockContainer(i, BlockId.from(hash), BigInteger.valueOf(i), Instant.now(), false, null, null, [], i.toInteger())
}
def "filters blocks"() {
def choice = new PriorityForkChoice()
choice.choose(blocks[1])
expect:
!choice.filter(blocks[0])
choice.filter(blocks[2])
!choice.filter(blocks[1])
}
def "chooses correct block according to node rating"() {
def choice = new PriorityForkChoice()
choice.choose(blocks[1])
when:
choice.choose(blocks[0])
then:
choice.getHead() == blocks[1]
when:
choice.choose(blocks[2])
then:
choice.getHead() == blocks[2]
when:
def seenblock = blocks[1].copyWithRating(20)
choice.choose(seenblock)
then:
choice.getHead() == blocks[2]
}
}

5
src/test/groovy/io/emeraldpay/dshackle/upstream/grpc/GrpcHeadSpec.groovy
View File

@@ -21,6 +21,7 @@ import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.test.MockGrpcServer
import io.emeraldpay.dshackle.test.TestingCommons
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.grpc.Chain
import io.grpc.stub.StreamObserver
import reactor.test.StepVerifier
@@ -60,7 +61,7 @@ class GrpcHeadSpec extends Specification {
Chain.BITCOIN,
Stub(DefaultUpstream),
client,
convert, null
convert, null, new MostWorkForkChoice()
)
when:
def act = head.getFlux()
@@ -121,7 +122,7 @@ class GrpcHeadSpec extends Specification {
Chain.BITCOIN,
Stub(DefaultUpstream),
client,
convert, null
convert, null, new MostWorkForkChoice()
)
when:
def act = head.getFlux()

9
src/test/resources/upstreams-ethereum-pos.yaml Normal file
View File

@@ -0,0 +1,9 @@
upstreams:
- id: eth2-1
chain: ropsten
connection:
ethereum-pos:
execution:
rpc:
url: "http://34.106.60.110:8545"
upstream-rating: 100