Merge branch 'master' into network_specific_methods_support

# Conflicts:
#	src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt
This commit is contained in:
Maksim Fomenkov
2022-11-28 18:38:48 +04:00
152 changed files with 775 additions and 1214 deletions

View File

@@ -0,0 +1,36 @@
package io.emeraldpay.dshackle
enum class BlockchainType {
BITCOIN, EVM_POW, EVM_POS;
companion object {
@JvmStatic
fun from(chain: Chain): BlockchainType {
if (chain == Chain.TESTNET_ROPSTEN ||
chain == Chain.POLYGON ||
chain == Chain.OPTIMISM ||
chain == Chain.BSC ||
chain == Chain.TESTNET_GOERLI ||
chain == Chain.ETHEREUM
) {
return EVM_POS
}
if (chain == Chain.ETHEREUM_CLASSIC ||
chain == Chain.ARBITRUM ||
chain == Chain.FANTOM ||
chain == Chain.RSK ||
chain == Chain.TESTNET_KOVAN ||
chain == Chain.TESTNET_MORDEN ||
chain == Chain.TESTNET_RINKEBY
) {
return EVM_POW
}
if (chain == Chain.BITCOIN ||
chain == Chain.TESTNET_BITCOIN
) {
return BITCOIN
}
throw IllegalArgumentException("Unknown type of blockchain: $chain")
}
}
}

View File

@@ -0,0 +1,64 @@
/*
* Copyright (c) 2016-2019 ETCDEV GmbH, All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.emeraldpay.dshackle
enum class Chain(val id: Int, val chainCode: String, val chainName: String) {
UNSPECIFIED(0, "UNSPECIFIED", "Unknown"),
BITCOIN(1, "BTC", "Bitcoin"), // GRIN(2, "GRIN", "Grin"),
// Networks with tokens
ETHEREUM(100, "ETH", "Ethereum"),
ETHEREUM_CLASSIC(101, "ETC", "Ethereum Classic"),
FANTOM(
102,
"FTM",
"Fantom"
), // LIGHTNING(1001, "BTC_LN", "Bitcoin Lightning"),
POLYGON(1002, "POLYGON", "Polygon Matic"),
RSK(1003, "RSK", "Bitcoin RSK"),
ARBITRUM(
1004,
"ARBITRUM",
"Arbitrum"
),
OPTIMISM(1005, "OPTIMISM", "Optimism"),
BSC(1006, "BSC", "Binance Smart Chain"), // Testnets
TESTNET_MORDEN(10001, "MORDEN", "Morden Testnet"),
TESTNET_KOVAN(10002, "KOVAN", "Kovan Testnet"),
TESTNET_BITCOIN(
10003,
"TESTNET_BITCOIN",
"Bitcoin Testnet"
), // TESTNET_FLOONET(10004, "FLOONET", "Floonet Testnet"),
TESTNET_GOERLI(10005, "GOERLI", "Goerli Testnet"),
TESTNET_ROPSTEN(
10006,
"ROPSTEN",
"Ropsten Testnet"
),
TESTNET_RINKEBY(10007, "RINKEBY", "Rinkeby Testnet");
companion object {
fun byId(id: Int): Chain {
for (chain in values()) {
if (chain.id == id) {
return chain
}
}
return UNSPECIFIED
}
}
}

View File

@@ -16,7 +16,6 @@
package io.emeraldpay.dshackle
import io.emeraldpay.api.proto.Common
import io.emeraldpay.grpc.Chain
import java.util.EnumMap
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock

View File

@@ -33,9 +33,11 @@ import org.springframework.boot.SpringApplication
import org.springframework.context.ApplicationContext
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.Profile
import org.springframework.core.env.Environment
import org.springframework.scheduling.annotation.EnableAsync
import org.springframework.scheduling.annotation.EnableScheduling
import org.springframework.util.ResourceUtils
import reactor.core.scheduler.Scheduler
import reactor.core.scheduler.Schedulers
import java.io.File
@@ -47,8 +49,8 @@ import kotlin.system.exitProcess
@EnableScheduling
@EnableAsync
open class Config(
@Autowired private val env: Environment,
@Autowired private val ctx: ApplicationContext
private val env: Environment,
private val ctx: ApplicationContext
) {
companion object {
@@ -61,7 +63,12 @@ open class Config(
private var configFilePath: File? = null
init {
configFilePath = getConfigPath()
configFilePath = if (env.activeProfiles.contains("integration-test")) {
ResourceUtils.getFile("classpath:integration/dshackle.yaml")
} else {
getConfigPath()
}
Global.version = env.getProperty("version.app", Global.version).let {
if (it.contains("SNAPSHOT")) {
listOfNotNull(it, env.getProperty("version.commit")).joinToString("-")
@@ -95,6 +102,7 @@ open class Config(
}
@Bean
@Profile("!integration-test")
open fun mainConfig(@Autowired fileResolver: FileResolver): MainConfig {
val f = configFilePath ?: throw IllegalStateException("Config path is not set")
log.info("Using config: ${f.absolutePath}")

View File

@@ -27,7 +27,6 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent
import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import java.text.SimpleDateFormat
import java.util.Locale
import java.util.TimeZone

View File

@@ -16,7 +16,6 @@
package io.emeraldpay.dshackle
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.grpc.Chain
/**
* Exception that should be handled/logged without a stacktrace in production

View File

@@ -20,12 +20,10 @@ import org.slf4j.LoggerFactory
import org.springframework.boot.ResourceBanner
import org.springframework.boot.SpringApplication
import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.context.annotation.Import
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.support.ResourcePropertySource
@SpringBootApplication(scanBasePackages = ["io.emeraldpay.dshackle"])
@Import(Config::class)
open class Starter
private val log = LoggerFactory.getLogger(Starter::class.java)

View File

@@ -16,12 +16,12 @@
package io.emeraldpay.dshackle.cache
import com.google.protobuf.ByteString
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono

View File

@@ -15,8 +15,8 @@
*/
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.CacheConfig
import io.emeraldpay.grpc.Chain
import io.lettuce.core.RedisClient
import io.lettuce.core.RedisConnectionException
import io.lettuce.core.RedisURI
@@ -42,7 +42,7 @@ open class CachesFactory(
}
private var redis: StatefulRedisConnection<String, ByteArray>? = null
private val all = EnumMap<Chain, Caches>(io.emeraldpay.grpc.Chain::class.java)
private val all = EnumMap<Chain, Caches>(Chain::class.java)
@PostConstruct
fun init() {

View File

@@ -15,10 +15,10 @@
*/
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono

View File

@@ -16,12 +16,12 @@
package io.emeraldpay.dshackle.cache
import com.google.protobuf.ByteString
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.dshackle.proto.CachesProto.ValueContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono

View File

@@ -16,12 +16,12 @@
package io.emeraldpay.dshackle.cache
import com.google.protobuf.ByteString
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono

View File

@@ -15,10 +15,10 @@
*/
package io.emeraldpay.dshackle.cache
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.DefaultContainer
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import reactor.core.publisher.Mono

View File

@@ -16,13 +16,13 @@
package io.emeraldpay.dshackle.cache
import com.google.protobuf.ByteString
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.data.TxContainer
import io.emeraldpay.dshackle.data.TxId
import io.emeraldpay.dshackle.proto.CachesProto
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.grpc.Chain
import io.lettuce.core.api.reactive.RedisReactiveCommands
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono

View File

@@ -15,7 +15,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.Chain
class HealthConfig {

View File

@@ -15,8 +15,8 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.CollectionNode
import org.yaml.snakeyaml.nodes.MappingNode

View File

@@ -16,7 +16,7 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.Chain
/**
* Configure HTTP Proxy to Upstreams

View File

@@ -16,8 +16,8 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.grpc.Chain
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.yaml.snakeyaml.nodes.MappingNode

View File

@@ -15,9 +15,9 @@
*/
package io.emeraldpay.dshackle.config
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
class TokensConfig(
val tokens: List<Token>

View File

@@ -0,0 +1,78 @@
package io.emeraldpay.dshackle.config.context
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.CachesFactory
import io.emeraldpay.dshackle.upstream.CallTargetsHolder
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
@Configuration
open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) {
@Bean
open fun allMultistreams(
cachesFactory: CachesFactory,
callTargetsHolder: CallTargetsHolder
): List<Multistream> {
return Chain.values()
.filterNot { it == Chain.UNSPECIFIED }
.mapNotNull { chain ->
when (BlockchainType.from(chain)) {
BlockchainType.EVM_POS -> ethereumPosMultistream(chain, cachesFactory)
BlockchainType.EVM_POW -> ethereumMultistream(chain, cachesFactory)
BlockchainType.BITCOIN -> bitcoinMultistream(chain, cachesFactory)
else -> null
}
}
}
private fun ethereumMultistream(
chain: Chain,
cachesFactory: CachesFactory
): EthereumMultistream {
val name = "multi-ethereum-$chain"
return EthereumMultistream(
chain,
ArrayList(),
cachesFactory.getCaches(chain)
).also { register(it, name) }
}
open fun ethereumPosMultistream(
chain: Chain,
cachesFactory: CachesFactory
): EthereumPosMultiStream {
val name = "multi-ethereum-pos-$chain"
return EthereumPosMultiStream(
chain,
ArrayList(),
cachesFactory.getCaches(chain)
).also { register(it, name) }
}
open fun bitcoinMultistream(
chain: Chain,
cachesFactory: CachesFactory
): BitcoinMultistream {
val name = "multi-bitcoin-$chain"
return BitcoinMultistream(
chain,
ArrayList(),
cachesFactory.getCaches(chain)
).also { register(it, name) }
}
private fun register(bean: Any, name: String) {
beanFactory.initializeBean(bean, name)
beanFactory.autowireBean(bean)
this.beanFactory.registerSingleton(name, bean)
}
}

View File

@@ -1,9 +1,9 @@
package io.emeraldpay.dshackle.monitoring.accesslog
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.MainConfig
import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.monitoring.accesslog
import com.fasterxml.jackson.annotation.JsonInclude
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.Chain
import org.slf4j.LoggerFactory
import java.time.Instant
import java.util.UUID

View File

@@ -17,8 +17,8 @@ package io.emeraldpay.dshackle.monitoring.accesslog
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.AccessLogConfig
import io.emeraldpay.grpc.Chain
import io.grpc.Attributes
import io.grpc.Grpc
import io.grpc.Metadata

View File

@@ -17,9 +17,9 @@ package io.emeraldpay.dshackle.proxy
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux

View File

@@ -15,13 +15,13 @@
*/
package io.emeraldpay.dshackle.proxy
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ProxyConfig
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled
import org.reactivestreams.Publisher

View File

@@ -16,13 +16,13 @@
*/
package io.emeraldpay.dshackle.proxy
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.TlsSetup
import io.emeraldpay.dshackle.config.ProxyConfig
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.rpc.NativeSubscribe
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Timer

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.proxy
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.BlockchainOuterClass.Selector
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.ProxyConfig
import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp
@@ -25,7 +26,6 @@ import io.emeraldpay.dshackle.rpc.NativeCall
import io.emeraldpay.dshackle.rpc.NativeSubscribe
import io.emeraldpay.etherjar.rpc.json.RequestJson
import io.emeraldpay.etherjar.rpc.json.ResponseJson
import io.emeraldpay.grpc.Chain
import io.netty.buffer.ByteBufInputStream
import io.netty.buffer.Unpooled
import org.reactivestreams.Publisher

View File

@@ -19,9 +19,9 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.ChainValue
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Timer

View File

@@ -44,6 +44,7 @@ class Describe(
.setChain(Common.ChainRef.forNumber(chain.id))
.addAllSupportedMethods(targets)
.setStatus(status)
.setCurrentHeight(chainUpstreams.getHead().getCurrentHeight() ?: 0)
chainUpstreams.getAll().let { ups ->
ups.forEach { up ->
val nodes = QuorumForLabels()

View File

@@ -1,9 +1,9 @@
package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.grpc.Chain
import io.grpc.Status
import io.grpc.StatusException
import org.slf4j.LoggerFactory

View File

@@ -19,6 +19,8 @@ package io.emeraldpay.dshackle.rpc
import com.fasterxml.jackson.databind.ObjectMapper
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.quorum.CallQuorum
@@ -26,11 +28,11 @@ import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
import io.emeraldpay.dshackle.quorum.QuorumReaderFactory
import io.emeraldpay.dshackle.quorum.QuorumRpcReader
import io.emeraldpay.dshackle.startup.ConfiguredUpstreams
import io.emeraldpay.dshackle.upstream.ApiSource
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError
@@ -40,12 +42,10 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.RpcResponseError
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Metrics
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.event.EventListener
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
@@ -55,9 +55,9 @@ import java.util.concurrent.atomic.AtomicInteger
@Service
open class NativeCall(
@Autowired private val multistreamHolder: MultistreamHolder,
@Autowired private val configuredUpstreams: ConfiguredUpstreams,
@Autowired private val signer: ResponseSigner
private val multistreamHolder: MultistreamHolder,
private val configuredUpstreams: ConfiguredUpstreams,
private val signer: ResponseSigner
) {
private val log = LoggerFactory.getLogger(NativeCall::class.java)
@@ -66,18 +66,19 @@ open class NativeCall(
var quorumReaderFactory: QuorumReaderFactory = QuorumReaderFactory.default()
private val ethereumCallSelectors = EnumMap<Chain, EthereumCallSelector>(Chain::class.java)
init {
val casting = mapOf(
companion object {
val casting: Map<BlockchainType, Class<out EthereumLikeMultistream>> = mapOf(
BlockchainType.EVM_POS to EthereumPosMultiStream::class.java,
BlockchainType.EVM_POW to EthereumMultistream::class.java,
BlockchainType.EVM_POW to EthereumMultistream::class.java
)
}
multistreamHolder.observeChains().subscribe { chain ->
casting[BlockchainType.from(chain)]?.let { cast ->
multistreamHolder.getUpstream(chain)?.let { up ->
val reader = up.cast(cast).getReader()
ethereumCallSelectors.putIfAbsent(chain, EthereumCallSelector(reader.heightByHash()))
}
@EventListener
fun onUpstreamChangeEvent(event: UpstreamChangeEvent) {
casting[BlockchainType.from(event.chain)]?.let { cast ->
multistreamHolder.getUpstream(event.chain)?.let { up ->
val reader = up.cast(cast).getReader()
ethereumCallSelectors.putIfAbsent(event.chain, EthereumCallSelector(reader.heightByHash()))
}
}
}
@@ -166,7 +167,8 @@ open class NativeCall(
}
val matcher = Selector.convertToMatcher(request.selector)
if (!configuredUpstreams.hasMatchingUpstream(chain, matcher)) {
if (!multistreamHolder.getUpstream(chain).hasMatchingUpstream(matcher)) {
if (Global.metricsExtended) {
Metrics.globalRegistry
.counter("no_matching_upstream", "chain", chain.chainCode, "matcher", matcher.describeInternal())
@@ -176,7 +178,6 @@ open class NativeCall(
}
val upstream = multistreamHolder.getUpstream(chain)
?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain)))
return prepareCall(request, upstream)
}
@@ -255,17 +256,13 @@ open class NativeCall(
}
private fun getRequestDecorator(method: String): RequestDecorator =
if (method in listOf(
"eth_getFilterChanges",
"eth_uninstallFilter",
)
)
GetFilterUpdatesDecorator()
if (method in DefaultEthereumMethods.withFilterIdMethods)
WithFilterIdDecorator()
else
NoneRequestDecorator()
private fun getResultDecorator(method: String): ResultDecorator =
if (CreateFilterDecorator.createFilterMethods.contains(method)) CreateFilterDecorator() else NoneResultDecorator()
if (method in DefaultEthereumMethods.newFilterMethods) CreateFilterDecorator() else NoneResultDecorator()
fun fetch(ctx: ValidCallContext<ParsedCallDetails>): Mono<CallResult> {
return ctx.upstream.getRoutedApi(ctx.matcher)
@@ -385,11 +382,6 @@ open class NativeCall(
companion object {
const val quoteCode = '"'.code.toByte()
val createFilterMethods = listOf(
"eth_newFilter",
"eth_newBlockFilter",
"eth_newPendingTransactionFilter",
)
}
override fun processResult(result: QuorumRpcReader.Result): ByteArray {
val bytes = result.value
@@ -410,7 +402,7 @@ open class NativeCall(
override fun processRequest(request: List<Any>): List<Any> = request
}
open class GetFilterUpdatesDecorator : RequestDecorator {
open class WithFilterIdDecorator : RequestDecorator {
override fun processRequest(request: List<Any>): List<Any> {
val filterId = request.first().toString()
val sanitized = filterId.substring(0, filterId.lastIndex - 1)

View File

@@ -18,6 +18,8 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.BlockchainOuterClass.NativeSubscribeReplyItem
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
@@ -25,8 +27,6 @@ import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.ethereum.EthereumLikeMultistream
import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.HasUpstream
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.grpc.Status
import io.grpc.StatusException
import org.reactivestreams.Publisher

View File

@@ -19,9 +19,9 @@ package io.emeraldpay.dshackle.rpc
import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

View File

@@ -18,10 +18,10 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.grpc.Chain
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux

View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.Chain
import reactor.core.publisher.Flux
/**

View File

@@ -18,27 +18,28 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.apache.commons.lang3.StringUtils
import org.bitcoinj.params.MainNetParams
import org.bitcoinj.params.TestNet3Params
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.event.EventListener
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.math.BigInteger
import java.util.concurrent.ConcurrentHashMap
import javax.annotation.PostConstruct
@Service
class TrackBitcoinAddress(
@@ -67,15 +68,13 @@ class TrackBitcoinAddress(
Selector.CapabilityMatcher(Capability.BALANCE)
)
@PostConstruct
fun listenChains() {
multistreamHolder.observeChains().subscribe { chain ->
multistreamHolder.getUpstream(chain)?.let { mup ->
val available = mup.getAll().any { up ->
!up.isGrpc() && up.getCapabilities().contains(Capability.BALANCE)
}
setBalanceAvailability(chain, available)
@EventListener
fun onUpstreamChangeEvent(event: UpstreamChangeEvent) {
multistreamHolder.getUpstream(event.chain)?.let { mup ->
val available = mup.getAll().any { up ->
!up.isGrpc() && up.getCapabilities().contains(Capability.BALANCE)
}
setBalanceAvailability(event.chain, available)
}
}

View File

@@ -17,12 +17,12 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

View File

@@ -17,6 +17,8 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.config.TokensConfig
import io.emeraldpay.dshackle.upstream.MultistreamHolder
@@ -26,8 +28,6 @@ import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.domain.EventId
import io.emeraldpay.etherjar.erc20.ERC20Token
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

View File

@@ -18,14 +18,14 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.upstream.MultistreamHolder
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
import io.emeraldpay.etherjar.domain.Address
import io.emeraldpay.etherjar.domain.Wei
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

View File

@@ -18,6 +18,8 @@ package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.SilentException
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.TxId
@@ -29,8 +31,6 @@ import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.json.BlockJson
import io.emeraldpay.etherjar.rpc.json.TransactionJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

View File

@@ -16,7 +16,7 @@
package io.emeraldpay.dshackle.rpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.Chain
import reactor.core.publisher.Flux
interface TrackTx {

View File

@@ -17,17 +17,13 @@
package io.emeraldpay.dshackle.startup
import com.google.common.annotations.VisibleForTesting
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.CurrentMultistreamHolder
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.HttpRpcFactory
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcHead
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinZMQHead
@@ -47,31 +43,30 @@ import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstreams
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import org.springframework.boot.ApplicationArguments
import org.springframework.boot.ApplicationRunner
import org.springframework.context.ApplicationEventPublisher
import org.springframework.stereotype.Component
import java.net.URI
import java.util.concurrent.atomic.AtomicInteger
import java.util.function.Function
import javax.annotation.PostConstruct
import kotlin.math.abs
@Repository
@Component
open class ConfiguredUpstreams(
@Autowired private val currentUpstreams: CurrentMultistreamHolder,
@Autowired private val fileResolver: FileResolver,
@Autowired private val config: UpstreamsConfig
) {
private val fileResolver: FileResolver,
private val config: UpstreamsConfig,
private val callTargets: CallTargetsHolder,
private val eventPublisher: ApplicationEventPublisher
) : ApplicationRunner {
private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java)
private var seq = AtomicInteger(0)
private val hashes: MutableMap<Byte, Boolean> = HashMap()
@PostConstruct
fun start() {
override fun run(args: ApplicationArguments) {
log.debug("Starting upstreams")
val defaultOptions = buildDefaultOptions(config)
config.upstreams.forEach { up ->
@@ -115,17 +110,13 @@ open class ConfiguredUpstreams(
}
}
upstream?.let {
currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED))
val event = UpstreamChangeEvent(chain, upstream, UpstreamChangeEvent.ChangeType.ADDED)
eventPublisher.publishEvent(event)
}
}
}
}
fun hasMatchingUpstream(chain: Chain, matcher: Selector.LabelSelectorMatcher): Boolean =
config.upstreams.any { up ->
(up.chain?.let { Global.chainById(it) == chain } ?: true) && matcher.matches(up.labels)
}
private fun buildDefaultOptions(config: UpstreamsConfig): HashMap<Chain, UpstreamsConfig.Options> {
val defaultOptions = HashMap<Chain, UpstreamsConfig.Options>()
config.defaultOptions.forEach { defaultsConfig ->
@@ -150,7 +141,7 @@ open class ConfiguredUpstreams(
fun buildMethods(config: UpstreamsConfig.Upstream<*>, chain: Chain): CallMethods {
return if (config.methods != null) {
ManagedCallMethods(
currentUpstreams.getDefaultMethods(chain),
callTargets.getDefaultMethods(chain),
config.methods!!.enabled.map { it.name }.toSet(),
config.methods!!.disabled.map { it.name }.toSet()
).also {
@@ -164,7 +155,7 @@ open class ConfiguredUpstreams(
}
}
} else {
currentUpstreams.getDefaultMethods(chain)
callTargets.getDefaultMethods(chain)
}
}
@@ -315,7 +306,7 @@ open class ConfiguredUpstreams(
.doOnNext {
log.info("Chain ${it.chain} ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}. With caps: ${it.upstream.getCapabilities()}")
}
.subscribe(currentUpstreams::update)
.subscribe(eventPublisher::publishEvent)
}
private fun buildHttpFactory(conn: UpstreamsConfig.RpcConnection, urls: ArrayList<URI>? = null): HttpRpcFactory? {

View File

@@ -16,15 +16,15 @@
*/
package io.emeraldpay.dshackle.startup
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.grpc.Chain
/**
* An update event to the list of currently available upstreams.
*/
class UpstreamChange(
class UpstreamChangeEvent(
/**
* Target blockchain
*/

View File

@@ -0,0 +1,29 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import org.springframework.stereotype.Component
import java.util.HashMap
@Component
class CallTargetsHolder {
private val callTargets = HashMap<Chain, CallMethods>()
fun getDefaultMethods(chain: Chain): CallMethods {
return callTargets[chain] ?: return setupDefaultMethods(chain)
}
private fun setupDefaultMethods(chain: Chain): CallMethods {
val created = when (BlockchainType.from(chain)) {
BlockchainType.EVM_POW -> DefaultEthereumMethods(chain)
BlockchainType.BITCOIN -> DefaultBitcoinMethods()
BlockchainType.EVM_POS -> DefaultEthereumMethods(chain)
else -> throw IllegalStateException("Unsupported chain: $chain")
}
callTargets[chain] = created
return created
}
}

View File

@@ -16,156 +16,40 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.cache.CachesFactory
import io.emeraldpay.dshackle.startup.UpstreamChange
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.emeraldpay.dshackle.Chain
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Repository
import reactor.core.publisher.Flux
import reactor.core.publisher.Sinks
import java.util.Collections
import java.util.concurrent.Callable
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.locks.ReentrantLock
import org.springframework.stereotype.Component
import javax.annotation.PreDestroy
import kotlin.concurrent.withLock
@Repository
@Component
open class CurrentMultistreamHolder(
@Autowired private val cachesFactory: CachesFactory
multistreams: List<Multistream>
) : MultistreamHolder {
private val log = LoggerFactory.getLogger(CurrentMultistreamHolder::class.java)
private val chainMapping = ConcurrentHashMap<Chain, Multistream>()
private val chainsBus = Sinks.many()
.multicast()
.directBestEffort<Chain>()
private val callTargets = HashMap<Chain, CallMethods>()
private val updateLock = ReentrantLock()
private val chainMapping = multistreams.associateBy { it.chain }
fun update(change: UpstreamChange) {
updateLock.withLock {
log.debug("Upstream update: ${change.type} ${change.chain} via ${change.upstream.getId()}")
val chain = change.chain
try {
when (BlockchainType.from(chain)) {
BlockchainType.EVM_POW -> {
val up = change.upstream.cast(EthereumUpstream::class.java)
val current = chainMapping[chain]
val factory = Callable<Multistream> {
EthereumMultistream(chain, ArrayList(), cachesFactory.getCaches(chain))
}
processUpdate(change, up, current, factory)
}
BlockchainType.EVM_POS -> {
val up = change.upstream.cast(EthereumPosUpstream::class.java)
val current = chainMapping[chain]
val factory = Callable<Multistream> {
EthereumPosMultiStream(chain, ArrayList(), cachesFactory.getCaches(chain))
}
processUpdate(change, up, current, factory)
}
BlockchainType.BITCOIN -> {
val up = change.upstream.cast(BitcoinUpstream::class.java)
val current = chainMapping[chain]
val factory = Callable<Multistream> {
BitcoinMultistream(chain, ArrayList(), cachesFactory.getCaches(chain))
}
processUpdate(change, up, current, factory)
}
else -> {
log.error("Update for unsupported chain: $chain")
}
}
} catch (e: Throwable) {
log.error("Failed to update upstream", e)
}
}
}
fun processUpdate(change: UpstreamChange, up: Upstream, current: Multistream?, factory: Callable<Multistream>) {
val chain = change.chain
if (change.type == UpstreamChange.ChangeType.REMOVED) {
current?.removeUpstream(up.getId())
log.info("Upstream ${change.upstream.getId()} with chain $chain has been removed")
} else {
if (current == null) {
val created = factory.call()
if (up is CachesEnabled) {
up.setCaches(created.caches)
}
created.addUpstream(up)
created.start()
chainMapping[chain] = created
chainsBus.tryEmitNext(chain)
} else {
if (up is CachesEnabled) {
up.setCaches(current.caches)
}
current.addUpstream(up)
}
if (!callTargets.containsKey(chain)) {
setupDefaultMethods(chain)
}
log.info("Upstream ${change.upstream.getId()} with chain $chain has been added")
}
}
override fun getUpstream(chain: Chain): Multistream? {
return chainMapping[chain]
override fun getUpstream(chain: Chain): Multistream {
return chainMapping.getValue(chain)
}
override fun getAvailable(): List<Chain> {
return Collections.unmodifiableList(chainMapping.keys.toList())
}
override fun observeChains(): Flux<Chain> {
return Flux.concat(
Flux.fromIterable(getAvailable()),
chainsBus.asFlux()
)
}
override fun getDefaultMethods(chain: Chain): CallMethods {
return callTargets[chain] ?: return setupDefaultMethods(chain)
}
fun setupDefaultMethods(chain: Chain): CallMethods {
val created = when (BlockchainType.from(chain)) {
BlockchainType.EVM_POW -> DefaultEthereumMethods(chain)
BlockchainType.BITCOIN -> DefaultBitcoinMethods()
BlockchainType.EVM_POS -> DefaultEthereumMethods(chain)
else -> throw IllegalStateException("Unsupported chain: $chain")
}
callTargets[chain] = created
return created
return chainMapping.values.asSequence()
.filter { it.haveUpstreams() }
.map { it.chain }
.toList()
}
override fun isAvailable(chain: Chain): Boolean {
return chainMapping.containsKey(chain) && callTargets.containsKey(chain)
return chainMapping.getValue(chain).isAvailable()
}
@PreDestroy
fun shutdown() {
log.info("Closing upstream connections...")
updateLock.withLock {
chainMapping.values.forEach {
it.stop()
}
chainMapping.clear()
chainMapping.values.forEach {
it.stop()
}
}
}

View File

@@ -16,9 +16,9 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.DistributionSummary
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag

View File

@@ -1,9 +1,9 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
interface HttpFactory {
fun create(id: String?, chain: Chain): Reader<JsonRpcRequest, JsonRpcResponse>

View File

@@ -1,12 +1,12 @@
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag

View File

@@ -0,0 +1,7 @@
package io.emeraldpay.dshackle.upstream
interface Lifecycle {
fun start()
fun stop()
fun isRunning(): Boolean
}

View File

@@ -21,7 +21,6 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
@@ -43,7 +42,7 @@ class MergedHead(
override fun start() {
super.start()
sources.forEach { head ->
if (head is Lifecycle && !head.isRunning) {
if (head is Lifecycle && !head.isRunning()) {
head.start()
}
}
@@ -56,7 +55,7 @@ class MergedHead(
override fun stop() {
super.stop()
sources.forEach { head ->
if (head is Lifecycle && head.isRunning) {
if (head is Lifecycle && head.isRunning()) {
head.stop()
}
}

View File

@@ -16,26 +16,30 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag
import org.apache.commons.collections4.Factory
import org.apache.commons.collections4.FunctorException
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import org.springframework.context.event.EventListener
import org.springframework.core.Ordered
import org.springframework.core.annotation.Order
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.time.Duration
import java.time.Instant
import java.util.Locale
import java.util.*
import java.util.concurrent.atomic.AtomicReference
import java.util.concurrent.locks.ReentrantLock
import java.util.function.Predicate
@@ -56,6 +60,8 @@ abstract class Multistream(
private const val metrics = "upstreams"
}
private var started = false
private var cacheSubscription: Disposable? = null
private val reconfigLock = ReentrantLock()
private var callMethods: CallMethods? = null
@@ -231,6 +237,7 @@ abstract class Multistream(
// print status _change_ every 15 seconds, at most; otherwise prints it on interval of 30 seconds
.sample(Duration.ofSeconds(15))
.subscribe { printStatus() }
started = true
}
override fun stop() {
@@ -244,6 +251,7 @@ abstract class Multistream(
}
}
lagObserver?.stop()
started = false
}
fun onHeadUpdated(head: Head) {
@@ -316,6 +324,38 @@ abstract class Multistream(
log.info("State of ${chain.chainCode}: height=${height ?: '?'}, status=[$statuses], lag=[$lag], weak=[$weak]")
}
fun test(event: UpstreamChangeEvent): Boolean {
return event.chain == this.chain
}
@EventListener
@Order(Ordered.HIGHEST_PRECEDENCE)
fun onUpstreamChange(event: UpstreamChangeEvent) {
val chain = event.chain
if (this.chain == chain) {
if (event.type == UpstreamChangeEvent.ChangeType.REMOVED) {
removeUpstream(event.upstream.getId())
log.error("Upstream ${event.upstream.getId()} with chain $chain has been removed")
} else {
if (event.upstream is CachesEnabled) {
event.upstream.setCaches(caches)
}
addUpstream(event.upstream)
if (!started) {
start()
}
log.error("Upstream ${event.upstream.getId()} with chain $chain has been added")
}
}
}
fun haveUpstreams(): Boolean =
upstreams.isNotEmpty()
fun hasMatchingUpstream(matcher: Selector.LabelSelectorMatcher): Boolean {
return upstreams.any { matcher.matches(it) }
}
// --------------------------------------------------------------------------------------------------------
class UpstreamStatus(val upstream: Upstream, val status: UpstreamAvailability, val ts: Instant = Instant.now())

View File

@@ -16,17 +16,13 @@
*/
package io.emeraldpay.dshackle.upstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.grpc.Chain
import reactor.core.publisher.Flux
import io.emeraldpay.dshackle.Chain
/**
* Holds Multistreams configured for a chain.
*/
interface MultistreamHolder {
fun getUpstream(chain: Chain): Multistream?
fun getUpstream(chain: Chain): Multistream
fun getAvailable(): List<Chain>
fun observeChains(): Flux<Chain>
fun getDefaultMethods(chain: Chain): CallMethods
fun isAvailable(chain: Chain): Boolean
}

View File

@@ -15,24 +15,17 @@
*/
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.EmptyHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.RequestPostprocessor
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Mono
@Suppress("UNCHECKED_CAST")
@@ -142,7 +135,7 @@ open class BitcoinMultistream(
}
override fun isRunning(): Boolean {
return super.isRunning() || reader.isRunning
return super.isRunning() || reader.isRunning()
}
override fun start() {

View File

@@ -19,13 +19,13 @@ import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.bitcoinj.core.Address
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Mono
import reactor.kotlin.core.publisher.cast
@@ -72,7 +72,7 @@ open class BitcoinReader(
}
override fun isRunning(): Boolean {
return mempool.isRunning
return mempool.isRunning()
}
override fun start() {

View File

@@ -19,11 +19,11 @@ import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import org.springframework.scheduling.concurrent.CustomizableThreadFactory
import reactor.core.Disposable
import reactor.core.publisher.Flux

View File

@@ -15,19 +15,19 @@
*/
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
open class BitcoinRpcUpstream(
@@ -92,7 +92,7 @@ open class BitcoinRpcUpstream(
override fun isRunning(): Boolean {
var runningAny = validatorSubscription != null
if (head is Lifecycle) {
runningAny = runningAny || head.isRunning
runningAny = runningAny || head.isRunning()
}
return runningAny
}
@@ -100,7 +100,7 @@ open class BitcoinRpcUpstream(
override fun start() {
log.info("Configured for ${chain.chainName}")
if (head is Lifecycle) {
if (!head.isRunning) {
if (!head.isRunning()) {
head.start()
}
}

View File

@@ -15,12 +15,12 @@
*/
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
abstract class BitcoinUpstream(

View File

@@ -5,12 +5,12 @@ import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.apache.commons.codec.binary.Hex
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
@@ -65,6 +65,6 @@ class BitcoinZMQHead(
}
override fun isRunning(): Boolean {
return server.isRunning || refreshSubscription != null
return server.isRunning() || refreshSubscription != null
}
}

View File

@@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.upstream.bitcoin
import com.fasterxml.jackson.databind.ObjectMapper
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Mono
import java.time.Duration

View File

@@ -1,7 +1,7 @@
package io.emeraldpay.dshackle.upstream.bitcoin
import io.emeraldpay.dshackle.upstream.Lifecycle
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import org.zeromq.SocketType
import org.zeromq.ZContext
import org.zeromq.ZMQ

View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.upstream.calls
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.quorum.AlwaysQuorum
import io.emeraldpay.dshackle.quorum.BroadcastQuorum
@@ -23,7 +24,6 @@ import io.emeraldpay.dshackle.quorum.CallQuorum
import io.emeraldpay.dshackle.quorum.NonceQuorum
import io.emeraldpay.dshackle.quorum.NotLaggingQuorum
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
/**
* Default configuration for Ethereum based RPC. Defines optimal Quorum strategies for different methods, and provides
@@ -35,6 +35,20 @@ class DefaultEthereumMethods(
private val version = "\"EmeraldDshackle/${Global.version}\""
companion object {
val withFilterIdMethods = listOf(
"eth_getFilterChanges",
"eth_getFilterLogs",
"eth_uninstallFilter"
)
val newFilterMethods = listOf(
"eth_newFilter",
"eth_newBlockFilter",
"eth_newPendingTransactionFilter",
)
}
private val anyResponseMethods = listOf(
"eth_gasPrice",
"eth_call",
@@ -70,13 +84,7 @@ class DefaultEthereumMethods(
"eth_feeHistory"
)
private val filterMethods = listOf(
"eth_getFilterChanges",
"eth_newFilter",
"eth_newBlockFilter",
"eth_newPendingTransactionFilter",
"eth_uninstallFilter"
)
private val filterMethods = withFilterIdMethods + newFilterMethods
private val hardcodedMethods = listOf(
"net_version",

View File

@@ -57,7 +57,7 @@ class EthereumCallSelector(
return blockTagSelector(params, 1, head)
} else if (method == "eth_getStorageAt") {
return blockTagSelector(params, 2, head)
} else if (method == "eth_getFilterChanges" || method == "eth_uninstallFilter") {
} else if (method in DefaultEthereumMethods.withFilterIdMethods) {
return sameUpstreamMatcher(params)
}
return Mono.empty()

View File

@@ -17,23 +17,17 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.EmptyHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import org.springframework.util.ConcurrentReferenceHashMap
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
@@ -85,7 +79,7 @@ open class EthereumMultistream(
}
override fun isRunning(): Boolean {
return super.isRunning() || reader.isRunning
return super.isRunning() || reader.isRunning()
}
override fun getReader(): EthereumReader {

View File

@@ -30,6 +30,7 @@ import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.reader.RekeyingReader
import io.emeraldpay.dshackle.reader.RpcReader
import io.emeraldpay.dshackle.reader.TransformingReader
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.calls.CallMethods
import io.emeraldpay.etherjar.domain.Address
@@ -41,7 +42,6 @@ import io.emeraldpay.etherjar.rpc.json.TransactionJson
import io.emeraldpay.etherjar.rpc.json.TransactionRefJson
import org.apache.commons.collections4.Factory
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import java.util.function.Function
/**
@@ -174,7 +174,7 @@ open class EthereumReader(
override fun isRunning(): Boolean {
// TODO should be always running?
return up.isRunning
return true // up.isRunning
}
override fun start() {

View File

@@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import org.springframework.scheduling.concurrent.CustomizableThreadFactory
import reactor.core.Disposable
import reactor.core.publisher.Flux

View File

@@ -16,6 +16,7 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
@@ -29,7 +30,6 @@ import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
@@ -80,7 +80,7 @@ open class EthereumRpcUpstream(
}
override fun isRunning(): Boolean {
return connector.isRunning
return connector.isRunning()
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {

View File

@@ -16,11 +16,11 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.Chain
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Metrics
import io.micrometer.core.instrument.Tag

View File

@@ -17,10 +17,10 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsClient
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux

View File

@@ -1,8 +1,8 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.grpc.Chain
interface ConnectorFactory {
fun create(upstream: DefaultUpstream, validator: EthereumUpstreamValidator, chain: Chain): EthereumConnector

View File

@@ -2,9 +2,9 @@ package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.springframework.context.Lifecycle
interface EthereumConnector : Lifecycle {
fun getHead(): Head

View File

@@ -1,12 +1,12 @@
package io.emeraldpay.dshackle.upstream.ethereum.connectors
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.HttpFactory
import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstreamValidator
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
open class EthereumConnectorFactory(

View File

@@ -5,6 +5,7 @@ import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.BlockValidator
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumRpcHead
import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory
@@ -14,7 +15,6 @@ import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import java.time.Duration
class EthereumRpcConnector(
@@ -61,7 +61,7 @@ class EthereumRpcConnector(
override fun isRunning(): Boolean {
if (head is Lifecycle) {
return head.isRunning
return head.isRunning()
}
return true
}

View File

@@ -36,7 +36,7 @@ class EthereumWsConnector(
}
override fun isRunning(): Boolean {
return head.isRunning
return head.isRunning()
}
override fun stop() {

View File

@@ -17,23 +17,17 @@
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.ChainFees
import io.emeraldpay.dshackle.upstream.EmptyHead
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.MergedHead
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.*
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.forkchoice.PriorityForkChoice
import io.emeraldpay.dshackle.upstream.grpc.GrpcUpstream
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import org.springframework.util.ConcurrentReferenceHashMap
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
@@ -80,7 +74,7 @@ open class EthereumPosMultiStream(
}
override fun isRunning(): Boolean {
return super.isRunning() || reader.isRunning
return super.isRunning() || reader.isRunning()
}
override fun getReader(): EthereumReader {

View File

@@ -16,12 +16,14 @@
*/
package io.emeraldpay.dshackle.upstream.ethereum
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.cache.CachesEnabled
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.calls.CallMethods
@@ -29,9 +31,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.connectors.ConnectorFactory
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnector
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
open class EthereumPosRpcUpstream(
@@ -80,7 +80,7 @@ open class EthereumPosRpcUpstream(
}
override fun isRunning(): Boolean {
return connector.isRunning
return connector.isRunning()
}
override fun getApi(): Reader<JsonRpcRequest, JsonRpcResponse> {

View File

@@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
@@ -24,6 +25,7 @@ import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
@@ -34,10 +36,8 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.math.BigInteger

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
@@ -26,6 +27,7 @@ import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
@@ -37,10 +39,8 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.math.BigInteger

View File

@@ -18,6 +18,7 @@ package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.data.BlockContainer
@@ -26,6 +27,7 @@ import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.startup.QuorumForLabels
import io.emeraldpay.dshackle.upstream.Capability
import io.emeraldpay.dshackle.upstream.Head
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.Upstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
@@ -37,10 +39,8 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse
import io.emeraldpay.etherjar.domain.BlockHash
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
import java.math.BigInteger

View File

@@ -18,16 +18,16 @@ package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.Common
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.AbstractHead
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.Lifecycle
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.forkchoice.ForkChoice
import io.emeraldpay.grpc.Chain
import org.reactivestreams.Publisher
import org.slf4j.LoggerFactory
import org.springframework.context.Lifecycle
import reactor.core.Disposable
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono
@@ -60,7 +60,7 @@ class GrpcHead(
* Initiate a new head subscription with connection to the remote
*/
private fun internalStart(remote: ReactorBlockchainGrpc.ReactorBlockchainStub) {
if (this.isRunning) {
if (this.isRunning()) {
stop()
}
log.debug("Start Head subscription to ${parent.getId()}")

View File

@@ -18,17 +18,17 @@ package io.emeraldpay.dshackle.upstream.grpc
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Defaults
import io.emeraldpay.dshackle.FileResolver
import io.emeraldpay.dshackle.config.AuthConfig
import io.emeraldpay.dshackle.config.UpstreamsConfig
import io.emeraldpay.dshackle.startup.UpstreamChange
import io.emeraldpay.dshackle.startup.UpstreamChangeEvent
import io.emeraldpay.dshackle.upstream.DefaultUpstream
import io.emeraldpay.dshackle.upstream.UpstreamAvailability
import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient
import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics
import io.emeraldpay.grpc.BlockchainType
import io.emeraldpay.grpc.Chain
import io.grpc.ManagedChannelBuilder
import io.grpc.netty.NettyChannelBuilder
import io.micrometer.core.instrument.Counter
@@ -69,7 +69,7 @@ class GrpcUpstreams(
private val known = HashMap<Chain, DefaultUpstream>()
private val lock = ReentrantLock()
fun start(): Flux<UpstreamChange> {
fun start(): Flux<UpstreamChangeEvent> {
val channel: ManagedChannelBuilder<*> = if (auth != null && StringUtils.isNotEmpty(auth.ca)) {
NettyChannelBuilder.forAddress(host, port)
// some messages are very large. many of them in megabytes, some even in gigabytes (ex. ETH Traces)
@@ -122,7 +122,7 @@ class GrpcUpstreams(
return updates
}
fun processDescription(value: BlockchainOuterClass.DescribeResponse): Flux<UpstreamChange> {
fun processDescription(value: BlockchainOuterClass.DescribeResponse): Flux<UpstreamChangeEvent> {
val current = value.chainsList.filter {
Chain.byId(it.chain.number) != Chain.UNSPECIFIED
}.mapNotNull { chainDetails ->
@@ -138,14 +138,14 @@ class GrpcUpstreams(
}
val added = current.filter {
it.type == UpstreamChange.ChangeType.ADDED
it.type == UpstreamChangeEvent.ChangeType.ADDED
}
val removed = known.filterNot { kv ->
val stillCurrent = current.any { c -> c.chain == kv.key }
stillCurrent
}.map {
UpstreamChange(it.key, known.remove(it.key)!!, UpstreamChange.ChangeType.REMOVED)
UpstreamChangeEvent(it.key, known.remove(it.key)!!, UpstreamChangeEvent.ChangeType.REMOVED)
}
return Flux.fromIterable(removed + added)
}
@@ -172,7 +172,7 @@ class GrpcUpstreams(
return sslContext.build()
}
fun getOrCreate(chain: Chain): UpstreamChange {
fun getOrCreate(chain: Chain): UpstreamChangeEvent {
val metricsTags = listOf(
Tag.of("upstream", id),
Tag.of("chain", chain.chainCode)
@@ -202,7 +202,7 @@ class GrpcUpstreams(
}
}
fun getOrCreateEthereum(chain: Chain, metrics: RpcMetrics): UpstreamChange {
fun getOrCreateEthereum(chain: Chain, metrics: RpcMetrics): UpstreamChangeEvent {
lock.withLock {
val current = known[chain]
return if (current == null) {
@@ -211,14 +211,14 @@ class GrpcUpstreams(
created.timeout = this.timeout
known[chain] = created
created.start()
UpstreamChange(chain, created, UpstreamChange.ChangeType.ADDED)
UpstreamChangeEvent(chain, created, UpstreamChangeEvent.ChangeType.ADDED)
} else {
UpstreamChange(chain, current, UpstreamChange.ChangeType.REVALIDATED)
UpstreamChangeEvent(chain, current, UpstreamChangeEvent.ChangeType.REVALIDATED)
}
}
}
fun getOrCreateEthereumPos(chain: Chain, metrics: RpcMetrics): UpstreamChange {
fun getOrCreateEthereumPos(chain: Chain, metrics: RpcMetrics): UpstreamChangeEvent {
lock.withLock {
val current = known[chain]
return if (current == null) {
@@ -227,14 +227,14 @@ class GrpcUpstreams(
created.timeout = this.timeout
known[chain] = created
created.start()
UpstreamChange(chain, created, UpstreamChange.ChangeType.ADDED)
UpstreamChangeEvent(chain, created, UpstreamChangeEvent.ChangeType.ADDED)
} else {
UpstreamChange(chain, current, UpstreamChange.ChangeType.REVALIDATED)
UpstreamChangeEvent(chain, current, UpstreamChangeEvent.ChangeType.REVALIDATED)
}
}
}
fun getOrCreateBitcoin(chain: Chain, metrics: RpcMetrics): UpstreamChange {
fun getOrCreateBitcoin(chain: Chain, metrics: RpcMetrics): UpstreamChangeEvent {
lock.withLock {
val current = known[chain]
return if (current == null) {
@@ -243,9 +243,9 @@ class GrpcUpstreams(
created.timeout = this.timeout
known[chain] = created
created.start()
UpstreamChange(chain, created, UpstreamChange.ChangeType.ADDED)
UpstreamChangeEvent(chain, created, UpstreamChangeEvent.ChangeType.ADDED)
} else {
UpstreamChange(chain, current, UpstreamChange.ChangeType.REVALIDATED)
UpstreamChangeEvent(chain, current, UpstreamChangeEvent.ChangeType.REVALIDATED)
}
}
}

View File

@@ -19,13 +19,13 @@ import com.google.protobuf.ByteString
import io.emeraldpay.api.proto.BlockchainOuterClass
import io.emeraldpay.api.proto.BlockchainOuterClass.NativeCallReplySignature
import io.emeraldpay.api.proto.ReactorBlockchainGrpc
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.Global
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.upstream.Selector
import io.emeraldpay.dshackle.upstream.signature.ResponseSigner
import io.emeraldpay.etherjar.rpc.RpcException
import io.emeraldpay.etherjar.rpc.RpcResponseError
import io.emeraldpay.grpc.Chain
import org.slf4j.LoggerFactory
import reactor.core.publisher.Mono
import java.util.concurrent.TimeUnit

View File

@@ -41,4 +41,4 @@
</Root>
</Loggers>
</Configuration>
</Configuration>