From 5167b8b125a47a15e9037ebf8b7cc7be771625d3 Mon Sep 17 00:00:00 2001 From: Aldo Borrero <82811+aldoborrero@users.noreply.github.com> Date: Thu, 21 Oct 2021 02:00:38 +0200 Subject: [PATCH] solution: kotlin code conventions --- .gitignore | 8 +- build.gradle | 1 + gradle.properties | 2 + .../io/emeraldpay/dshackle/ChainValue.kt | 6 +- .../kotlin/io/emeraldpay/dshackle/Config.kt | 30 +- .../kotlin/io/emeraldpay/dshackle/Defaults.kt | 2 +- .../io/emeraldpay/dshackle/FileResolver.kt | 5 +- .../kotlin/io/emeraldpay/dshackle/Global.kt | 10 +- .../io/emeraldpay/dshackle/GrpcServer.kt | 29 +- .../io/emeraldpay/dshackle/ProxyStarter.kt | 22 +- .../io/emeraldpay/dshackle/SilentException.kt | 3 +- .../kotlin/io/emeraldpay/dshackle/Starter.kt | 4 +- .../kotlin/io/emeraldpay/dshackle/TlsSetup.kt | 15 +- .../dshackle/cache/BlockByHeight.kt | 9 +- .../dshackle/cache/BlocksMemCache.kt | 8 +- .../dshackle/cache/BlocksRedisCache.kt | 31 +- .../io/emeraldpay/dshackle/cache/Caches.kt | 53 +-- .../dshackle/cache/CachesEnabled.kt | 3 +- .../dshackle/cache/CachesFactory.kt | 13 +- .../dshackle/cache/CurrentBlockCache.kt | 3 +- .../dshackle/cache/HeightByHashAdding.kt | 39 ++- .../dshackle/cache/HeightByHashCache.kt | 2 +- .../dshackle/cache/HeightByHashMemCache.kt | 8 +- .../dshackle/cache/HeightByHashRedisCache.kt | 60 ++-- .../emeraldpay/dshackle/cache/HeightCache.kt | 9 +- .../dshackle/cache/OnBlockRedisCache.kt | 82 ++--- .../dshackle/cache/OnTxRedisCache.kt | 96 +++--- .../dshackle/cache/ReceiptMemCache.kt | 11 +- .../dshackle/cache/ReceiptRedisCache.kt | 9 +- .../emeraldpay/dshackle/cache/TxMemCache.kt | 16 +- .../emeraldpay/dshackle/cache/TxRedisCache.kt | 24 +- .../dshackle/config/AccessLogConfig.kt | 7 +- .../dshackle/config/AccessLogReader.kt | 3 +- .../emeraldpay/dshackle/config/AuthConfig.kt | 12 +- .../dshackle/config/AuthConfigReader.kt | 3 +- .../emeraldpay/dshackle/config/CacheConfig.kt | 12 +- .../dshackle/config/CacheConfigReader.kt | 2 +- .../dshackle/config/ConfigReader.kt | 3 +- .../dshackle/config/EnvVariables.kt | 2 +- .../dshackle/config/InvalidConfigException.kt | 10 +- .../emeraldpay/dshackle/config/MainConfig.kt | 2 +- .../dshackle/config/MainConfigReader.kt | 5 +- .../dshackle/config/MonitoringConfig.kt | 17 +- .../dshackle/config/MonitoringConfigReader.kt | 5 +- .../emeraldpay/dshackle/config/ProxyConfig.kt | 20 +- .../dshackle/config/ProxyConfigReader.kt | 5 +- .../dshackle/config/TokensConfig.kt | 15 +- .../dshackle/config/TokensConfigReader.kt | 5 +- .../dshackle/config/UpstreamsConfig.kt | 35 +- .../dshackle/config/UpstreamsConfigReader.kt | 36 +- .../dshackle/config/YamlConfigReader.kt | 54 +-- .../dshackle/data/BlockContainer.kt | 36 +- .../io/emeraldpay/dshackle/data/BlockId.kt | 6 +- .../dshackle/data/DefaultContainer.kt | 12 +- .../io/emeraldpay/dshackle/data/HashId.kt | 6 +- .../dshackle/data/RawJsonBuilder.kt | 6 +- .../dshackle/data/SourceContainer.kt | 9 +- .../emeraldpay/dshackle/data/TxContainer.kt | 24 +- .../io/emeraldpay/dshackle/data/TxId.kt | 5 +- .../dshackle/monitoring/MonitoringSetup.kt | 24 +- .../monitoring/accesslog/AccessHandlerGrpc.kt | 126 +++---- .../monitoring/accesslog/AccessHandlerHttp.kt | 26 +- .../monitoring/accesslog/AccessLogWriter.kt | 9 +- .../dshackle/monitoring/accesslog/Events.kt | 152 +++++---- .../monitoring/accesslog/EventsBuilder.kt | 209 ++++++------ .../io/emeraldpay/dshackle/proxy/ProxyCall.kt | 10 +- .../emeraldpay/dshackle/proxy/ProxyServer.kt | 130 ++++---- .../emeraldpay/dshackle/proxy/ReadRpcJson.kt | 64 ++-- .../emeraldpay/dshackle/proxy/WriteRpcJson.kt | 65 ++-- .../dshackle/quorum/AlwaysQuorum.kt | 5 +- .../dshackle/quorum/BroadcastQuorum.kt | 4 +- .../emeraldpay/dshackle/quorum/CallQuorum.kt | 6 +- .../dshackle/quorum/NonEmptyQuorum.kt | 8 +- .../emeraldpay/dshackle/quorum/NonceQuorum.kt | 7 +- .../dshackle/quorum/NotLaggingQuorum.kt | 5 +- .../dshackle/quorum/QuorumReaderFactory.kt | 2 +- .../dshackle/quorum/QuorumRpcReader.kt | 127 ++++--- .../dshackle/quorum/ValueAwareQuorum.kt | 10 +- .../dshackle/reader/CompoundReader.kt | 22 +- .../emeraldpay/dshackle/reader/EmptyReader.kt | 4 +- .../io/emeraldpay/dshackle/reader/Reader.kt | 3 +- .../dshackle/reader/RekeyingReader.kt | 27 +- .../emeraldpay/dshackle/reader/RpcReader.kt | 15 +- .../dshackle/reader/TransformingReader.kt | 19 +- .../emeraldpay/dshackle/rpc/BlockchainRpc.kt | 166 +++++----- .../io/emeraldpay/dshackle/rpc/Describe.kt | 41 +-- .../dshackle/rpc/EthereumAddresses.kt | 5 +- .../io/emeraldpay/dshackle/rpc/NativeCall.kt | 148 +++++---- .../dshackle/rpc/NativeSubscribe.kt | 25 +- .../io/emeraldpay/dshackle/rpc/StreamHead.kt | 29 +- .../dshackle/rpc/SubscribeStatus.kt | 25 +- .../emeraldpay/dshackle/rpc/TrackAddress.kt | 4 +- .../dshackle/rpc/TrackBitcoinAddress.kt | 187 ++++++----- .../emeraldpay/dshackle/rpc/TrackBitcoinTx.kt | 116 ++++--- .../dshackle/rpc/TrackERC20Address.kt | 91 +++--- .../dshackle/rpc/TrackEthereumAddress.kt | 101 +++--- .../dshackle/rpc/TrackEthereumTx.kt | 280 ++++++++-------- .../io/emeraldpay/dshackle/rpc/TrackTx.kt | 2 +- .../dshackle/startup/ConfiguredUpstreams.kt | 173 +++++----- .../dshackle/startup/QuorumForLabels.kt | 9 +- .../dshackle/startup/UpstreamChange.kt | 28 +- .../dshackle/upstream/AbstractHead.kt | 64 ++-- .../emeraldpay/dshackle/upstream/ApiSource.kt | 3 +- .../dshackle/upstream/Capability.kt | 2 +- .../upstream/CurrentMultistreamHolder.kt | 32 +- .../dshackle/upstream/DefaultUpstream.kt | 55 +++- .../emeraldpay/dshackle/upstream/EmptyHead.kt | 3 +- .../dshackle/upstream/FilteredApis.kt | 86 ++--- .../io/emeraldpay/dshackle/upstream/Head.kt | 3 +- .../dshackle/upstream/HeadLagObserver.kt | 35 +- .../dshackle/upstream/MergedHead.kt | 5 +- .../dshackle/upstream/Multistream.kt | 88 ++--- .../dshackle/upstream/MultistreamHolder.kt | 2 +- .../dshackle/upstream/RequestPostprocessor.kt | 22 +- .../emeraldpay/dshackle/upstream/Selector.kt | 87 +++-- .../emeraldpay/dshackle/upstream/Upstream.kt | 3 +- .../dshackle/upstream/UpstreamAvailability.kt | 6 +- .../upstream/bitcoin/AddressActiveCheck.kt | 7 +- .../bitcoin/BitcoinHeadLagObserver.kt | 9 +- .../upstream/bitcoin/BitcoinMultistream.kt | 18 +- .../upstream/bitcoin/BitcoinReader.kt | 16 +- .../upstream/bitcoin/BitcoinRpcHead.kt | 44 +-- .../upstream/bitcoin/BitcoinRpcUpstream.kt | 32 +- .../upstream/bitcoin/BitcoinUpstream.kt | 27 +- .../bitcoin/BitcoinUpstreamValidator.kt | 38 +-- .../upstream/bitcoin/CachingMempoolData.kt | 23 +- .../upstream/bitcoin/EsploraClient.kt | 45 +-- .../upstream/bitcoin/EsploraUnspentReader.kt | 17 +- .../dshackle/upstream/bitcoin/ExtractBlock.kt | 22 +- .../upstream/bitcoin/RpcUnspentReader.kt | 31 +- .../upstream/bitcoin/UnspentReader.kt | 3 +- .../upstream/bitcoin/XpubAddresses.kt | 39 ++- .../upstream/bitcoin/data/EsploraUnspent.kt | 12 +- .../data/EsploraUnspentDeserializer.kt | 12 +- .../upstream/bitcoin/data/RpcUnspent.kt | 12 +- .../bitcoin/data/RpcUnspentDeserializer.kt | 13 +- .../upstream/bitcoin/data/SimpleUnspent.kt | 10 +- .../upstream/calls/AggregatedCallMethods.kt | 9 +- .../dshackle/upstream/calls/CallMethods.kt | 2 +- .../upstream/calls/DefaultBitcoinMethods.kt | 34 +- .../upstream/calls/DefaultEthereumMethods.kt | 79 ++--- .../upstream/calls/DirectCallMethods.kt | 2 +- .../upstream/calls/EthereumCallSelector.kt | 23 +- .../upstream/calls/ManagedCallMethods.kt | 15 +- .../upstream/ethereum/CacheRequested.kt | 16 +- .../upstream/ethereum/DefaultEthereumHead.kt | 48 +-- .../upstream/ethereum/EthereumDirectReader.kt | 93 +++--- .../ethereum/EthereumFullBlocksReader.kt | 72 ++-- .../ethereum/EthereumHeadLagObserver.kt | 15 +- .../upstream/ethereum/EthereumMultistream.kt | 14 +- .../upstream/ethereum/EthereumReader.kt | 77 +++-- .../upstream/ethereum/EthereumRpcHead.kt | 24 +- .../upstream/ethereum/EthereumRpcUpstream.kt | 36 +- .../upstream/ethereum/EthereumSubscribe.kt | 9 +- .../upstream/ethereum/EthereumUpstream.kt | 15 +- .../ethereum/EthereumUpstreamValidator.kt | 67 ++-- .../upstream/ethereum/EthereumWsFactory.kt | 309 +++++++++--------- .../upstream/ethereum/EthereumWsHead.kt | 11 +- .../upstream/ethereum/EthereumWsUpstream.kt | 42 +-- .../upstream/ethereum/NativeCallRouter.kt | 16 +- .../ethereum/subscribe/ConnectBlockUpdates.kt | 54 +-- .../ethereum/subscribe/ConnectLogs.kt | 20 +- .../ethereum/subscribe/ConnectNewHeads.kt | 20 +- .../ethereum/subscribe/ConnectSyncing.kt | 18 +- .../ethereum/subscribe/ProduceLogs.kt | 68 ++-- .../ethereum/subscribe/ProduceNewHeads.kt | 45 ++- .../ethereum/subscribe/json/LogMessage.kt | 36 +- .../ethereum/subscribe/json/NewHeadMessage.kt | 42 +-- .../subscribe/json/NumberAsHexSerializer.kt | 3 +- .../subscribe/json/TimestampSerializer.kt | 3 +- .../upstream/grpc/BitcoinGrpcUpstream.kt | 67 ++-- .../upstream/grpc/EthereumGrpcUpstream.kt | 75 +++-- .../dshackle/upstream/grpc/GrpcHead.kt | 60 ++-- .../dshackle/upstream/grpc/GrpcUpstream.kt | 3 +- .../upstream/grpc/GrpcUpstreamStatus.kt | 22 +- .../dshackle/upstream/grpc/GrpcUpstreams.kt | 122 +++---- .../upstream/grpc/RemoteCapabilities.kt | 3 +- .../upstream/rpcclient/JsonRpcError.kt | 6 +- .../upstream/rpcclient/JsonRpcException.kt | 8 +- .../upstream/rpcclient/JsonRpcGrpcClient.kt | 80 ++--- .../upstream/rpcclient/JsonRpcHttpClient.kt | 67 ++-- .../upstream/rpcclient/JsonRpcRequest.kt | 17 +- .../upstream/rpcclient/JsonRpcResponse.kt | 10 +- .../upstream/rpcclient/JsonRpcWsClient.kt | 5 +- .../upstream/rpcclient/ResponseParser.kt | 42 ++- .../upstream/rpcclient/ResponseRpcParser.kt | 5 +- .../upstream/rpcclient/ResponseWSParser.kt | 32 +- .../dshackle/upstream/rpcclient/RpcMetrics.kt | 6 +- 188 files changed, 3261 insertions(+), 3111 deletions(-) diff --git a/.gitignore b/.gitignore index fdb8ad06..a2988251 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ .gradle/ build/ out/ -*.iml ./dshackle.yaml ./upstream.yaml -testsetup/ \ No newline at end of file +testsetup/ +.idea/ +.idea_modules/ +*.iml +*.ipr +*.iws \ No newline at end of file diff --git a/build.gradle b/build.gradle index 44309b3a..76cb14d7 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ plugins { id 'io.spring.dependency-management' version '1.0.6.RELEASE' id 'com.palantir.git-version' version '0.12.2' id "com.google.protobuf" version "0.8.12" + id "org.jlleitschuh.gradle.ktlint" version "10.2.0" } diff --git a/gradle.properties b/gradle.properties index dcde211e..e67fc02c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,5 @@ +kotlin.code.style=official + # Languages groovyVersion=2.5.14 kotlinVersion=1.5.30 diff --git a/src/main/kotlin/io/emeraldpay/dshackle/ChainValue.kt b/src/main/kotlin/io/emeraldpay/dshackle/ChainValue.kt index 15d318e8..ce947786 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/ChainValue.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/ChainValue.kt @@ -17,7 +17,7 @@ package io.emeraldpay.dshackle import io.emeraldpay.api.proto.Common import io.emeraldpay.grpc.Chain -import java.util.* +import java.util.EnumMap import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock @@ -25,7 +25,7 @@ import kotlin.concurrent.withLock * Keeps a lazily created value associated with a Chain */ class ChainValue( - private val factory: (chain: Chain) -> V + private val factory: (chain: Chain) -> V ) { private val values = EnumMap(Chain::class.java) @@ -52,4 +52,4 @@ class ChainValue( } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index c012ba7c..a619f181 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -16,26 +16,26 @@ */ package io.emeraldpay.dshackle -import com.fasterxml.jackson.core.Version -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.module.SimpleModule -import io.emeraldpay.dshackle.config.* +import io.emeraldpay.dshackle.config.CacheConfig +import io.emeraldpay.dshackle.config.MainConfig +import io.emeraldpay.dshackle.config.MainConfigReader +import io.emeraldpay.dshackle.config.MonitoringConfig +import io.emeraldpay.dshackle.config.TokensConfig +import io.emeraldpay.dshackle.config.UpstreamsConfig import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.boot.ExitCodeGenerator import org.springframework.boot.SpringApplication import org.springframework.context.ApplicationContext -import org.springframework.context.annotation.* +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration import org.springframework.core.env.Environment import org.springframework.scheduling.annotation.EnableAsync import org.springframework.scheduling.annotation.EnableScheduling import reactor.core.scheduler.Scheduler import reactor.core.scheduler.Schedulers import java.io.File -import java.text.SimpleDateFormat -import java.util.* import java.util.concurrent.Executors import kotlin.system.exitProcess @@ -43,8 +43,8 @@ import kotlin.system.exitProcess @EnableScheduling @EnableAsync open class Config( - @Autowired private val env: Environment, - @Autowired private val ctx: ApplicationContext + @Autowired private val env: Environment, + @Autowired private val ctx: ApplicationContext ) { companion object { @@ -68,14 +68,15 @@ open class Config( if (!FileResolver.isAccessible(target)) { target = File(LOCAL_CONFIG) if (!FileResolver.isAccessible(target)) { - throw IllegalStateException("Configuration is not found neither at ${DEFAULT_CONFIG} nor ${LOCAL_CONFIG}") + throw IllegalStateException("Configuration is not found neither at $DEFAULT_CONFIG nor $LOCAL_CONFIG") } } target = target.normalize() return target } - @Bean @Qualifier("upstreamScheduler") + @Bean + @Qualifier("upstreamScheduler") open fun upstreamScheduler(): Scheduler { return Schedulers.fromExecutorService(Executors.newFixedThreadPool(16)) } @@ -91,7 +92,7 @@ open class Config( } val reader = MainConfigReader(fileResolver) return reader.read(f.inputStream()) - ?: throw IllegalStateException("Config is not available at ${f.absolutePath}") + ?: throw IllegalStateException("Config is not available at ${f.absolutePath}") } @Bean @@ -119,5 +120,4 @@ open class Config( open fun monitoringConfig(@Autowired mainConfig: MainConfig): MonitoringConfig { return mainConfig.monitoring } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Defaults.kt b/src/main/kotlin/io/emeraldpay/dshackle/Defaults.kt index d79ee7d9..fe9bc976 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Defaults.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Defaults.kt @@ -25,4 +25,4 @@ class Defaults { val timeoutInternal: Duration = timeout.dividedBy(4) val retryConnection: Duration = Duration.ofSeconds(10) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt b/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt index f499c0e1..d09043bf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/FileResolver.kt @@ -19,7 +19,7 @@ package io.emeraldpay.dshackle import java.io.File open class FileResolver( - private val baseDir: File + private val baseDir: File ) { companion object { @@ -35,5 +35,4 @@ open class FileResolver( } return File(baseDir, path) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index db3c8da7..78650ea8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -28,7 +28,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspentDeserializer import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import java.text.SimpleDateFormat -import java.util.* +import java.util.TimeZone import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService @@ -57,12 +57,10 @@ class Global { objectMapper.registerModule(JavaTimeModule()) objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) objectMapper - .setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS")) - .setTimeZone(TimeZone.getTimeZone("UTC")) + .setDateFormat(SimpleDateFormat("yyyy-MM-dd\'T\'HH:mm:ss.SSS")) + .setTimeZone(TimeZone.getTimeZone("UTC")) return objectMapper } - } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt index 62a09f4a..8923263a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt @@ -18,7 +18,7 @@ package io.emeraldpay.dshackle import io.emeraldpay.dshackle.config.MainConfig import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerGrpc -import io.grpc.* +import io.grpc.Server import io.grpc.netty.NettyServerBuilder import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -29,15 +29,15 @@ import javax.annotation.PreDestroy @Service open class GrpcServer( - @Autowired val rpcs: List, - @Autowired val mainConfig: MainConfig, - @Autowired val tlsSetup: TlsSetup, - @Autowired val accessHandler: AccessHandlerGrpc + @Autowired val rpcs: List, + @Autowired val mainConfig: MainConfig, + @Autowired val tlsSetup: TlsSetup, + @Autowired val accessHandler: AccessHandlerGrpc ) { private val log = LoggerFactory.getLogger(GrpcServer::class.java) - private var server: Server? = null; + private var server: Server? = null @PostConstruct fun start() { @@ -45,14 +45,14 @@ open class GrpcServer( log.debug("Running with DEBUG LOGGING") log.info("Listening Native gRPC on ${mainConfig.host}:${mainConfig.port}") val serverBuilder = NettyServerBuilder - .forAddress(InetSocketAddress(mainConfig.host, mainConfig.port)) - .let { - if (mainConfig.accessLogConfig.enabled) { - it.intercept(accessHandler) - } else { - it - } + .forAddress(InetSocketAddress(mainConfig.host, mainConfig.port)) + .let { + if (mainConfig.accessLogConfig.enabled) { + it.intercept(accessHandler) + } else { + it } + } tlsSetup.setupServer("Native gRPC", mainConfig.tls, true)?.let { serverBuilder.sslContext(it) @@ -75,5 +75,4 @@ open class GrpcServer( server?.shutdownNow() log.info("GRPC Server shot down") } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt b/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt index 23149321..f1f248c8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt @@ -17,7 +17,6 @@ package io.emeraldpay.dshackle import io.emeraldpay.dshackle.config.MainConfig -import io.emeraldpay.dshackle.config.ProxyConfig import io.emeraldpay.dshackle.monitoring.MonitoringSetup import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp import io.emeraldpay.dshackle.proxy.ProxyServer @@ -26,8 +25,6 @@ import io.emeraldpay.dshackle.proxy.WriteRpcJson import io.emeraldpay.dshackle.rpc.NativeCall import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired -import org.springframework.context.annotation.DependsOn -import org.springframework.core.env.Environment import org.springframework.stereotype.Service import javax.annotation.PostConstruct @@ -36,14 +33,14 @@ import javax.annotation.PostConstruct */ @Service class ProxyStarter( - @Autowired private val mainConfig: MainConfig, - @Autowired private val readRpcJson: ReadRpcJson, - @Autowired private val writeRpcJson: WriteRpcJson, - @Autowired private val nativeCall: NativeCall, - @Autowired private val tlsSetup: TlsSetup, - @Autowired private val accessHandlerHttp: AccessHandlerHttp, - // depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy - @Autowired private val monitoringSetup: MonitoringSetup + @Autowired private val mainConfig: MainConfig, + @Autowired private val readRpcJson: ReadRpcJson, + @Autowired private val writeRpcJson: WriteRpcJson, + @Autowired private val nativeCall: NativeCall, + @Autowired private val tlsSetup: TlsSetup, + @Autowired private val accessHandlerHttp: AccessHandlerHttp, + // depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy + @Autowired private val monitoringSetup: MonitoringSetup ) { companion object { @@ -60,5 +57,4 @@ class ProxyStarter( val server = ProxyServer(config, readRpcJson, writeRpcJson, nativeCall, tlsSetup, accessHandlerHttp.factory) server.start() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/SilentException.kt b/src/main/kotlin/io/emeraldpay/dshackle/SilentException.kt index 5fe77aa7..7b15b6c0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/SilentException.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/SilentException.kt @@ -22,7 +22,6 @@ import io.emeraldpay.grpc.Chain */ open class SilentException(message: String) : Exception(message) { - /** * Blockchain is not available or not supported by current instance of the Dshackle */ @@ -31,4 +30,4 @@ open class SilentException(message: String) : Exception(message) { } class DataUnavailable(val code: String) : SilentException("Data is unavailable: $code") -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt index 34515d63..807b399b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt @@ -24,7 +24,7 @@ import org.springframework.context.annotation.Import import org.springframework.core.io.ClassPathResource import org.springframework.core.io.support.ResourcePropertySource -@SpringBootApplication(scanBasePackages = [ "io.emeraldpay.dshackle" ]) +@SpringBootApplication(scanBasePackages = ["io.emeraldpay.dshackle"]) @Import(Config::class) open class Starter @@ -35,4 +35,4 @@ fun main(args: Array) { app.setDefaultProperties(ResourcePropertySource("version.properties").source) app.setBanner(ResourceBanner(ClassPathResource("banner.txt"))) app.run(*args) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt index c4f0d8e0..2772f36d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt @@ -29,7 +29,7 @@ import org.springframework.stereotype.Service @Service open class TlsSetup( - @Autowired val fileResolver: FileResolver + @Autowired val fileResolver: FileResolver ) { companion object { @@ -68,19 +68,19 @@ open class TlsSetup( val sslContextBuilder = if (grpc) { GrpcSslContexts.forServer( - fileResolver.resolve(config.certificate!!), - fileResolver.resolve(config.key!!) + fileResolver.resolve(config.certificate!!), + fileResolver.resolve(config.key!!) ) } else { SslContextBuilder.forServer( - fileResolver.resolve(config.certificate!!), - fileResolver.resolve(config.key!!) + fileResolver.resolve(config.certificate!!), + fileResolver.resolve(config.key!!) ) } if (StringUtils.isNotEmpty(config.clientCa)) { log.info("Using TLS for client authentication for $category") sslContextBuilder.trustManager( - fileResolver.resolve(config.clientCa!!) + fileResolver.resolve(config.clientCa!!) ) if (config.clientRequire != null && config.clientRequire!!) { sslContextBuilder.clientAuth(ClientAuth.REQUIRE) @@ -96,5 +96,4 @@ open class TlsSetup( } return null } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt index a8862b1d..b396abe0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlockByHeight.kt @@ -25,8 +25,8 @@ import reactor.core.publisher.Mono * Connects two caches to read through them. First is cache height->hash, second is hash->block. */ open class BlockByHeight( - private val heights: Reader, - private val blocks: Reader + private val heights: Reader, + private val blocks: Reader ) : Reader { companion object { @@ -35,7 +35,6 @@ open class BlockByHeight( override fun read(key: Long): Mono { return heights.read(key) - .flatMap { blocks.read(it) } + .flatMap { blocks.read(it) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt index 841a39b7..bb1d403b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksMemCache.kt @@ -23,12 +23,12 @@ import io.emeraldpay.dshackle.reader.Reader import reactor.core.publisher.Mono open class BlocksMemCache( - maxSize: Int = 64 + maxSize: Int = 64 ) : Reader { private val mapping = Caffeine.newBuilder() - .maximumSize(maxSize.toLong()) - .build() + .maximumSize(maxSize.toLong()) + .build() override fun read(key: BlockId): Mono { return Mono.justOrEmpty(get(key)) @@ -45,4 +45,4 @@ open class BlocksMemCache( open fun purge() { mapping.cleanUp() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt index 7bf3de73..c4c421e8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/BlocksRedisCache.kt @@ -32,10 +32,10 @@ import java.time.Instant * Cache blocks in Redis database */ class BlocksRedisCache( - redis: RedisReactiveCommands, - chain: Chain + redis: RedisReactiveCommands, + chain: Chain ) : Reader, - OnBlockRedisCache(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) { + OnBlockRedisCache(redis, chain, CachesProto.ValueContainer.ValueType.BLOCK) { companion object { private val log = LoggerFactory.getLogger(BlocksRedisCache::class.java) @@ -59,21 +59,21 @@ class BlocksRedisCache( } val meta = value.blockMeta return BlockContainer( - meta.height, - BlockId(meta.hash.toByteArray()), - BigInteger(meta.difficulty.toByteArray()), - Instant.ofEpochMilli(meta.timestamp), - false, - value.value.toByteArray(), - null, - meta.txHashesList.map { - TxId(it.toByteArray()) - } + meta.height, + BlockId(meta.hash.toByteArray()), + BigInteger(meta.difficulty.toByteArray()), + Instant.ofEpochMilli(meta.timestamp), + false, + value.value.toByteArray(), + null, + meta.txHashesList.map { + TxId(it.toByteArray()) + } ) } fun add(block: BlockContainer): Mono { - if (block.timestamp == null || block.hash == null) { //null in unit tests + if (block.timestamp == null || block.hash == null) { // null in unit tests return Mono.empty() } if (block.full) { @@ -81,5 +81,4 @@ class BlocksRedisCache( } return super.add(block, block) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt index 179089cc..8a5b71db 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt @@ -16,9 +16,12 @@ package io.emeraldpay.dshackle.cache import io.emeraldpay.dshackle.Global -import io.emeraldpay.dshackle.data.* +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.DefaultContainer +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.reader.CompoundReader -import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.ethereum.EthereumFullBlocksReader @@ -30,14 +33,14 @@ import reactor.core.publisher.Flux import reactor.core.publisher.Mono open class Caches( - private val memBlocksByHash: BlocksMemCache, - private val blocksByHeight: HeightCache, - private val memTxsByHash: TxMemCache, - private val memReceipts: ReceiptMemCache, - private val redisBlocksByHash: BlocksRedisCache?, - private val redisTxsByHash: TxRedisCache?, - private val redisReceipts: ReceiptRedisCache?, - private val redisHeightByHashCache: HeightByHashRedisCache? + private val memBlocksByHash: BlocksMemCache, + private val blocksByHeight: HeightCache, + private val memTxsByHash: TxMemCache, + private val memReceipts: ReceiptMemCache, + private val redisBlocksByHash: BlocksRedisCache?, + private val redisTxsByHash: TxRedisCache?, + private val redisReceipts: ReceiptRedisCache?, + private val redisHeightByHashCache: HeightByHashRedisCache? ) { companion object { @@ -91,17 +94,17 @@ open class Caches( if (currentHeight != null && data.height != null && memReceipts.acceptsRecentBlocks(currentHeight - data.height)) { memReceipts.add(data) } - //TODO move subscription to the caller + // TODO move subscription to the caller redisReceipts?.add(data)?.subscribe() } fun cache(tag: Tag, tx: TxContainer) { - //do not cache transactions that are not in a block yet + // do not cache transactions that are not in a block yet if (tx.blockId == null) { return } memTxsByHash.add(tx) - //TODO move subscription to the caller + // TODO move subscription to the caller getBlocksByHash().read(tx.blockId).flatMap { block -> redisTxsByHash?.add(tx, block) ?: Mono.empty() }.subscribe() @@ -113,14 +116,14 @@ open class Caches( redisHeightByHashCache?.add(block)?.let(job::add) if (tag == Tag.LATEST) { - //for LATEST data cache it in memory, it may be short living so better to avoid Redis + // for LATEST data cache it in memory, it may be short living so better to avoid Redis memoizeBlock(block) } else if (tag == Tag.REQUESTED) { val blockOnlyContainer: BlockContainer? var jsonValue: BlockJson<*>? = null if (block.full) { jsonValue = Global.objectMapper.readValue>(block.json, BlockJson::class.java) - //shouldn't cache block json with transactions, separate txes and blocks with refs + // shouldn't cache block json with transactions, separate txes and blocks with refs val blockOnly = jsonValue.withoutTransactionDetails() blockOnlyContainer = BlockContainer.from(blockOnly) } else { @@ -138,15 +141,17 @@ open class Caches( TxContainer.from(tx) } if (redisTxsByHash != null) { - job.add(Flux.fromIterable(transactions) + job.add( + Flux.fromIterable(transactions) .doOnNext { memTxsByHash.add(it) } .flatMap { redisTxsByHash.add(it, block) } - .then()) + .then() + ) } } } } - Flux.fromIterable(job).flatMap { it }.subscribe() //TODO move out to a caller + Flux.fromIterable(job).flatMap { it }.subscribe() // TODO move out to a caller } /** @@ -156,7 +161,7 @@ open class Caches( memBlocksByHash.add(block) memHeightByHash.add(block) val replaced = blocksByHeight.add(block) - //evict cached transactions if an existing block was updated + // evict cached transactions if an existing block was updated replaced?.let { evict(it) } } @@ -222,7 +227,7 @@ open class Caches( REQUESTED } - class Builder() { + class Builder { private var blocksByHash: BlocksMemCache? = null private var blocksByHeight: HeightCache? = null private var txsByHash: TxMemCache? = null @@ -285,8 +290,10 @@ open class Caches( if (receipts == null) { receipts = ReceiptMemCache() } - return Caches(blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!, - redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache) + return Caches( + blocksByHash!!, blocksByHeight!!, txsByHash!!, receipts!!, + redisBlocksByHash, redisTxsByHash, redisReceiptCache, redisHeightByHashCache + ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesEnabled.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesEnabled.kt index 4b6bd0a5..f0eeb667 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesEnabled.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesEnabled.kt @@ -21,5 +21,4 @@ package io.emeraldpay.dshackle.cache interface CachesEnabled { fun setCaches(caches: Caches) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt index 53718fec..c37b2aca 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt @@ -27,14 +27,13 @@ import io.lettuce.core.codec.StringCodec import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository -import java.util.* +import java.util.EnumMap import javax.annotation.PostConstruct import kotlin.system.exitProcess - @Repository open class CachesFactory( - @Autowired private val cacheConfig: CacheConfig + @Autowired private val cacheConfig: CacheConfig ) { companion object { @@ -50,14 +49,14 @@ open class CachesFactory( val redisConfig = cacheConfig.redis ?: return var uri = RedisURI.builder() - .withHost(redisConfig.host) - .withPort(redisConfig.port) + .withHost(redisConfig.host) + .withPort(redisConfig.port) redisConfig.db?.let { value -> uri = uri.withDatabase(value) } - //log URI _before_ adding a password, to avoid leaking it to the log + // log URI _before_ adding a password, to avoid leaking it to the log log.info("Use Redis cache at: ${uri.build().toURI()}") redisConfig.password?.let { value -> @@ -113,4 +112,4 @@ open class CachesFactory( } return existing } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt index 6b11d8e3..c1bf3b17 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt @@ -20,5 +20,4 @@ class CurrentBlockCache : Reader { fun evict() { cache.set(ConcurrentHashMap()) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashAdding.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashAdding.kt index fce0e617..87c6fdbc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashAdding.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashAdding.kt @@ -29,9 +29,9 @@ import reactor.core.publisher.Mono * */ class HeightByHashAdding( - private val mem: Reader, - private val redis: HeightByHashCache?, - private val upstreamReader: Reader + private val mem: Reader, + private val redis: HeightByHashCache?, + private val upstreamReader: Reader ) : Reader { companion object { @@ -39,7 +39,7 @@ class HeightByHashAdding( } constructor(caches: Caches, upstreamReader: Reader) : - this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader) + this(caches.getLastHeightByHash(), caches.getRedisHeightByHash(), upstreamReader) private val delegate: Reader @@ -48,26 +48,26 @@ class HeightByHashAdding( delegate = object : Reader { override fun read(key: BlockId): Mono { return mem.read(key) - .switchIfEmpty( - Mono.just(key) - .flatMap { redis.read(it) } - ) - .switchIfEmpty( - Mono.just(key) - .flatMap { upstreamReader.read(it) } - .flatMap { redis.add(it).then(Mono.just(it.height)) } - ) + .switchIfEmpty( + Mono.just(key) + .flatMap { redis.read(it) } + ) + .switchIfEmpty( + Mono.just(key) + .flatMap { upstreamReader.read(it) } + .flatMap { redis.add(it).then(Mono.just(it.height)) } + ) } } } else { delegate = object : Reader { override fun read(key: BlockId): Mono { return mem.read(key) - .switchIfEmpty( - Mono.just(key) - .flatMap { upstreamReader.read(it) } - .map { it.height } - ) + .switchIfEmpty( + Mono.just(key) + .flatMap { upstreamReader.read(it) } + .map { it.height } + ) } } } @@ -76,5 +76,4 @@ class HeightByHashAdding( override fun read(key: BlockId): Mono { return delegate.read(key) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashCache.kt index 3dfe08e5..dd9762f3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashCache.kt @@ -23,4 +23,4 @@ import reactor.core.publisher.Mono interface HeightByHashCache : Reader { fun add(block: BlockContainer): Mono -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt index e63ef853..731e6f8e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt @@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Mono class HeightByHashMemCache( - maxSize: Int = 256 + maxSize: Int = 256 ) : Reader { companion object { @@ -31,8 +31,8 @@ class HeightByHashMemCache( } private val heights = Caffeine.newBuilder() - .maximumSize(maxSize.toLong()) - .build() + .maximumSize(maxSize.toLong()) + .build() override fun read(key: BlockId): Mono { return Mono.justOrEmpty(heights.getIfPresent(key)) @@ -41,4 +41,4 @@ class HeightByHashMemCache( fun add(block: BlockContainer) { heights.put(block.hash, block.height) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashRedisCache.kt index 535ef3b9..c9c37dca 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashRedisCache.kt @@ -32,8 +32,8 @@ import java.util.concurrent.TimeUnit * reader would use full block's cache to find out height). */ class HeightByHashRedisCache( - private val redis: RedisReactiveCommands, - private val chain: Chain + private val redis: RedisReactiveCommands, + private val chain: Chain ) : Reader, HeightByHashCache { companion object { @@ -45,41 +45,41 @@ class HeightByHashRedisCache( @Suppress("UNCHECKED_CAST") override fun read(key: BlockId): Mono { return redis.get(key(key)) - .flatMap { data -> - Mono.justOrEmpty(fromBytes(data)) as Mono - }.onErrorResume { - log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}") - Mono.empty() - } + .flatMap { data -> + Mono.justOrEmpty(fromBytes(data)) as Mono + }.onErrorResume { + log.warn("Failed to read Block Height. ${it.javaClass}:${it.message}") + Mono.empty() + } } override fun add(block: BlockContainer): Mono { return Mono.just(block) - .flatMap { blockData -> - // even if block replaced, the mapping hash-long is still valid, so can be cached for long time - // even for fresh blocks - val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES) + .flatMap { blockData -> + // even if block replaced, the mapping hash-long is still valid, so can be cached for long time + // even for fresh blocks + val ttl = TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES) - val key = key(blockData.hash) - val value = asBytes(blockData.height) - redis.setex(key, ttl, value) - } - .doOnError { - log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}") - } - //if failed to cache, just continue without it - .onErrorResume { - Mono.empty() - } - .then() + val key = key(blockData.hash) + val value = asBytes(blockData.height) + redis.setex(key, ttl, value) + } + .doOnError { + log.warn("Failed to save Block Height. ${it.javaClass}:${it.message}") + } + // if failed to cache, just continue without it + .onErrorResume { + Mono.empty() + } + .then() } fun asBytes(value: Long): ByteArray { val result = ByteArray(8) val bb = ByteBuffer.allocate(8) - .order(ByteOrder.BIG_ENDIAN) + .order(ByteOrder.BIG_ENDIAN) bb.asLongBuffer() - .put(value) + .put(value) bb.get(result) return result } @@ -89,9 +89,9 @@ class HeightByHashRedisCache( return null } return ByteBuffer.wrap(value) - .order(ByteOrder.BIG_ENDIAN) - .asLongBuffer() - .get() + .order(ByteOrder.BIG_ENDIAN) + .asLongBuffer() + .get() } /** @@ -100,4 +100,4 @@ class HeightByHashRedisCache( fun key(hash: BlockId): String { return "height:${chain.id}:${hash.toHex()}" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt index 6e8e646a..75e07e61 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/HeightCache.kt @@ -25,12 +25,12 @@ import reactor.core.publisher.Mono * Memory cache for blocks heights, keeps mapping height->hash. */ open class HeightCache( - maxSize: Int = 512 + maxSize: Int = 512 ) : Reader { private val heights = Caffeine.newBuilder() - .maximumSize(maxSize.toLong()) - .build() + .maximumSize(maxSize.toLong()) + .build() override fun read(key: Long): Mono { return Mono.justOrEmpty(heights.getIfPresent(key)) @@ -45,5 +45,4 @@ open class HeightCache( fun purge() { heights.cleanUp() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt index b733fdb7..b4f99c8c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt @@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit import kotlin.math.min abstract class OnBlockRedisCache( - private val redis: RedisReactiveCommands, - private val chain: Chain, - private val valueType: ValueContainer.ValueType + private val redis: RedisReactiveCommands, + private val chain: Chain, + private val valueType: ValueContainer.ValueType ) : Reader { companion object { @@ -51,18 +51,18 @@ abstract class OnBlockRedisCache( fun toProto(block: BlockContainer, value: T): ValueContainer { return ValueContainer.newBuilder() - .setType(valueType) - .setValue(ByteString.copyFrom(serializeValue(value))) - .setBlockMeta(buildMeta(block)) - .build() + .setType(valueType) + .setValue(ByteString.copyFrom(serializeValue(value))) + .setBlockMeta(buildMeta(block)) + .build() } open fun buildMeta(block: BlockContainer): CachesProto.BlockMeta.Builder { return CachesProto.BlockMeta.newBuilder() - .setHash(ByteString.copyFrom(block.hash.value)) - .setHeight(block.height) - .setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray())) - .setTimestamp(block.timestamp.toEpochMilli()) + .setHash(ByteString.copyFrom(block.hash.value)) + .setHeight(block.height) + .setDifficulty(ByteString.copyFrom(block.difficulty.toByteArray())) + .setTimestamp(block.timestamp.toEpochMilli()) } abstract fun serializeValue(value: T): ByteArray @@ -83,7 +83,7 @@ abstract class OnBlockRedisCache( * Key in Redis */ fun key(hash: BlockId): String { - return "${prefix}:${chain.id}:${hash.toHex()}" + return "$prefix:${chain.id}:${hash.toHex()}" } /** @@ -92,51 +92,51 @@ abstract class OnBlockRedisCache( */ open fun add(container: BlockContainer, value: T): Mono { return Mono.just(container) - .flatMap { block -> - val ttl = cachingTime(block.timestamp) - if (ttl > MIN_CACHE_TIME_SECONDS) { - val key = key(block.hash) - val proto = toProto(block, value) - redis.setex(key, ttl, proto.toByteArray()) - } else { - Mono.empty() - } - } - .doOnError { - log.warn("Failed to save Block to Redis: ${it.message}") - } - //if failed to cache, just continue without it - .onErrorResume { + .flatMap { block -> + val ttl = cachingTime(block.timestamp) + if (ttl > MIN_CACHE_TIME_SECONDS) { + val key = key(block.hash) + val proto = toProto(block, value) + redis.setex(key, ttl, proto.toByteArray()) + } else { Mono.empty() } - .then() + } + .doOnError { + log.warn("Failed to save Block to Redis: ${it.message}") + } + // if failed to cache, just continue without it + .onErrorResume { + Mono.empty() + } + .then() } /** * Calculate time to cache the value */ fun cachingTime(blockTime: Instant): Long { - //default caching time is age of the block, i.e. block create hour ago - //keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it - //still can be replaced in the blockchain + // default caching time is age of the block, i.e. block create hour ago + // keep for hour, but block created 10 seconds ago cache only for 10 seconds, because it + // still can be replaced in the blockchain val age = Instant.now().epochSecond - blockTime.epochSecond return min(age, TimeUnit.MINUTES.toSeconds(MAX_CACHE_TIME_MINUTES)) } fun evict(id: BlockId): Mono { return Mono.just(id) - .flatMap { - redis.del(key(it)) - } - .then() + .flatMap { + redis.del(key(it)) + } + .then() } override fun read(key: BlockId): Mono { return redis.get(key(key)) - .map { data -> - fromProto(data) - }.onErrorResume { - Mono.empty() - } + .map { data -> + fromProto(data) + }.onErrorResume { + Mono.empty() + } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt index e8f2a711..912ffc1d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt @@ -30,9 +30,9 @@ import java.util.concurrent.TimeUnit import kotlin.math.min abstract class OnTxRedisCache( - private val redis: RedisReactiveCommands, - private val chain: Chain, - private val valueType: CachesProto.ValueContainer.ValueType + private val redis: RedisReactiveCommands, + private val chain: Chain, + private val valueType: CachesProto.ValueContainer.ValueType ) : Reader { companion object { @@ -56,42 +56,42 @@ abstract class OnTxRedisCache( * Key in Redis */ fun key(hash: TxId): String { - return "${prefix}:${chain.id}:${hash.toHex()}" + return "$prefix:${chain.id}:${hash.toHex()}" } fun evict(container: BlockContainer): Mono { return Mono.just(container) - .map { block -> - block.transactions.map { - key(it) - }.toTypedArray() - }.flatMap { keys -> - redis.del(*keys) - }.then() + .map { block -> + block.transactions.map { + key(it) + }.toTypedArray() + }.flatMap { keys -> + redis.del(*keys) + }.then() } fun evict(id: TxId): Mono { return Mono.just(id) - .flatMap { - redis.del(key(it)) - } - .then() + .flatMap { + redis.del(key(it)) + } + .then() } fun toProto(id: TxId, value: T): ByteArray { val meta = buildMeta(id, value) return CachesProto.ValueContainer.newBuilder() - .setType(valueType) - .setValue(ByteString.copyFrom(serializeValue(value))) - .setTxMeta(meta) - .build() - .toByteArray() + .setType(valueType) + .setValue(ByteString.copyFrom(serializeValue(value))) + .setTxMeta(meta) + .build() + .toByteArray() } open fun buildMeta(id: TxId, value: T): CachesProto.TxMeta.Builder { return CachesProto.TxMeta.newBuilder() - .setHash(ByteString.copyFrom(id.value)) + .setHash(ByteString.copyFrom(id.value)) } abstract fun serializeValue(value: T): ByteArray @@ -110,43 +110,43 @@ abstract class OnTxRedisCache( override fun read(key: TxId): Mono { return redis.get(key(key)) - .map { data -> - fromProto(data) - }.onErrorResume { - Mono.empty() - } + .map { data -> + fromProto(data) + }.onErrorResume { + Mono.empty() + } } open fun add(id: TxId, value: T, block: BlockContainer?, blockHeight: Long?): Mono { return Mono.just(id) - .flatMap { - val key = key(it) - val encodedValue = toProto(it, value) - val ttl = if (block?.timestamp != null) { - cachingTime(block.timestamp) - } else { - cachingTime(blockHeight) - } - //store - redis.setex(key, ttl, encodedValue) + .flatMap { + val key = key(it) + val encodedValue = toProto(it, value) + val ttl = if (block?.timestamp != null) { + cachingTime(block.timestamp) + } else { + cachingTime(blockHeight) } - .doOnError { - log.warn("Failed to save TX to Redis: ${it.message}", it) - } - //if failed to cache, just continue without it - .onErrorResume { - Mono.empty() - } - .then() + // store + redis.setex(key, ttl, encodedValue) + } + .doOnError { + log.warn("Failed to save TX to Redis: ${it.message}", it) + } + // if failed to cache, just continue without it + .onErrorResume { + Mono.empty() + } + .then() } /** * Calculate time to cache the value, based on block time */ fun cachingTime(blockTime: Instant): Long { - //default caching time is age of the block, i.e. block create hour ago - //keep for hour, but block create 10 seconds ago cache for 10 seconds, as it - //still can be replaced in the blockchain + // default caching time is age of the block, i.e. block create hour ago + // keep for hour, but block create 10 seconds ago cache for 10 seconds, as it + // still can be replaced in the blockchain val age = Instant.now().epochSecond - blockTime.epochSecond return min(age, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS)) } @@ -165,4 +165,4 @@ abstract class OnTxRedisCache( } return min(confirmations * BLOCK_TIME_SECONDS, TimeUnit.HOURS.toSeconds(MAX_CACHE_TIME_HOURS)) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptMemCache.kt index 1724683f..eccd2455 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptMemCache.kt @@ -28,8 +28,8 @@ import reactor.core.publisher.Mono * Keeps receipts for recent blocks in memory */ open class ReceiptMemCache( - // how many blocks to keeps in memory - val blocks: Int = 6 + // how many blocks to keeps in memory + val blocks: Int = 6 ) : Reader { companion object { @@ -37,8 +37,8 @@ open class ReceiptMemCache( } private val mapping = Caffeine.newBuilder() - .maximumSize(blocks * 200L) - .build() + .maximumSize(blocks * 200L) + .build() open fun evict(block: BlockContainer) { block.transactions.forEach { @@ -60,5 +60,4 @@ open class ReceiptMemCache( open fun acceptsRecentBlocks(heightDelta: Long): Boolean { return blocks <= heightDelta && heightDelta >= 0 } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptRedisCache.kt index 66297163..3ffad9e7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/ReceiptRedisCache.kt @@ -16,16 +16,15 @@ package io.emeraldpay.dshackle.cache import io.emeraldpay.dshackle.data.DefaultContainer -import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.proto.CachesProto -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson +import io.emeraldpay.grpc.Chain import io.lettuce.core.api.reactive.RedisReactiveCommands import reactor.core.publisher.Mono open class ReceiptRedisCache( - redis: RedisReactiveCommands, - chain: Chain + redis: RedisReactiveCommands, + chain: Chain ) : OnTxRedisCache(redis, chain, CachesProto.ValueContainer.ValueType.TX_RECEIPT) { override fun deserializeValue(value: CachesProto.ValueContainer): ByteArray { @@ -39,4 +38,4 @@ open class ReceiptRedisCache( fun add(json: DefaultContainer): Mono { return super.add(json.txId!!, json.json!!, null, json.height) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt index 5d8bbe08..c13fbdcc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxMemCache.kt @@ -28,8 +28,8 @@ import reactor.core.publisher.Mono * Memory cache for transactions */ open class TxMemCache( - // usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default - private val maxSize: Int = 125 * 32 + // usually there is 100-150 tx per block on Ethereum, we keep data for about 32 blocks by default + private val maxSize: Int = 125 * 32 ) : Reader { companion object { @@ -37,8 +37,8 @@ open class TxMemCache( } private val mapping = Caffeine.newBuilder() - .maximumSize(maxSize.toLong()) - .build() + .maximumSize(maxSize.toLong()) + .build() override fun read(key: TxId): Mono { return Mono.justOrEmpty(mapping.getIfPresent(key)) @@ -52,13 +52,13 @@ open class TxMemCache( open fun evict(block: BlockId) { val ids = mapping.asMap() - .filter { it.value.blockId == block } - .map { it.key } + .filter { it.value.blockId == block } + .map { it.key } mapping.invalidateAll(ids) } open fun add(tx: TxContainer) { - //do not cache fresh transactions + // do not cache fresh transactions if (tx.blockId == null) { return } @@ -68,4 +68,4 @@ open class TxMemCache( open fun purge() { mapping.cleanUp() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt index 1591d4ac..2daae9d1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/TxRedisCache.kt @@ -15,31 +15,26 @@ */ package io.emeraldpay.dshackle.cache -import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.TxContainer import io.emeraldpay.dshackle.data.TxId +import io.emeraldpay.dshackle.proto.CachesProto import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.grpc.Chain -import io.emeraldpay.dshackle.proto.CachesProto import io.lettuce.core.api.reactive.RedisReactiveCommands import org.slf4j.LoggerFactory import reactor.core.publisher.Mono -import reactor.util.function.Tuples -import java.time.Instant -import java.util.concurrent.TimeUnit -import kotlin.math.min /** * Cache transactions in Redis, up to 24 hours. */ open class TxRedisCache( - private val redis: RedisReactiveCommands, - private val chain: Chain + private val redis: RedisReactiveCommands, + private val chain: Chain ) : Reader, - OnTxRedisCache(redis, chain, CachesProto.ValueContainer.ValueType.TX) { + OnTxRedisCache(redis, chain, CachesProto.ValueContainer.ValueType.TX) { companion object { private val log = LoggerFactory.getLogger(TxRedisCache::class.java) @@ -67,15 +62,14 @@ open class TxRedisCache( } val meta = value.txMeta return TxContainer( - meta.height, - TxId(meta.hash.toByteArray()), - BlockId(meta.blockHash.toByteArray()), - value.value.toByteArray() + meta.height, + TxId(meta.hash.toByteArray()), + BlockId(meta.blockHash.toByteArray()), + value.value.toByteArray() ) } open fun add(tx: TxContainer, block: BlockContainer): Mono { return super.add(tx.hash, tx, block, tx.height) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogConfig.kt index 5dde856c..e105a691 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogConfig.kt @@ -1,7 +1,7 @@ package io.emeraldpay.dshackle.config class AccessLogConfig( - val enabled: Boolean = false + val enabled: Boolean = false ) { var filename: String = "./access_log.jsonl" @@ -14,9 +14,8 @@ class AccessLogConfig( fun disabled(): AccessLogConfig { return AccessLogConfig( - enabled = false + enabled = false ) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt index 7174efe9..55422082 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AccessLogReader.kt @@ -23,5 +23,4 @@ class AccessLogReader : YamlConfigReader(), ConfigReader { } } ?: AccessLogConfig.default() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt index 07778d66..8d8c2d4e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt @@ -29,14 +29,14 @@ class AuthConfig { } class ClientBasicAuth( - val username: String, - val password: String + val username: String, + val password: String ) : ClientAuth() class ClientTlsAuth( - var ca: String? = null, - var certificate: String? = null, - var key: String? = null + var ca: String? = null, + var certificate: String? = null, + var key: String? = null ) : ClientAuth() /** @@ -58,4 +58,4 @@ class AuthConfig { var clientRequire: Boolean? = null var clientCa: String? = null } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt index cc36f2ef..18efced6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt @@ -82,5 +82,4 @@ class AuthConfigReader : YamlConfigReader() { auth } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfig.kt index ccbe117c..e6255b4c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfig.kt @@ -17,12 +17,12 @@ package io.emeraldpay.dshackle.config class CacheConfig { - var redis: Redis? = null; + var redis: Redis? = null class Redis( - var host: String = "127.0.0.1", - var port: Int = 6379, - var db: Int? = 0, - var password: String? = null + var host: String = "127.0.0.1", + var port: Int = 6379, + var db: Int? = 0, + var password: String? = null ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt index 40c65d15..aedc2762 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/CacheConfigReader.kt @@ -58,4 +58,4 @@ class CacheConfigReader : YamlConfigReader(), ConfigReader { config } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ConfigReader.kt index 9f38f4c0..4f61d847 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ConfigReader.kt @@ -20,5 +20,4 @@ import org.yaml.snakeyaml.nodes.MappingNode interface ConfigReader { fun read(input: MappingNode?): T? - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/EnvVariables.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/EnvVariables.kt index e3c7e94a..91ab6ecc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/EnvVariables.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/EnvVariables.kt @@ -31,4 +31,4 @@ class EnvVariables { } ?: "" } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/InvalidConfigException.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/InvalidConfigException.kt index 91012ee4..d2278344 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/InvalidConfigException.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/InvalidConfigException.kt @@ -19,11 +19,11 @@ package io.emeraldpay.dshackle.config import org.yaml.snakeyaml.error.Mark open class InvalidConfigException( - message: String + message: String ) : Exception(message) class InvalidConfigYamlException( - filename: String, - mark: Mark, - message: String -) : InvalidConfigException("Invalid YAML configuration ${message}, at ${filename}:${mark.line}") \ No newline at end of file + filename: String, + mark: Mark, + message: String +) : InvalidConfigException("Invalid YAML configuration $message, at $filename:${mark.line}") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index 637fd352..286bff26 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -25,4 +25,4 @@ class MainConfig { var tokens: TokensConfig? = null var monitoring: MonitoringConfig = MonitoringConfig.default() var accessLogConfig: AccessLogConfig = AccessLogConfig.default() -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index bed43f8c..cba114f8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -21,7 +21,7 @@ import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream class MainConfigReader( - fileResolver: FileResolver + fileResolver: FileResolver ) : YamlConfigReader(), ConfigReader { companion object { @@ -73,5 +73,4 @@ class MainConfigReader( } return config } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfig.kt index d232f571..6e3a6114 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfig.kt @@ -16,14 +16,15 @@ package io.emeraldpay.dshackle.config class MonitoringConfig( - val enabled: Boolean, - val prometheus: PrometheusConfig + val enabled: Boolean, + val prometheus: PrometheusConfig ) { companion object { fun default(): MonitoringConfig { return MonitoringConfig(true, PrometheusConfig.default()) } + fun disabled(): MonitoringConfig { return MonitoringConfig(false, PrometheusConfig.disabled()) } @@ -33,19 +34,19 @@ class MonitoringConfig( var enableExtended: Boolean = false data class PrometheusConfig( - val enabled: Boolean, - val path: String, - val host: String, - val port: Int + val enabled: Boolean, + val path: String, + val host: String, + val port: Int ) { companion object { fun default(): PrometheusConfig { return PrometheusConfig(true, "/metrics", "127.0.0.1", 8081) } + fun disabled(): PrometheusConfig { return PrometheusConfig(false, "/", "127.0.0.1", 0) } } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt index b8f4e319..db0a84f1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MonitoringConfigReader.kt @@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream -class MonitoringConfigReader: YamlConfigReader(), ConfigReader { +class MonitoringConfigReader : YamlConfigReader(), ConfigReader { companion object { private val log = LoggerFactory.getLogger(MonitoringConfigReader::class.java) @@ -64,5 +64,4 @@ class MonitoringConfigReader: YamlConfigReader(), ConfigReader val port = getValueAsInt(input, "port") ?: default.port return MonitoringConfig.PrometheusConfig(enabled, path, host, port) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfig.kt index 0b6fd473..2c53e5cb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfig.kt @@ -24,7 +24,7 @@ import io.emeraldpay.grpc.Chain open class ProxyConfig { companion object { - public const val CONFIG_ID = "parsed.proxy" + const val CONFIG_ID = "parsed.proxy" } var enabled: Boolean = true @@ -50,13 +50,13 @@ open class ProxyConfig { var routes: List = ArrayList() class Route( - /** - * URL binding for the route. http://$host:$port/$id - */ - val id: String, - /** - * Blockchain to dispatch requests - */ - val blockchain: Chain + /** + * URL binding for the route. http://$host:$port/$id + */ + val id: String, + /** + * Blockchain to dispatch requests + */ + val blockchain: Chain ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt index bc62f069..4fcc9cae 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/ProxyConfigReader.kt @@ -19,10 +19,8 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.grpc.Chain import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory -import org.yaml.snakeyaml.Yaml import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream -import java.io.InputStreamReader /** * Read YAML config, part related to Proxy configuration @@ -84,5 +82,4 @@ class ProxyConfigReader : YamlConfigReader(), ConfigReader { config.tls = authConfigReader.readServerTls(input) return config } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt index f62128ac..56debf05 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt @@ -15,12 +15,12 @@ */ package io.emeraldpay.dshackle.config +import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain -import io.emeraldpay.etherjar.domain.Address class TokensConfig( - val tokens: List + val tokens: List ) { class Token { @@ -30,7 +30,7 @@ class TokensConfig( // coin name var name: String? = null - var type: Type? = null; + var type: Type? = null var address: String? = null fun validate(): String? { @@ -40,9 +40,9 @@ class TokensConfig( name.isNullOrBlank() -> "name" type == null -> type address.isNullOrBlank() -> "address" - blockchain != null - && BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM - && !Address.isValidAddress(address) -> "address" + blockchain != null && + BlockchainType.from(blockchain!!) == BlockchainType.ETHEREUM && + !Address.isValidAddress(address) -> "address" else -> null } } @@ -51,5 +51,4 @@ class TokensConfig( enum class Type { ERC20 } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt index 169f7ffe..9d4f6da3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfigReader.kt @@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.config import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode import java.io.InputStream -import java.util.* +import java.util.Locale class TokensConfigReader : YamlConfigReader(), ConfigReader { @@ -60,5 +60,4 @@ class TokensConfigReader : YamlConfigReader(), ConfigReader { TokensConfig(it) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt index 3567fe18..ec44ed86 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt @@ -17,11 +17,9 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.Defaults -import java.lang.ClassCastException import java.net.URI -import java.util.* -import kotlin.collections.ArrayList -import kotlin.collections.HashMap +import java.util.Arrays +import java.util.Locale open class UpstreamsConfig { var defaultOptions: MutableList = ArrayList() @@ -46,8 +44,10 @@ open class UpstreamsConfig { } val copy = Options() copy.minPeers = if (this.minPeers != null) this.minPeers else additional.minPeers - copy.disableValidation = if (this.disableValidation != null) this.disableValidation else additional.disableValidation - copy.providesBalance = if (this.providesBalance != null) this.providesBalance else additional.providesBalance + copy.disableValidation = + if (this.disableValidation != null) this.disableValidation else additional.disableValidation + copy.providesBalance = + if (this.providesBalance != null) this.providesBalance else additional.providesBalance return copy } @@ -60,7 +60,6 @@ open class UpstreamsConfig { return options } } - } class DefaultOptions : Options() { @@ -125,14 +124,14 @@ open class UpstreamsConfig { var msgSize: Int? = null } - - //TODO make it unmodifiable after initial load - class Labels: HashMap() { + // TODO make it unmodifiable after initial load + class Labels : HashMap() { companion object { - @JvmStatic fun fromMap(map: Map): Labels { + @JvmStatic + fun fromMap(map: Map): Labels { val labels = Labels() - map.entries.forEach() { kv -> + map.entries.forEach { kv -> labels.put(kv.key, kv.value) } return labels @@ -140,7 +139,7 @@ open class UpstreamsConfig { } } - enum class UpstreamType private constructor(vararg code: String) { + enum class UpstreamType(vararg code: String) { ETHEREUM_JSON_RPC("ethereum"), BITCOIN_JSON_RPC("bitcoin"), DSHACKLE("dshackle", "grpc"), @@ -168,12 +167,12 @@ open class UpstreamsConfig { } class Methods( - val enabled: Set, - val disabled: Set + val enabled: Set, + val disabled: Set ) class Method( - val name: String, - val quorum: String? = null + val name: String, + val quorum: String? = null ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 9f6e7cbe..b91957b8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -23,14 +23,12 @@ import org.yaml.snakeyaml.nodes.MappingNode import org.yaml.snakeyaml.nodes.ScalarNode import reactor.util.function.Tuples import java.io.InputStream -import java.lang.IllegalArgumentException import java.net.URI import java.time.Duration -import java.util.* -import kotlin.collections.ArrayList +import java.util.Locale class UpstreamsConfigReader( - private val fileResolver: FileResolver + private val fileResolver: FileResolver ) : YamlConfigReader(), ConfigReader { private val log = LoggerFactory.getLogger(UpstreamsConfigReader::class.java) @@ -198,7 +196,10 @@ class UpstreamsConfigReader( upstream.methods = tryReadMethods(upNode) } - internal fun readUpstreamGrpc(upNode: MappingNode, upstream: UpstreamsConfig.Upstream) { + internal fun readUpstreamGrpc( + upNode: MappingNode, + upstream: UpstreamsConfig.Upstream + ) { if (hasAny(upNode, "labels")) { log.warn("Labels should be not applied to gRPC upstream") } @@ -221,13 +222,13 @@ class UpstreamsConfigReader( if (hasAny(upNode, "labels")) { getMapping(upNode, "labels")?.let { labels -> labels.value.stream() - .filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode } - .map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) } - .map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) } - .filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) } - .forEach { kv -> - upstream.labels[kv.t1] = kv.t2 - } + .filter { n -> n.keyNode is ScalarNode && n.valueNode is ScalarNode } + .map { n -> Tuples.of((n.keyNode as ScalarNode).value, (n.valueNode as ScalarNode).value) } + .map { kv -> Tuples.of(kv.t1.trim(), kv.t2.trim()) } + .filter { kv -> StringUtils.isNotEmpty(kv.t1) && StringUtils.isNotEmpty(kv.t2) } + .forEach { kv -> + upstream.labels[kv.t1] = kv.t2 + } } } } @@ -247,21 +248,21 @@ class UpstreamsConfigReader( val enabled = getList(mnode, "enabled")?.value?.map { m -> getValueAsString(m, "name")?.let { name -> UpstreamsConfig.Method( - name = name, - quorum = getValueAsString(m, "quorum") + name = name, + quorum = getValueAsString(m, "quorum") ) } }?.filterNotNull()?.toSet() ?: emptySet() val disabled = getList(mnode, "disabled")?.value?.map { m -> getValueAsString(m, "name")?.let { name -> UpstreamsConfig.Method( - name = name + name = name ) } }?.filterNotNull()?.toSet() ?: emptySet() UpstreamsConfig.Methods( - enabled, disabled + enabled, disabled ) } } @@ -282,5 +283,4 @@ class UpstreamsConfigReader( } return options } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt index ea508a74..414e7322 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/YamlConfigReader.kt @@ -24,7 +24,7 @@ import org.yaml.snakeyaml.nodes.Node import org.yaml.snakeyaml.nodes.ScalarNode import java.io.InputStream import java.io.InputStreamReader -import java.util.* +import java.util.Locale abstract class YamlConfigReader { private val envVariables = EnvVariables() @@ -43,12 +43,12 @@ abstract class YamlConfigReader { return false } return mappingNode.value - .stream() - .filter { n -> n.keyNode is ScalarNode } - .filter { n -> - val sn = n.keyNode as ScalarNode - key == sn.value - }.count() > 0 + .stream() + .filter { n -> n.keyNode is ScalarNode } + .filter { n -> + val sn = n.keyNode as ScalarNode + key == sn.value + }.count() > 0 } @Suppress("UNCHECKED_CAST") @@ -57,20 +57,20 @@ abstract class YamlConfigReader { return null } return mappingNode.value - .stream() - .filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) } - .filter { n -> - val sn = n.keyNode as ScalarNode - key == sn.value - } - .map { n -> n.valueNode as T } - .findFirst().let { - if (it.isPresent) { - it.get() - } else { - null - } + .stream() + .filter { n -> n.keyNode is ScalarNode && type.isAssignableFrom(n.valueNode.javaClass) } + .filter { n -> + val sn = n.keyNode as ScalarNode + key == sn.value + } + .map { n -> n.valueNode as T } + .findFirst().let { + if (it.isPresent) { + it.get() + } else { + null } + } } protected fun getMapping(mappingNode: MappingNode?, key: String): MappingNode? { @@ -89,8 +89,8 @@ abstract class YamlConfigReader { protected fun getListOfString(mappingNode: MappingNode?, key: String): List? { return getList(mappingNode, key)?.value - ?.map { it.value } - ?.map(envVariables::postProcess) + ?.map { it.value } + ?.map(envVariables::postProcess) } protected fun getValueAsString(mappingNode: MappingNode?, key: String): String? { @@ -130,7 +130,7 @@ abstract class YamlConfigReader { fun getValueAsBytes(mappingNode: MappingNode?, key: String): Int? { return getValueAsString(mappingNode, key)?.let(envVariables::postProcess)?.let { val m = Regex("^(\\d+)(m|mb|k|kb|b)?$").find(it.lowercase().trim()) - ?: throw IllegalArgumentException("Not a data size: ${it}. Example of correct values: '1024', '1kb', '5mb'") + ?: throw IllegalArgumentException("Not a data size: $it. Example of correct values: '1024', '1kb', '5mb'") val multiplier = m.groups[2]?.let { when (it.value) { "k", "kb" -> 1024 @@ -147,9 +147,9 @@ abstract class YamlConfigReader { fun getBlockchain(id: String): Chain { return Chain.values().find { chain -> - chain.name == id.uppercase(Locale.getDefault()) - || chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) - || chain.id.toString() == id + chain.name == id.uppercase(Locale.getDefault()) || + chain.chainCode.uppercase(Locale.getDefault()) == id.uppercase(Locale.getDefault()) || + chain.id.toString() == id } ?: Chain.UNSPECIFIED } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt index 04a333a5..f5c47d3f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockContainer.kt @@ -23,14 +23,14 @@ import java.math.BigInteger import java.time.Instant class BlockContainer( - val height: Long, - val hash: BlockId, - val difficulty: BigInteger, - val timestamp: Instant, - val full: Boolean, - json: ByteArray?, - val parsed: Any?, - val transactions: List = emptyList() + val height: Long, + val hash: BlockId, + val difficulty: BigInteger, + val timestamp: Instant, + val full: Boolean, + json: ByteArray?, + val parsed: Any?, + val transactions: List = emptyList() ) : SourceContainer(json, parsed) { companion object { @@ -38,14 +38,14 @@ class BlockContainer( fun from(block: BlockJson<*>, raw: ByteArray): BlockContainer { val hasTransactions = block.transactions?.filterIsInstance()?.count() ?: 0 > 0 return BlockContainer( - block.number, - BlockId.from(block), - block.totalDifficulty, - block.timestamp, - hasTransactions, - raw, - block, - block.transactions?.map { TxId.from(it.hash) } ?: emptyList() + block.number, + BlockId.from(block), + block.totalDifficulty, + block.timestamp, + hasTransactions, + raw, + block, + block.transactions?.map { TxId.from(it.hash) } ?: emptyList() ) } @@ -88,6 +88,4 @@ class BlockContainer( result = 31 * result + hash.hashCode() return result } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt index 914294c4..d4f1a0ab 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt @@ -21,7 +21,7 @@ import io.emeraldpay.etherjar.rpc.json.BlockJson import org.bouncycastle.util.encoders.Hex class BlockId( - value: ByteArray + value: ByteArray ) : HashId(value) { companion object { @@ -51,6 +51,4 @@ class BlockId( return BlockId(bytes) } } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/DefaultContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/DefaultContainer.kt index ffc82389..492ece1b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/DefaultContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/DefaultContainer.kt @@ -18,14 +18,14 @@ package io.emeraldpay.dshackle.data import org.slf4j.LoggerFactory class DefaultContainer( - val txId: TxId?, - val blockId: BlockId?, - val height: Long?, - json: ByteArray, - parsed: T + val txId: TxId?, + val blockId: BlockId?, + val height: Long?, + json: ByteArray, + parsed: T ) : SourceContainer(json, parsed) { companion object { private val log = LoggerFactory.getLogger(DefaultContainer::class.java) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt index affe473c..7355b511 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/HashId.kt @@ -17,7 +17,7 @@ package io.emeraldpay.dshackle.data open class HashId( - val value: ByteArray + val value: ByteArray ) { companion object { @@ -56,6 +56,4 @@ open class HashId( override fun hashCode(): Int { return value.contentHashCode() } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt index b31832a0..8b8d14c8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/RawJsonBuilder.kt @@ -36,7 +36,7 @@ class RawJsonBuilder { buf.write(START) buf.write(COMMA) buf.write(ID_START) - buf.write(id.toString().toByteArray()); + buf.write(id.toString().toByteArray()) buf.write(COMMA) buf.write(RESULT_START) buf.write(data) @@ -44,6 +44,4 @@ class RawJsonBuilder { return buf.toByteArray() } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt index eeb27ad4..6e42afdd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/SourceContainer.kt @@ -16,11 +16,9 @@ */ package io.emeraldpay.dshackle.data -import java.lang.ClassCastException - abstract class SourceContainer( - val json: ByteArray?, - private val parsed: Any? + val json: ByteArray?, + private val parsed: Any? ) { @Suppress("UNCHECKED_CAST") @@ -34,7 +32,6 @@ abstract class SourceContainer( throw ClassCastException("Cannot cast ${parsed.javaClass} to $clazz") } - override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is SourceContainer) return false @@ -50,4 +47,4 @@ abstract class SourceContainer( override fun hashCode(): Int { return json?.contentHashCode() ?: 0 } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt index 2e38db52..a99ef8d8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/TxContainer.kt @@ -20,11 +20,11 @@ import io.emeraldpay.dshackle.Global import io.emeraldpay.etherjar.rpc.json.TransactionJson class TxContainer( - val height: Long?, - val hash: TxId, - val blockId: BlockId?, - json: ByteArray?, - parsed: Any? = null + val height: Long?, + val hash: TxId, + val blockId: BlockId?, + json: ByteArray?, + parsed: Any? = null ) : SourceContainer(json, parsed) { companion object { @@ -41,11 +41,11 @@ class TxContainer( fun from(tx: TransactionJson, raw: ByteArray): TxContainer { return TxContainer( - tx.blockNumber, - TxId.from(tx.hash), - tx.blockHash?.let { BlockId.from(it) }, - raw, - tx + tx.blockNumber, + TxId.from(tx.hash), + tx.blockHash?.let { BlockId.from(it) }, + raw, + tx ) } } @@ -70,6 +70,4 @@ class TxContainer( result = 31 * result + hash.hashCode() return result } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt b/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt index 4eafd661..ce8a6406 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/data/TxId.kt @@ -19,10 +19,9 @@ package io.emeraldpay.dshackle.data import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.rpc.json.TransactionJson import org.bouncycastle.util.encoders.Hex -import java.math.BigInteger class TxId( - value: ByteArray + value: ByteArray ) : HashId(value) { companion object { @@ -47,4 +46,4 @@ class TxId( return TxId(bytes) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt index 3b8fd1f7..b407d687 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt @@ -26,20 +26,18 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics import io.micrometer.core.instrument.binder.system.ProcessorMetrics import io.micrometer.core.instrument.config.MeterFilter -import org.slf4j.LoggerFactory -import org.springframework.stereotype.Service import io.micrometer.prometheus.PrometheusConfig - import io.micrometer.prometheus.PrometheusMeterRegistry +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service import java.io.IOException import java.net.InetSocketAddress import javax.annotation.PostConstruct - @Service class MonitoringSetup( - @Autowired private val monitoringConfig: MonitoringConfig + @Autowired private val monitoringConfig: MonitoringConfig ) { companion object { @@ -50,7 +48,7 @@ class MonitoringSetup( fun setup() { val prometheusRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT) Metrics.globalRegistry.add(prometheusRegistry) - Metrics.globalRegistry.config().meterFilter(object: MeterFilter { + Metrics.globalRegistry.config().meterFilter(object : MeterFilter { override fun map(id: Meter.Id): Meter.Id { if (id.name.startsWith("jvm") || id.name.startsWith("process") || id.name.startsWith("system")) { return id @@ -76,18 +74,24 @@ class MonitoringSetup( // prometheus is a single thread periodic call, no reason to setup anything complex try { log.info("Run Prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}") - val server = HttpServer.create(InetSocketAddress(monitoringConfig.prometheus.host, monitoringConfig.prometheus.port), 0); + val server = HttpServer.create( + InetSocketAddress( + monitoringConfig.prometheus.host, + monitoringConfig.prometheus.port + ), + 0 + ) server.createContext(monitoringConfig.prometheus.path) { httpExchange -> val response = prometheusRegistry.scrape() - httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong()); + httpExchange.sendResponseHeaders(200, response.toByteArray().size.toLong()) httpExchange.responseBody.use { os -> os.write(response.toByteArray()) } } - Thread(server::start).start(); + Thread(server::start).start() } catch (e: IOException) { log.error("Failed to start Prometheus Server", e) } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt index ea5e0c16..7e3c849f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt @@ -15,14 +15,20 @@ */ package io.emeraldpay.dshackle.monitoring.accesslog -import io.grpc.* +import io.grpc.ForwardingServerCall +import io.grpc.ForwardingServerCallListener +import io.grpc.Metadata +import io.grpc.MethodDescriptor +import io.grpc.ServerCall +import io.grpc.ServerCallHandler +import io.grpc.ServerInterceptor import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @Service class AccessHandlerGrpc( - @Autowired private val accessLogWriter: AccessLogWriter + @Autowired private val accessLogWriter: AccessLogWriter ) : ServerInterceptor { companion object { @@ -30,9 +36,10 @@ class AccessHandlerGrpc( } override fun interceptCall( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler): ServerCall.Listener { + call: ServerCall, + headers: Metadata, + next: ServerCallHandler + ): ServerCall.Listener { return when (val method = call.methodDescriptor.bareMethodName) { "SubscribeHead" -> processSubscribeHead(call, headers, next) @@ -51,103 +58,109 @@ class AccessHandlerGrpc( } private fun process( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler, - builder: EventsBuilder.RequestReply + call: ServerCall, + headers: Metadata, + next: ServerCallHandler, + builder: EventsBuilder.RequestReply ): ServerCall.Listener { builder.start(headers, call.attributes) val callWrapper: ServerCall = StdCallResponse( - call, builder, accessLogWriter + call, builder, accessLogWriter ) return StdCallListener( - next.startCall(callWrapper, headers), - builder + next.startCall(callWrapper, headers), + builder ) } @Suppress("UNCHECKED_CAST") private fun processSubscribeHead( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler + call: ServerCall, + headers: Metadata, + next: ServerCallHandler ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.SubscribeHead() as EventsBuilder.RequestReply<*, ReqT, RespT> ) } @Suppress("UNCHECKED_CAST") private fun processSubscribeBalance( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler, - subscribe: Boolean + call: ServerCall, + headers: Metadata, + next: ServerCallHandler, + subscribe: Boolean ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.SubscribeBalance(subscribe) as EventsBuilder.RequestReply<*, ReqT, RespT> ) } @Suppress("UNCHECKED_CAST") private fun processSubscribeTxStatus( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler + call: ServerCall, + headers: Metadata, + next: ServerCallHandler ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.TxStatus() as EventsBuilder.RequestReply<*, ReqT, RespT> ) } @Suppress("UNCHECKED_CAST") private fun processNativeCall( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler + call: ServerCall, + headers: Metadata, + next: ServerCallHandler ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.NativeCall() as EventsBuilder.RequestReply<*, ReqT, RespT> ) } @Suppress("UNCHECKED_CAST") private fun processNativeSubscribe( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler + call: ServerCall, + headers: Metadata, + next: ServerCallHandler ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.NativeSubscribe() as EventsBuilder.RequestReply<*, ReqT, RespT> ) } - @Suppress("UNCHECKED_CAST") private fun processDescribe( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler + call: ServerCall, + headers: Metadata, + next: ServerCallHandler ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.Describe() as EventsBuilder.RequestReply<*, ReqT, RespT> ) } @Suppress("UNCHECKED_CAST") private fun processStatus( - call: ServerCall, - headers: Metadata, - next: ServerCallHandler + call: ServerCall, + headers: Metadata, + next: ServerCallHandler ): ServerCall.Listener { - return process(call, headers, next, - EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT> + return process( + call, headers, next, + EventsBuilder.Status() as EventsBuilder.RequestReply<*, ReqT, RespT> ) } open class StdCallListener>( - val next: ServerCall.Listener, - val builder: EB + val next: ServerCall.Listener, + val builder: EB ) : ForwardingServerCallListener() { override fun onMessage(message: Req) { @@ -161,9 +174,9 @@ class AccessHandlerGrpc( } open class StdCallResponse>( - val next: ServerCall, - val builder: EB, - val accessLogWriter: AccessLogWriter + val next: ServerCall, + val builder: EB, + val accessLogWriter: AccessLogWriter ) : ForwardingServerCall() { override fun getMethodDescriptor(): MethodDescriptor { @@ -177,9 +190,8 @@ class AccessHandlerGrpc( override fun sendMessage(message: RespT) { super.sendMessage(message) accessLogWriter.submit( - builder.onReply(message)!! + builder.onReply(message)!! ) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt index 68daa1b4..eb468ddf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt @@ -19,8 +19,8 @@ import kotlin.concurrent.withLock */ @Service class AccessHandlerHttp( - @Autowired private val mainConfig: MainConfig, - @Autowired accessLogWriter: AccessLogWriter + @Autowired private val mainConfig: MainConfig, + @Autowired accessLogWriter: AccessLogWriter ) { companion object { @@ -40,7 +40,7 @@ class AccessHandlerHttp( fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler } - class NoOpFactory() : HandlerFactory { + class NoOpFactory : HandlerFactory { override fun create(req: HttpServerRequest, blockchain: Chain): RequestHandler { return NoOpHandler() } @@ -70,9 +70,9 @@ class AccessHandlerHttp( } class StandardHandler( - private val accessLogWriter: AccessLogWriter, - private val httpRequest: HttpServerRequest, - private val blockchain: Chain + private val accessLogWriter: AccessLogWriter, + private val httpRequest: HttpServerRequest, + private val blockchain: Chain ) : RequestHandler { private var request: BlockchainOuterClass.NativeCallRequest? = null @@ -89,13 +89,13 @@ class AccessHandlerHttp( builder.start(httpRequest) builder.onRequest(request!!) responses - .map { - builder.onReply(it, Events.Channel.JSONRPC).also { item -> - //since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp - item.ts = responseTime - } + .map { + builder.onReply(it, Events.Channel.JSONRPC).also { item -> + // since for JSON RPC you get a single response then the timestamp of all items included in it must have the same timestamp + item.ts = responseTime } - .let(accessLogWriter::submit) + } + .let(accessLogWriter::submit) } override fun onRequest(request: BlockchainOuterClass.NativeCallRequest) { @@ -108,4 +108,4 @@ class AccessHandlerHttp( } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt index 64ef6017..dd6e022f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt @@ -20,7 +20,9 @@ import io.emeraldpay.dshackle.config.MainConfig import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository -import java.io.* +import java.io.BufferedOutputStream +import java.io.File +import java.io.FileOutputStream import java.time.Duration import java.time.Instant import java.util.concurrent.ConcurrentLinkedQueue @@ -30,7 +32,7 @@ import javax.annotation.PostConstruct @Repository class AccessLogWriter( - @Autowired mainConfig: MainConfig + @Autowired mainConfig: MainConfig ) { companion object { @@ -120,5 +122,4 @@ class AccessLogWriter( } } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/Events.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/Events.kt index 427d1d9e..793fb820 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/Events.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/Events.kt @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import java.time.Instant -import java.util.* +import java.util.UUID class Events { @@ -32,140 +32,152 @@ class Events { } abstract class Base( - val id: UUID, - val method: String, - val channel: Channel + val id: UUID, + val method: String, + val channel: Channel ) { val version = "accesslog/v1beta" var ts = Instant.now() } abstract class ChainBase( - val blockchain: Chain, method: String, id: UUID, channel: Channel + val blockchain: Chain, + method: String, + id: UUID, + channel: Channel ) : Base(id, method, channel) @JsonInclude(JsonInclude.Include.NON_NULL) class SubscribeHead( - blockchain: Chain, id: UUID, - // initial request details - val request: StreamRequestDetails, - // index of the current response - val index: Int + blockchain: Chain, + id: UUID, + // initial request details + val request: StreamRequestDetails, + // index of the current response + val index: Int ) : ChainBase(blockchain, "SubscribeHead", id, Channel.GRPC) @JsonInclude(JsonInclude.Include.NON_NULL) class SubscribeBalance( - blockchain: Chain, id: UUID, subscribe: Boolean, - // initial request details - val request: StreamRequestDetails, - val balanceRequest: BalanceRequest, - val addressBalance: AddressBalance, - // index of the current response - val index: Int + blockchain: Chain, + id: UUID, + subscribe: Boolean, + // initial request details + val request: StreamRequestDetails, + val balanceRequest: BalanceRequest, + val addressBalance: AddressBalance, + // index of the current response + val index: Int ) : ChainBase(blockchain, if (subscribe) "SubscribeBalance" else "GetBalance", id, Channel.GRPC) @JsonInclude(JsonInclude.Include.NON_NULL) class TxStatus( - blockchain: Chain, id: UUID, - val request: StreamRequestDetails, - val txStatusRequest: TxStatusRequest, - val txStatus: TxStatusResponse, - // index of the current response - val index: Int + blockchain: Chain, + id: UUID, + val request: StreamRequestDetails, + val txStatusRequest: TxStatusRequest, + val txStatus: TxStatusResponse, + // index of the current response + val index: Int ) : ChainBase(blockchain, "SubscribeTxStatus", id, Channel.GRPC) data class TxStatusRequest( - val txId: String + val txId: String ) data class TxStatusResponse( - val confirmations: Int + val confirmations: Int ) @JsonInclude(JsonInclude.Include.NON_NULL) class NativeCall( - blockchain: Chain, id: UUID, channel: Channel, + blockchain: Chain, + id: UUID, + channel: Channel, - // info about the initial request, that may include several native calls - val request: StreamRequestDetails, - // total native calls passes within the initial request - val total: Int, - // index of the call specific for the current response - val index: Int, - val selector: String? = null, - val quorum: Long? = null, - val minAvailability: String? = null, + // info about the initial request, that may include several native calls + val request: StreamRequestDetails, + // total native calls passes within the initial request + val total: Int, + // index of the call specific for the current response + val index: Int, + val selector: String? = null, + val quorum: Long? = null, + val minAvailability: String? = null, - val succeed: Boolean, - val rpcError: Int? = null, - val payloadSizeBytes: Long, - val nativeCall: NativeCallItemDetails + val succeed: Boolean, + val rpcError: Int? = null, + val payloadSizeBytes: Long, + val nativeCall: NativeCallItemDetails ) : ChainBase(blockchain, "NativeCall", id, channel) @JsonInclude(JsonInclude.Include.NON_NULL) class NativeSubscribe( - blockchain: Chain, id: UUID, channel: Channel, + blockchain: Chain, + id: UUID, + channel: Channel, - // info about the initial request, that may include several native calls - val request: StreamRequestDetails, - val payloadSizeBytes: Long, - val nativeSubscribe: NativeSubscribeItemDetails + // info about the initial request, that may include several native calls + val request: StreamRequestDetails, + val payloadSizeBytes: Long, + val nativeSubscribe: NativeSubscribeItemDetails ) : ChainBase(blockchain, "NativeSubscribe", id, channel) @JsonInclude(JsonInclude.Include.NON_NULL) class Describe( - id: UUID, - val request: StreamRequestDetails + id: UUID, + val request: StreamRequestDetails ) : Base(id, "Describe", Channel.GRPC) @JsonInclude(JsonInclude.Include.NON_NULL) class Status( - blockchain: Chain, id: UUID, - val request: StreamRequestDetails + blockchain: Chain, + id: UUID, + val request: StreamRequestDetails ) : ChainBase(blockchain, "Status", id, Channel.GRPC) data class StreamRequestDetails( - val id: UUID, - val start: Instant, - val remote: Remote + val id: UUID, + val start: Instant, + val remote: Remote ) data class Remote( - val ips: List, - val ip: String, - val userAgent: String + val ips: List, + val ip: String, + val userAgent: String ) data class NativeCallItemDetails( - val method: String, - val id: Int, - val payloadSizeBytes: Long + val method: String, + val id: Int, + val payloadSizeBytes: Long ) data class NativeCallReplyDetails( - val id: Int, - val succeed: Boolean, - val replySizeBytes: Long, - val ts: Instant = Instant.now() + val id: Int, + val succeed: Boolean, + val replySizeBytes: Long, + val ts: Instant = Instant.now() ) data class NativeSubscribeItemDetails( - val method: String, - val payloadSizeBytes: Long + val method: String, + val payloadSizeBytes: Long ) data class NativeSubscribeReplyDetails( - val replySizeBytes: Long, - val ts: Instant = Instant.now() + val replySizeBytes: Long, + val ts: Instant = Instant.now() ) data class BalanceRequest( - val asset: String, - val addressType: String + val asset: String, + val addressType: String ) data class AddressBalance( - val asset: String, - val address: String + val asset: String, + val address: String ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/EventsBuilder.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/EventsBuilder.kt index b0e6d882..ed2a5de7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/EventsBuilder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/EventsBuilder.kt @@ -27,7 +27,8 @@ import reactor.netty.http.server.HttpServerRequest import java.net.InetAddress import java.net.InetSocketAddress import java.time.Instant -import java.util.* +import java.util.Locale +import java.util.UUID class EventsBuilder { @@ -48,23 +49,23 @@ class EventsBuilder { fun onReply(msg: Resp): E } - abstract class Base() : StartingHttp2Request, StartingHttp1Request { + abstract class Base : StartingHttp2Request, StartingHttp1Request { companion object { private val remoteIpHeaders = listOf( - "x-real-ip", - "x-forwarded-for" + "x-real-ip", + "x-forwarded-for" ) private val remoteIpKeys = listOf( - Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER), - Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER) + Metadata.Key.of("x-real-ip", Metadata.ASCII_STRING_MARSHALLER), + Metadata.Key.of("x-forwarded-for", Metadata.ASCII_STRING_MARSHALLER) ) private val invalidCharacters = Regex("[\n\t]+") } var requestDetails = Events.StreamRequestDetails( - UUID.randomUUID(), - Instant.now(), - Events.Remote(emptyList(), "", "") + UUID.randomUUID(), + Instant.now(), + Events.Remote(emptyList(), "", "") ) var chainId: Int = Chain.UNSPECIFIED.id @@ -84,35 +85,37 @@ class EventsBuilder { private fun findBestIp(ips: List): InetAddress? { // check if a real remote address is provided, otherwise use any local address - return ips.sortedWith(kotlin.Comparator { a, b -> - val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress - val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress - when { - aLocal && bLocal -> 0 - aLocal -> 1 - else -> -1 + return ips.sortedWith( + kotlin.Comparator { a, b -> + val aLocal = a.isLoopbackAddress || a.isSiteLocalAddress + val bLocal = b.isLoopbackAddress || b.isSiteLocalAddress + when { + aLocal && bLocal -> 0 + aLocal -> 1 + else -> -1 + } } - }).firstOrNull() + ).firstOrNull() } private fun clean(s: String): String { return StringUtils.truncate(s, 128) - .replace(invalidCharacters, " ") - .trim() + .replace(invalidCharacters, " ") + .trim() } protected abstract fun getT(): T override fun start(metadata: Metadata, attributes: Attributes) { val userAgent = metadata.get(Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER)) - ?.let(this@Base::clean) - ?: "" + ?.let(this@Base::clean) + ?: "" val ips = ArrayList() remoteIpKeys.forEach { key -> metadata.get(key)?.let { it.trim().ifEmpty { null } - ?.let(this@Base::toInetAddress) - ?.let(ips::add) + ?.let(this@Base::toInetAddress) + ?.let(ips::add) } } attributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)?.let { addr -> @@ -122,24 +125,26 @@ class EventsBuilder { } val ip = findBestIp(ips)?.hostAddress ?: "" this.requestDetails = this.requestDetails - .copy(remote = Events.Remote( - ips = ips.map { it.hostAddress }, - ip = ip, - userAgent = userAgent - )) + .copy( + remote = Events.Remote( + ips = ips.map { it.hostAddress }, + ip = ip, + userAgent = userAgent + ) + ) } override fun start(request: HttpServerRequest) { val headers = request.requestHeaders() val userAgent = headers.get("user-agent") - ?.let(this@Base::clean) - ?: "" + ?.let(this@Base::clean) + ?: "" val ips = ArrayList() remoteIpHeaders.forEach { key -> headers.get(key)?.let { it.trim().ifEmpty { null } - ?.let(this@Base::toInetAddress) - ?.let(ips::add) + ?.let(this@Base::toInetAddress) + ?.let(ips::add) } } request.remoteAddress()?.let { addr -> @@ -147,11 +152,13 @@ class EventsBuilder { } val ip = findBestIp(ips)?.hostAddress ?: "" this.requestDetails = this.requestDetails - .copy(remote = Events.Remote( - ips = ips.map { it.hostAddress }, - ip = ip, - userAgent = userAgent - )) + .copy( + remote = Events.Remote( + ips = ips.map { it.hostAddress }, + ip = ip, + userAgent = userAgent + ) + ) } fun withChain(chain: Int): T { @@ -161,9 +168,9 @@ class EventsBuilder { } } - class SubscribeHead() : - Base(), - RequestReply { + class SubscribeHead : + Base(), + RequestReply { private var index = 0 @@ -177,14 +184,14 @@ class EventsBuilder { override fun onReply(msg: BlockchainOuterClass.ChainHead): Events.SubscribeHead { return Events.SubscribeHead( - chain, UUID.randomUUID(), requestDetails, index++ + chain, UUID.randomUUID(), requestDetails, index++ ) } } class SubscribeBalance(val subscribe: Boolean) : - Base(), - RequestReply { + Base(), + RequestReply { private var index = 0 private var balanceRequest: Events.BalanceRequest? = null @@ -195,8 +202,8 @@ class EventsBuilder { override fun onRequest(msg: BlockchainOuterClass.BalanceRequest) { balanceRequest = Events.BalanceRequest( - msg.asset.code.uppercase(Locale.getDefault()), - msg.address.addrTypeCase.name + msg.asset.code.uppercase(Locale.getDefault()), + msg.address.addrTypeCase.name ) } @@ -207,14 +214,14 @@ class EventsBuilder { val addressBalance = Events.AddressBalance(msg.asset.code, msg.address.address) val chain = Chain.byId(msg.asset.chain.number) return Events.SubscribeBalance( - chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++ + chain, UUID.randomUUID(), subscribe, requestDetails, balanceRequest!!, addressBalance, index++ ) } } - class TxStatus() : - Base(), - RequestReply { + class TxStatus : + Base(), + RequestReply { private var index = 0 private var txStatusRequest: Events.TxStatusRequest? = null @@ -225,21 +232,20 @@ class EventsBuilder { override fun onReply(msg: BlockchainOuterClass.TxStatus): Events.TxStatus { return Events.TxStatus( - chain, UUID.randomUUID(), requestDetails, txStatusRequest!!, - Events.TxStatusResponse(msg.confirmations), - index++ + chain, UUID.randomUUID(), requestDetails, txStatusRequest!!, + Events.TxStatusResponse(msg.confirmations), + index++ ) } override fun getT(): TxStatus { return this } - } class NativeCall : - Base(), - RequestReply { + Base(), + RequestReply { val items = ArrayList() val replies = HashMap() private var index = 0 @@ -252,11 +258,11 @@ class EventsBuilder { withChain(msg.chain.number) msg.itemsList.forEach { item -> this.items.add( - Events.NativeCallItemDetails( - item.method, - item.id, - item.payload.size().toLong() - ) + Events.NativeCallItemDetails( + item.method, + item.id, + item.payload.size().toLong() + ) ) } } @@ -264,38 +270,40 @@ class EventsBuilder { override fun onReply(msg: BlockchainOuterClass.NativeCallReplyItem): Events.NativeCall { val item = items.find { it.id == msg.id }!! return Events.NativeCall( - request = requestDetails, - total = items.size, - index = index++, - succeed = msg.succeed, - blockchain = chain, - nativeCall = item, - payloadSizeBytes = item.payloadSizeBytes, - id = UUID.randomUUID(), - channel = Events.Channel.GRPC + request = requestDetails, + total = items.size, + index = index++, + succeed = msg.succeed, + blockchain = chain, + nativeCall = item, + payloadSizeBytes = item.payloadSizeBytes, + id = UUID.randomUUID(), + channel = Events.Channel.GRPC ) } - fun onReply(reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult, - channel: Events.Channel): Events.NativeCall { + fun onReply( + reply: io.emeraldpay.dshackle.rpc.NativeCall.CallResult, + channel: Events.Channel + ): Events.NativeCall { val item = items.find { it.id == reply.id }!! return Events.NativeCall( - request = requestDetails, - total = items.size, - index = index++, - succeed = !reply.isError(), - blockchain = chain, - nativeCall = item, - payloadSizeBytes = item.payloadSizeBytes, - id = UUID.randomUUID(), - channel = channel + request = requestDetails, + total = items.size, + index = index++, + succeed = !reply.isError(), + blockchain = chain, + nativeCall = item, + payloadSizeBytes = item.payloadSizeBytes, + id = UUID.randomUUID(), + channel = channel ) } } class NativeSubscribe : - Base(), - RequestReply { + Base(), + RequestReply { var item: Events.NativeSubscribeItemDetails? = null val replies = HashMap() @@ -306,26 +314,26 @@ class EventsBuilder { override fun onRequest(msg: BlockchainOuterClass.NativeSubscribeRequest) { withChain(msg.chain.number) this.item = Events.NativeSubscribeItemDetails( - msg.method, - msg.payload.size().toLong() + msg.method, + msg.payload.size().toLong() ) } override fun onReply(msg: BlockchainOuterClass.NativeSubscribeReplyItem): Events.NativeSubscribe { return Events.NativeSubscribe( - request = requestDetails, - blockchain = chain, - nativeSubscribe = item!!, - payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L, - id = UUID.randomUUID(), - channel = Events.Channel.GRPC + request = requestDetails, + blockchain = chain, + nativeSubscribe = item!!, + payloadSizeBytes = msg.payload?.size()?.toLong() ?: 0L, + id = UUID.randomUUID(), + channel = Events.Channel.GRPC ) } } class Describe : - Base(), - RequestReply { + Base(), + RequestReply { override fun getT(): Describe { return this @@ -336,15 +344,15 @@ class EventsBuilder { override fun onReply(msg: BlockchainOuterClass.DescribeResponse): Events.Describe { return Events.Describe( - id = UUID.randomUUID(), - request = requestDetails + id = UUID.randomUUID(), + request = requestDetails ) } } class Status : - Base(), - RequestReply { + Base(), + RequestReply { override fun getT(): Status { return this } @@ -355,11 +363,10 @@ class EventsBuilder { override fun onReply(msg: BlockchainOuterClass.ChainStatus): Events.Status { val chain = Chain.byId(msg.chainValue) return Events.Status( - blockchain = chain, - request = requestDetails, - id = UUID.randomUUID() + blockchain = chain, + request = requestDetails, + id = UUID.randomUUID() ) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyCall.kt index 2839d990..e8efa5c2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyCall.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyCall.kt @@ -23,10 +23,10 @@ import org.slf4j.LoggerFactory * JSON RPC call to the proxy */ class ProxyCall( - /** - * Type of the request. The response format depends on it - */ - val type: RpcType + /** + * Type of the request. The response format depends on it + */ + val type: RpcType ) { companion object { @@ -55,4 +55,4 @@ class ProxyCall( */ BATCH } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt index 753b70b4..6e5ab4f9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt @@ -24,8 +24,8 @@ import io.emeraldpay.dshackle.config.ProxyConfig import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp import io.emeraldpay.dshackle.rpc.NativeCall import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.rpc.RpcException +import io.emeraldpay.grpc.Chain import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Timer @@ -38,16 +38,14 @@ import org.slf4j.LoggerFactory import org.springframework.http.HttpHeaders import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import reactor.netty.DisposableServer import reactor.netty.http.server.HttpServer import reactor.netty.http.server.HttpServerRequest import reactor.netty.http.server.HttpServerResponse import reactor.netty.http.server.HttpServerRoutes -import java.util.* +import java.util.EnumMap import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock import java.util.function.BiFunction -import kotlin.collections.HashMap import kotlin.concurrent.read import kotlin.concurrent.write @@ -55,12 +53,12 @@ import kotlin.concurrent.write * HTTP Proxy Server */ class ProxyServer( - private var config: ProxyConfig, - private val readRpcJson: ReadRpcJson, - private val writeRpcJson: WriteRpcJson, - private val nativeCall: NativeCall, - private val tlsSetup: TlsSetup, - private val accessHandler: AccessHandlerHttp.HandlerFactory + private var config: ProxyConfig, + private val readRpcJson: ReadRpcJson, + private val writeRpcJson: WriteRpcJson, + private val nativeCall: NativeCall, + private val tlsSetup: TlsSetup, + private val accessHandler: AccessHandlerHttp.HandlerFactory ) { companion object { @@ -101,19 +99,19 @@ class ProxyServer( } log.info("Listening Proxy on ${config.host}:${config.port}") var serverBuilder = HttpServer.create() - .doOnChannelInit { _, channel, _ -> - channel.pipeline().addFirst(errorHandler) - } - .host(config.host) - .port(config.port) + .doOnChannelInit { _, channel, _ -> + channel.pipeline().addFirst(errorHandler) + } + .host(config.host) + .port(config.port) tlsSetup.setupServer("proxy", config.tls, false)?.let { sslContext -> serverBuilder = serverBuilder.secure { secure -> secure.sslContext(sslContext) } } serverBuilder - .route(this::setupRoutes) - .bindNow() + .route(this::setupRoutes) + .bindNow() } fun setupRoutes(routes: HttpServerRoutes) { @@ -139,26 +137,26 @@ class ProxyServer( } } val request = BlockchainOuterClass.NativeCallRequest.newBuilder() - .setChain(Common.ChainRef.forNumber(chain.id)) - .addAllItems(call.items) - .build() + .setChain(Common.ChainRef.forNumber(chain.id)) + .addAllItems(call.items) + .build() handler.onRequest(request) val jsons = nativeCall - .nativeCallResult(Mono.just(request)) - .doOnNext { - metricById(it.id)?.requestMetric?.increment() + .nativeCallResult(Mono.just(request)) + .doOnNext { + metricById(it.id)?.requestMetric?.increment() + } + .doOnNext { + handler.onResponse(it) + metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS) + } + .doOnError { + // when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed + call.items.forEach { item -> + requestMetrics.get(chain, item.method).errorMetric.increment() } - .doOnNext { - handler.onResponse(it) - metricById(it.id)?.callMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS) - } - .doOnError { - //when error happened the whole flux is stopped and no result is produced, so we should mark all the requests as failed - call.items.forEach { item -> - requestMetrics.get(chain, item.method).errorMetric.increment() - } - } - .transform(writeRpcJson.toJsons(call)) + } + .transform(writeRpcJson.toJsons(call)) return if (call.type == ProxyCall.RpcType.SINGLE) { jsons.next() } else { @@ -166,21 +164,25 @@ class ProxyServer( } } - fun processRequest(chain: Chain, request: Mono, handler: AccessHandlerHttp.RequestHandler): Flux { + fun processRequest( + chain: Chain, + request: Mono, + handler: AccessHandlerHttp.RequestHandler + ): Flux { return request - .map(readRpcJson) - .flatMapMany { call -> - execute(chain, call, handler) - } - .onErrorResume(RpcException::class.java) { err -> - val id = err.details?.let { - if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1) - } ?: JsonRpcResponse.NumberId(-1) + .map(readRpcJson) + .flatMapMany { call -> + execute(chain, call, handler) + } + .onErrorResume(RpcException::class.java) { err -> + val id = err.details?.let { + if (it is JsonRpcResponse.Id) it else JsonRpcResponse.NumberId(-1) + } ?: JsonRpcResponse.NumberId(-1) - val json = JsonRpcResponse.error(err.code, err.rpcMessage, id) - Mono.just(Global.objectMapper.writeValueAsString(json)) - } - .map { Unpooled.wrappedBuffer(it.toByteArray()) } + val json = JsonRpcResponse.error(err.code, err.rpcMessage, id) + Mono.just(Global.objectMapper.writeValueAsString(json)) + } + .map { Unpooled.wrappedBuffer(it.toByteArray()) } } fun proxy(routeConfig: ProxyConfig.Route): BiFunction> { @@ -188,13 +190,13 @@ class ProxyServer( // handle access events val eventHandler = accessHandler.create(req, routeConfig.blockchain) val request = req.receive() - .aggregate() - .asByteArray() + .aggregate() + .asByteArray() val results = processRequest(routeConfig.blockchain, request, eventHandler) - // make sure that the access log handler is closed at the end, so it can render the logs - .doFinally { eventHandler.close() } + // make sure that the access log handler is closed at the end, so it can render the logs + .doFinally { eventHandler.close() } resp.addHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .send(results) + .send(results) } } @@ -251,25 +253,25 @@ class ProxyServer( class RequestMetricsBasic(chain: Chain) : RequestMetrics { override val callMetric = Timer.builder("request.jsonrpc.call") - .tags("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tags("chain", chain.chainCode) + .register(Metrics.globalRegistry) override val errorMetric = Counter.builder("request.jsonrpc.err") - .tags("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tags("chain", chain.chainCode) + .register(Metrics.globalRegistry) override val requestMetric = Counter.builder("request.jsonrpc.request.total") - .tags("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tags("chain", chain.chainCode) + .register(Metrics.globalRegistry) } class RequestMetricsWithMethod(chain: Chain, method: String) : RequestMetrics { override val callMetric = Timer.builder("request.jsonrpc.call") - .tags("chain", chain.chainCode, "method", method) - .register(Metrics.globalRegistry) + .tags("chain", chain.chainCode, "method", method) + .register(Metrics.globalRegistry) override val errorMetric = Counter.builder("request.jsonrpc.err") - .tags("chain", chain.chainCode, "method", method) - .register(Metrics.globalRegistry) + .tags("chain", chain.chainCode, "method", method) + .register(Metrics.globalRegistry) override val requestMetric = Counter.builder("request.jsonrpc.request.total") - .tags("chain", chain.chainCode, "method", method) - .register(Metrics.globalRegistry) + .tags("chain", chain.chainCode, "method", method) + .register(Metrics.globalRegistry) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ReadRpcJson.kt b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ReadRpcJson.kt index 78edd6cf..a5901f4c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ReadRpcJson.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ReadRpcJson.kt @@ -25,19 +25,16 @@ import io.emeraldpay.etherjar.rpc.RpcException import io.emeraldpay.etherjar.rpc.RpcResponseError import io.emeraldpay.etherjar.rpc.json.RequestJson import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.io.IOException -import java.util.* import java.util.function.Function import java.util.stream.Collectors - /** * Reader for JSON RPC request */ @Service -open class ReadRpcJson() : Function { +open class ReadRpcJson : Function { companion object { private val log = LoggerFactory.getLogger(ReadRpcJson::class.java) @@ -55,21 +52,37 @@ open class ReadRpcJson() : Function { val id = json["id"] if ("2.0" != json["jsonrpc"]) { if (json["jsonrpc"] == null) { - throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "jsonrpc version is not set", id?.let { JsonRpcResponse.Id.from(it) }) + throw RpcException( + RpcResponseError.CODE_INVALID_REQUEST, + "jsonrpc version is not set", + id?.let { JsonRpcResponse.Id.from(it) } + ) } - throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Unsupported JSON RPC version: " + json["jsonrpc"].toString(), id?.let { JsonRpcResponse.Id.from(it) }) + throw RpcException( + RpcResponseError.CODE_INVALID_REQUEST, + "Unsupported JSON RPC version: " + json["jsonrpc"].toString(), + id?.let { JsonRpcResponse.Id.from(it) } + ) } if (!(json["method"] != null && json["method"] is String)) { - throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Method is not set", id?.let { JsonRpcResponse.Id.from(it) }) + throw RpcException( + RpcResponseError.CODE_INVALID_REQUEST, + "Method is not set", + id?.let { JsonRpcResponse.Id.from(it) } + ) } if (json.containsKey("params") && json["params"] !is List<*>) { - throw RpcException(RpcResponseError.CODE_INVALID_REQUEST, "Params must be an array", id?.let { JsonRpcResponse.Id.from(it) }) + throw RpcException( + RpcResponseError.CODE_INVALID_REQUEST, + "Params must be an array", + id?.let { JsonRpcResponse.Id.from(it) } + ) } RequestJson( - json["method"].toString(), - //params MAY be omitted - (json["params"] ?: emptyList()) as List<*>, - id + json["method"].toString(), + // params MAY be omitted + (json["params"] ?: emptyList()) as List<*>, + id ) } } @@ -81,7 +94,7 @@ open class ReadRpcJson() : Function { fun getStartOfJson(buf: ByteArray): Byte { val count = buf.size var i = 0 - //if cannot find anything in the first 255 bytes, just consider it as invalid + // if cannot find anything in the first 255 bytes, just consider it as invalid while (i < 256 && i < count) { if (buf[i] != spaces[0] && buf[i] != spaces[1] && buf[i] != spaces[2]) { return buf[i] @@ -127,17 +140,17 @@ open class ReadRpcJson() : Function { // our internal ids for calls var seq = 0 val batch = list.stream() - .map>(jsonExtractor) - .map { json -> - val id = seq++ - context.ids[id] = json.id - BlockchainOuterClass.NativeCallItem.newBuilder() - .setId(id) - .setMethod(json.method) - .setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params))) - .build() - } - .collect(Collectors.toList()) + .map>(jsonExtractor) + .map { json -> + val id = seq++ + context.ids[id] = json.id + BlockchainOuterClass.NativeCallItem.newBuilder() + .setId(id) + .setMethod(json.method) + .setPayload(ByteString.copyFrom(objectMapper.writeValueAsBytes(json.params))) + .build() + } + .collect(Collectors.toList()) context.items.addAll(batch) return context } catch (e: RpcException) { @@ -147,5 +160,4 @@ open class ReadRpcJson() : Function { throw RpcException(RpcResponseError.CODE_INVALID_JSON, e.message) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/proxy/WriteRpcJson.kt b/src/main/kotlin/io/emeraldpay/dshackle/proxy/WriteRpcJson.kt index 865ff978..e7c6a691 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/proxy/WriteRpcJson.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/proxy/WriteRpcJson.kt @@ -17,7 +17,6 @@ package io.emeraldpay.dshackle.proxy import com.fasterxml.jackson.databind.ObjectMapper -import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.rpc.NativeCall import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse @@ -31,7 +30,7 @@ import java.util.function.Function * Writer for JSON RPC requests */ @Service -open class WriteRpcJson() { +open class WriteRpcJson { companion object { private val log = LoggerFactory.getLogger(WriteRpcJson::class.java) @@ -45,35 +44,35 @@ open class WriteRpcJson() { open fun toJsons(call: ProxyCall): Function, Flux> { return Function { flux -> flux - .flatMap { response -> - if (!call.ids.containsKey(response.id)) { - log.warn("ID wasn't requested: ${response.id}") - return@flatMap Flux.empty() - } - val json = toJson(call, response) - if (json == null) { - Flux.empty() - } else { - Flux.just(json) - } + .flatMap { response -> + if (!call.ids.containsKey(response.id)) { + log.warn("ID wasn't requested: ${response.id}") + return@flatMap Flux.empty() } - .onErrorResume { t -> - if (t is NativeCall.CallFailure) { - Mono.just(toJson(call, t)!!) - } else { - Mono.empty() - } + val json = toJson(call, response) + if (json == null) { + Flux.empty() + } else { + Flux.just(json) } - .onErrorContinue { t, _ -> - log.warn("Failed to convert to JSON", t) + } + .onErrorResume { t -> + if (t is NativeCall.CallFailure) { + Mono.just(toJson(call, t)!!) + } else { + Mono.empty() } + } + .onErrorContinue { t, _ -> + log.warn("Failed to convert to JSON", t) + } } } open fun toJson(call: ProxyCall, response: NativeCall.CallResult): String? { val id = call.ids[response.id]?.let { JsonRpcResponse.Id.from(it) - } ?: return null; + } ?: return null val json = if (response.isError()) { val error = response.error!! error.upstreamError?.let { upstreamError -> @@ -86,7 +85,7 @@ open class WriteRpcJson() { } fun toJson(call: ProxyCall, error: NativeCall.CallFailure): String? { - val id = call.ids[error.id] ?: return null; + val id = call.ids[error.id] ?: return null val json = JsonRpcResponse.error(-32003, error.reason.message ?: "", JsonRpcResponse.Id.from(id)) return objectMapper.writeValueAsString(json) } @@ -97,18 +96,18 @@ open class WriteRpcJson() { fun asArray(): Function, Flux> { return Function { flux -> val body = flux.zipWith(Flux.concat(Mono.just(false), Flux.just(true).repeat())) - .map { - if (it.t2) { - "," + it.t1 - } else { - it.t1 - } + .map { + if (it.t2) { + "," + it.t1 + } else { + it.t1 } + } Flux.concat( - Mono.just("["), - body, - Mono.just("]") + Mono.just("["), + body, + Mono.just("]") ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt index cb4e488a..235acfdd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/AlwaysQuorum.kt @@ -20,9 +20,8 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException -import io.emeraldpay.etherjar.rpc.RpcException -open class AlwaysQuorum: CallQuorum { +open class AlwaysQuorum : CallQuorum { private var resolved = false private var result: ByteArray? = null @@ -60,4 +59,4 @@ open class AlwaysQuorum: CallQuorum { override fun toString(): String { return "Quorum: Accept Any" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt index 0fecb435..90189744 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/BroadcastQuorum.kt @@ -20,7 +20,7 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream open class BroadcastQuorum( - val quorum: Int = 3 + val quorum: Int = 3 ) : CallQuorum, ValueAwareQuorum(String::class.java) { private var result: ByteArray? = null @@ -61,4 +61,4 @@ open class BroadcastQuorum( override fun toString(): String { return "Quorum: Broadcast to $quorum upstreams" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt index b02b7d04..76ee9483 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/CallQuorum.kt @@ -20,10 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException -import io.emeraldpay.etherjar.domain.TransactionId -import io.emeraldpay.etherjar.rpc.RpcException -import io.emeraldpay.etherjar.rpc.json.BlockJson -import io.emeraldpay.etherjar.rpc.json.TransactionRefJson import reactor.util.function.Tuple2 import java.util.function.BiFunction import java.util.function.Predicate @@ -54,4 +50,4 @@ interface CallQuorum { } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt index af35cded..e847fca5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonEmptyQuorum.kt @@ -16,15 +16,11 @@ */ package io.emeraldpay.dshackle.quorum -import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException -import io.emeraldpay.etherjar.rpc.JacksonRpcConverter -import io.emeraldpay.etherjar.rpc.RpcException open class NonEmptyQuorum( - val maxTries: Int = 3 + val maxTries: Int = 3 ) : CallQuorum, ValueAwareQuorum(Any::class.java) { private var result: ByteArray? = null @@ -59,4 +55,4 @@ open class NonEmptyQuorum( override fun toString(): String { return "Quorum: Accept Non Error Result" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt index 37fc0baf..563b07f3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NonceQuorum.kt @@ -16,17 +16,14 @@ */ package io.emeraldpay.dshackle.quorum -import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.etherjar.hex.HexQuantity -import io.emeraldpay.etherjar.rpc.JacksonRpcConverter -import io.emeraldpay.etherjar.rpc.RpcException import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock open class NonceQuorum( - val tries: Int = 3 + val tries: Int = 3 ) : CallQuorum, ValueAwareQuorum(String::class.java) { private val lock = ReentrantLock() @@ -74,4 +71,4 @@ open class NonceQuorum( override fun toString(): String { return "Quorum: Confirm with $tries upstreams" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt index 834fc506..1d3eb795 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/NotLaggingQuorum.kt @@ -20,7 +20,6 @@ import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException -import io.emeraldpay.etherjar.rpc.RpcException import java.util.concurrent.atomic.AtomicReference /** @@ -28,7 +27,7 @@ import java.util.concurrent.atomic.AtomicReference * * NOTE: NativeCall checks the quorums and applies a HeightSelector if NotLaggingQuorum is enabled for a call */ -class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum { +class NotLaggingQuorum(val maxLag: Long = 0) : CallQuorum { private val result: AtomicReference = AtomicReference() private val failed = AtomicReference(false) @@ -73,4 +72,4 @@ class NotLaggingQuorum(val maxLag: Long = 0): CallQuorum { override fun toString(): String { return "Quorum: late <= $maxLag blocks" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumReaderFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumReaderFactory.kt index 8e0727bb..6edfd7fc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumReaderFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumReaderFactory.kt @@ -35,4 +35,4 @@ interface QuorumReaderFactory { return QuorumRpcReader(apis, quorum) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt index ad2cea12..000c26e6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRpcReader.kt @@ -30,8 +30,8 @@ import reactor.util.function.Tuples * Makes request with applying Quorum */ class QuorumRpcReader( - private val apis: ApiSource, - private val quorum: CallQuorum + private val apis: ApiSource, + private val quorum: CallQuorum ) : Reader { companion object { @@ -58,8 +58,8 @@ class QuorumRpcReader( val defaultResult: Mono = Mono.just(quorum).flatMap { q -> if (q.isFailed()) { Mono.error( - q.getError()?.asException(JsonRpcResponse.NumberId(1)) - ?: RpcException(-32000, "Unknown Upstream error") + q.getError()?.asException(JsonRpcResponse.NumberId(1)) + ?: RpcException(-32000, "Unknown Upstream error") ) } else { log.warn("Did not get any result from upstream. Method [${key.method}] using [$q]") @@ -68,72 +68,71 @@ class QuorumRpcReader( } return Flux.from(apis) - .takeUntil { - quorum.isFailed() || quorum.isResolved() - } - .flatMap { api -> - api.getApi() - .read(key) - .flatMap { response -> - response.requireResult() - .onErrorResume { err -> - if (err is RpcException || err is JsonRpcException) { - // on error notify quorum, it may use error message or other details - val cleanErr: JsonRpcException = when (err) { - is RpcException -> JsonRpcException.from(err) - is JsonRpcException -> err - else -> throw IllegalStateException("Cannot convert from exception", err) - } - quorum.record(cleanErr, api) - // it it's failed after that, then we don't need more calls, stop api source - if (quorum.isFailed()) { - apis.resolve() - } else { - apis.request(1) - } - } else { - log.warn("Result processing error", err) - } - Mono.empty() - } + .takeUntil { + quorum.isFailed() || quorum.isResolved() + } + .flatMap { api -> + api.getApi() + .read(key) + .flatMap { response -> + response.requireResult() + .onErrorResume { err -> + if (err is RpcException || err is JsonRpcException) { + // on error notify quorum, it may use error message or other details + val cleanErr: JsonRpcException = when (err) { + is RpcException -> JsonRpcException.from(err) + is JsonRpcException -> err + else -> throw IllegalStateException("Cannot convert from exception", err) + } + quorum.record(cleanErr, api) + // it it's failed after that, then we don't need more calls, stop api source + if (quorum.isFailed()) { + apis.resolve() + } else { + apis.request(1) + } + } else { + log.warn("Result processing error", err) + } + Mono.empty() } - .map { Tuples.of(it, api) } - } - .retryWhen(retrySpec) - // record all correct responses until quorum reached - .reduce(quorum, { res, a -> - if (res.record(a.t1, a.t2)) { - apis.resolve() - } else { - apis.request(1) - } - res - }) - // if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum - .onErrorResume { err -> - if (quorum.isResolved()) { - Mono.just(quorum) - } else { - Mono.error(err) } + .map { Tuples.of(it, api) } + } + .retryWhen(retrySpec) + // record all correct responses until quorum reached + .reduce(quorum, { res, a -> + if (res.record(a.t1, a.t2)) { + apis.resolve() + } else { + apis.request(1) } - .doOnNext { - if (!it.isResolved() && !it.isFailed()) { - log.debug("No quorum for ${key.method} using [${quorum}]. Error: ${it.getError()?.message ?: ""}") - } + res + }) + // if last call resulted in error it's still possible that request was resolved correctly. i.e. for BroadcastQuorum + .onErrorResume { err -> + if (quorum.isResolved()) { + Mono.just(quorum) + } else { + Mono.error(err) } - // return nothing if not resolved - .filter { it.isResolved() } - .map { - // TODO find actual quorum number - QuorumRpcReader.Result(it.getResult()!!, 1) + } + .doOnNext { + if (!it.isResolved() && !it.isFailed()) { + log.debug("No quorum for ${key.method} using [$quorum]. Error: ${it.getError()?.message ?: ""}") } - .switchIfEmpty(defaultResult) + } + // return nothing if not resolved + .filter { it.isResolved() } + .map { + // TODO find actual quorum number + QuorumRpcReader.Result(it.getResult()!!, 1) + } + .switchIfEmpty(defaultResult) } - class Result( - val value: ByteArray, - val quorum: Int + val value: ByteArray, + val quorum: Int ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt index c5cb9845..87eca4b2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/ValueAwareQuorum.kt @@ -16,18 +16,16 @@ */ package io.emeraldpay.dshackle.quorum -import com.fasterxml.jackson.databind.ObjectMapper import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException -import io.emeraldpay.etherjar.rpc.JacksonRpcConverter import io.emeraldpay.etherjar.rpc.RpcException import org.slf4j.LoggerFactory abstract class ValueAwareQuorum( - val clazz: Class -): CallQuorum { + val clazz: Class +) : CallQuorum { private val log = LoggerFactory.getLogger(ValueAwareQuorum::class.java) private var rpcError: JsonRpcError? = null @@ -45,7 +43,7 @@ abstract class ValueAwareQuorum( } catch (e: Exception) { recordError(response, e.message, upstream) } - return isResolved(); + return isResolved() } override fun record(error: JsonRpcException, upstream: Upstream) { @@ -60,4 +58,4 @@ abstract class ValueAwareQuorum( override fun getError(): JsonRpcError? { return rpcError } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt index 7c068961..2d02b07b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt @@ -20,15 +20,14 @@ import io.emeraldpay.dshackle.Defaults import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import java.time.Duration /** * Composition of multiple readers. * Reader returns first value returned by any of the source readers by checking one by one until one of them returns a non-empty result. */ class CompoundReader( - private vararg val readers: Reader -): Reader { + private vararg val readers: Reader +) : Reader { companion object { private val log = LoggerFactory.getLogger(CompoundReader::class.java) @@ -39,13 +38,12 @@ class CompoundReader( return Mono.empty() } return Flux.fromIterable(readers.asIterable()) - .flatMap({ rdr -> - rdr.read(key) - .timeout(Defaults.timeoutInternal, Mono.empty()) - .doOnError { t -> log.warn("Failed to read from $rdr", t) } - .onErrorResume { Mono.empty() } - }, 1) - .next() + .flatMap({ rdr -> + rdr.read(key) + .timeout(Defaults.timeoutInternal, Mono.empty()) + .doOnError { t -> log.warn("Failed to read from $rdr", t) } + .onErrorResume { Mono.empty() } + }, 1) + .next() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt index 900d2445..f25cc08c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt @@ -18,9 +18,9 @@ package io.emeraldpay.dshackle.reader import reactor.core.publisher.Mono -class EmptyReader: Reader { +class EmptyReader : Reader { override fun read(key: K): Mono { return Mono.empty() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt index 5bcd2cd1..eec6fd9b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt @@ -21,5 +21,4 @@ import reactor.core.publisher.Mono interface Reader { fun read(key: K): Mono - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt index 7e37d1ad..5f66e4fa 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt @@ -22,22 +22,21 @@ import java.util.function.Function * Reader wrapper that maps the input key from ne value to another (ex. convert from Long to String) */ class RekeyingReader( - /** - * Mapping between original Key and Key supported by the reader - */ - private val rekey: Function, - /** - * Actual reader - */ - private val reader: Reader + /** + * Mapping between original Key and Key supported by the reader + */ + private val rekey: Function, + /** + * Actual reader + */ + private val reader: Reader ) : Reader { override fun read(key: K): Mono { return Mono.just(key) - .map(rekey) - .flatMap { - reader.read(it) - } + .map(rekey) + .flatMap { + reader.read(it) + } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/RpcReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/RpcReader.kt index 2d3277e4..cd920fb9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/RpcReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/RpcReader.kt @@ -25,8 +25,8 @@ import reactor.core.publisher.Mono * Reader that requests data through upstream RPC using provided JSON RPC request builder */ class RpcReader( - private val up: Multistream, - private val paramsBuilder: (T) -> JsonRpcRequest + private val up: Multistream, + private val paramsBuilder: (T) -> JsonRpcRequest ) : Reader { companion object { @@ -45,11 +45,10 @@ class RpcReader( override fun read(key: T): Mono { return up.getDirectApi(Selector.empty) - .flatMap { rdr -> - rdr.read(paramsBuilder(key)).flatMap { - it.requireResult() - } + .flatMap { rdr -> + rdr.read(paramsBuilder(key)).flatMap { + it.requireResult() } + } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt index 4a82a0a8..056a2831 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt @@ -22,18 +22,17 @@ import java.util.function.Function * Reader wrapper that transforms output of the reader to a different format */ class TransformingReader( - /** - * Actual reader - */ - private val reader: Reader, - /** - * Result transformation - */ - private val transformer: Function + /** + * Actual reader + */ + private val reader: Reader, + /** + * Result transformation + */ + private val transformer: Function ) : Reader { override fun read(key: K): Mono { return reader.read(key).map(transformer) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt index 0d9f0ff7..6d32a561 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt @@ -31,44 +31,45 @@ import org.springframework.context.annotation.DependsOn import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import java.util.* +import java.util.Locale import java.util.concurrent.TimeUnit -@Service @DependsOn("monitoringSetup") +@Service +@DependsOn("monitoringSetup") class BlockchainRpc( - @Autowired private val nativeCall: NativeCall, - @Autowired private val nativeSubscribe: NativeSubscribe, - @Autowired private val streamHead: StreamHead, - @Autowired private val trackTx: List, - @Autowired private val trackAddress: List, - @Autowired private val describe: Describe, - @Autowired private val subscribeStatus: SubscribeStatus -): ReactorBlockchainGrpc.BlockchainImplBase() { + @Autowired private val nativeCall: NativeCall, + @Autowired private val nativeSubscribe: NativeSubscribe, + @Autowired private val streamHead: StreamHead, + @Autowired private val trackTx: List, + @Autowired private val trackAddress: List, + @Autowired private val describe: Describe, + @Autowired private val subscribeStatus: SubscribeStatus +) : ReactorBlockchainGrpc.BlockchainImplBase() { private val log = LoggerFactory.getLogger(BlockchainRpc::class.java) private val describeMetric = Counter.builder("request.grpc.request") - .tag("type", "describe") - .tag("chain", "NA") - .register(Metrics.globalRegistry) + .tag("type", "describe") + .tag("chain", "NA") + .register(Metrics.globalRegistry) private val subscribeStatusMetric = Counter.builder("request.grpc.request") - .tag("type", "subscribeStatus") - .tag("chain", "NA") - .register(Metrics.globalRegistry) + .tag("type", "subscribeStatus") + .tag("chain", "NA") + .register(Metrics.globalRegistry) private val errorMetric = Counter.builder("request.grpc.err") - .register(Metrics.globalRegistry) + .register(Metrics.globalRegistry) private val chainMetrics = ChainValue { chain -> RequestMetrics(chain) } override fun nativeCall(request: Mono): Flux { var startTime = 0L var metrics: RequestMetrics? = null return nativeCall.nativeCall( - request - .doOnNext { - metrics = chainMetrics.get(it.chain) - metrics!!.nativeCallMetric.increment() - startTime = System.currentTimeMillis() - } + request + .doOnNext { + metrics = chainMetrics.get(it.chain) + metrics!!.nativeCallMetric.increment() + startTime = System.currentTimeMillis() + } ).doOnNext { metrics?.nativeCallRespMetric?.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS) }.doOnError { errorMetric.increment() } @@ -77,11 +78,11 @@ class BlockchainRpc( override fun nativeSubscribe(request: Mono): Flux { var metrics: RequestMetrics? = null return nativeSubscribe.nativeSubscribe( - request - .doOnNext { - metrics = chainMetrics.get(it.chain) - metrics!!.nativeSubscribeMetric.increment() - } + request + .doOnNext { + metrics = chainMetrics.get(it.chain) + metrics!!.nativeSubscribeMetric.increment() + } ).doOnNext { metrics?.nativeSubscribeRespMetric?.increment() }.doOnError { errorMetric.increment() } @@ -89,8 +90,8 @@ class BlockchainRpc( override fun subscribeHead(request: Mono): Flux { return streamHead.add( - request - .doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() } + request + .doOnNext { chainMetrics.get(it.type).subscribeHeadMetric.increment() } ).doOnError { errorMetric.increment() } } @@ -102,8 +103,8 @@ class BlockchainRpc( try { trackTx.find { it.isSupported(chain) }?.let { track -> track.subscribe(request) - .doOnNext { metrics.subscribeHeadRespMetric.increment() } - .doOnError { errorMetric.increment() } + .doOnNext { metrics.subscribeHeadRespMetric.increment() } + .doOnError { errorMetric.increment() } } ?: Flux.error(SilentException.UnsupportedBlockchain(chain)) } catch (t: Throwable) { log.error("Internal error during Tx Subscription", t) @@ -122,12 +123,12 @@ class BlockchainRpc( try { trackAddress.find { it.isSupported(chain, asset) }?.let { track -> track.subscribe(request) - .doOnNext { metrics.subscribeBalanceRespMetric.increment() } - .doOnError { errorMetric.increment() } + .doOnNext { metrics.subscribeBalanceRespMetric.increment() } + .doOnError { errorMetric.increment() } } ?: Flux.error(SilentException.UnsupportedBlockchain(chain)) - .doOnSubscribe { - log.error("Balance for $chain:$asset is not supported") - } + .doOnSubscribe { + log.error("Balance for $chain:$asset is not supported") + } } catch (t: Throwable) { log.error("Internal error during Balance Subscription", t) errorMetric.increment() @@ -146,13 +147,16 @@ class BlockchainRpc( try { trackAddress.find { it.isSupported(chain, asset) }?.let { track -> track.getBalance(request) - .doOnNext { - metrics.getBalanceRespMetric.record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS) - } + .doOnNext { + metrics.getBalanceRespMetric.record( + System.currentTimeMillis() - startTime, + TimeUnit.MILLISECONDS + ) + } } ?: Flux.error(SilentException.UnsupportedBlockchain(chain)) - .doOnSubscribe { - log.error("Balance for $chain:$asset is not supported") - } + .doOnSubscribe { + log.error("Balance for $chain:$asset is not supported") + } } catch (t: Throwable) { log.error("Internal error during Balance Request", t) errorMetric.increment() @@ -164,61 +168,61 @@ class BlockchainRpc( override fun describe(request: Mono): Mono { describeMetric.increment() return describe.describe(request) - .doOnError { errorMetric.increment() } + .doOnError { errorMetric.increment() } } override fun subscribeStatus(request: Mono): Flux { subscribeStatusMetric.increment() return subscribeStatus.subscribeStatus(request) - .doOnError { errorMetric.increment() } + .doOnError { errorMetric.increment() } } class RequestMetrics(chain: Chain) { val nativeCallMetric = Counter.builder("request.grpc.request") - .tag("type", "nativeCall") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "nativeCall") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val nativeCallRespMetric = Timer.builder("request.grpc.response") - .tag("type", "nativeCall") - .tag("chain", chain.chainCode) - .publishPercentileHistogram() - .register(Metrics.globalRegistry) + .tag("type", "nativeCall") + .tag("chain", chain.chainCode) + .publishPercentileHistogram() + .register(Metrics.globalRegistry) val nativeSubscribeMetric = Counter.builder("request.grpc.request") - .tag("type", "nativeSubscribe") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "nativeSubscribe") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val nativeSubscribeRespMetric = Counter.builder("request.grpc.response") - .tag("type", "nativeSubscribe") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "nativeSubscribe") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val subscribeHeadMetric = Counter.builder("request.grpc.request") - .tag("type", "subscribeHead") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "subscribeHead") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val subscribeHeadRespMetric = Counter.builder("request.grpc.reply") - .tag("type", "subscribeHead") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "subscribeHead") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val subscribeTxMetric = Counter.builder("request.grpc.request") - .tag("type", "subscribeTx") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "subscribeTx") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val subscribeBalanceMetric = Counter.builder("request.grpc.request") - .tag("type", "subscribeBalance") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "subscribeBalance") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val subscribeBalanceRespMetric = Counter.builder("request.grpc.reply") - .tag("type", "subscribeBalance") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "subscribeBalance") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val getBalanceMetric = Counter.builder("request.grpc.request") - .tag("type", "getBalance") - .tag("chain", chain.chainCode) - .register(Metrics.globalRegistry) + .tag("type", "getBalance") + .tag("chain", chain.chainCode) + .register(Metrics.globalRegistry) val getBalanceRespMetric = Timer.builder("request.grpc.response") - .tag("type", "getBalance") - .tag("chain", chain.chainCode) - .publishPercentileHistogram() - .register(Metrics.globalRegistry) + .tag("type", "getBalance") + .tag("chain", chain.chainCode) + .publishPercentileHistogram() + .register(Metrics.globalRegistry) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt index 7163853d..2d4cccf6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt @@ -19,15 +19,17 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.DefaultUpstream +import io.emeraldpay.dshackle.upstream.MultistreamHolder import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Mono @Service class Describe( - @Autowired private val multistreamHolder: MultistreamHolder, - @Autowired private val subscribeStatus: SubscribeStatus + @Autowired private val multistreamHolder: MultistreamHolder, + @Autowired private val subscribeStatus: SubscribeStatus ) { fun describe(requestMono: Mono): Mono { @@ -39,9 +41,9 @@ class Describe( val targets = chainUpstreams.getMethods().getSupportedMethods() val capabilities: MutableSet = mutableSetOf() val chainDescription = BlockchainOuterClass.DescribeChain.newBuilder() - .setChain(Common.ChainRef.forNumber(chain.id)) - .addAllSupportedMethods(targets) - .setStatus(status) + .setChain(Common.ChainRef.forNumber(chain.id)) + .addAllSupportedMethods(targets) + .setStatus(status) chainUpstreams.getAll().let { ups -> ups.forEach { up -> val nodes = QuorumForLabels() @@ -50,25 +52,27 @@ class Describe( } nodes.getAll().forEach { node -> val nodeDetails = BlockchainOuterClass.NodeDetails.newBuilder() - .setQuorum(node.quorum) - .addAllLabels(node.labels.entries.map { label -> + .setQuorum(node.quorum) + .addAllLabels( + node.labels.entries.map { label -> BlockchainOuterClass.Label.newBuilder() - .setName(label.key) - .setValue(label.value) - .build() - }) + .setName(label.key) + .setValue(label.value) + .build() + } + ) chainDescription.addNodes(nodeDetails) } capabilities.addAll(up.getCapabilities()) } } chainDescription.addAllCapabilities( - capabilities.map { - when (it) { - Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS - Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE - } + capabilities.map { + when (it) { + Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS + Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE } + } ) resp.addChains(chainDescription.build()) } @@ -76,5 +80,4 @@ class Describe( resp.build() } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/EthereumAddresses.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/EthereumAddresses.kt index 2c0dbbe7..917c5ec7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/EthereumAddresses.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/EthereumAddresses.kt @@ -17,12 +17,11 @@ class EthereumAddresses { Flux.just(Address.from(addresses.addressSingle.address)) Common.AnyAddress.AddrTypeCase.ADDRESS_MULTI -> Flux.fromIterable(addresses.addressMulti.addressesList) - .map { Address.from(it.address) } + .map { Address.from(it.address) } else -> { log.error("Unsupported address type: ${addresses.addrTypeCase}") Flux.empty() } } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt index 788ab66a..b638b079 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt @@ -21,33 +21,35 @@ import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.SilentException -import io.emeraldpay.dshackle.upstream.* -import io.emeraldpay.dshackle.quorum.AlwaysQuorum import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.quorum.NotLaggingQuorum import io.emeraldpay.dshackle.quorum.QuorumReaderFactory +import io.emeraldpay.dshackle.upstream.ApiSource +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.calls.EthereumCallSelector import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcException import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.grpc.BlockchainType -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.rpc.RpcException import io.emeraldpay.etherjar.rpc.RpcResponseError +import io.emeraldpay.grpc.BlockchainType +import io.emeraldpay.grpc.Chain import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import reactor.core.publisher.* +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.toMono -import java.lang.Exception -import java.util.* +import java.util.EnumMap @Service open class NativeCall( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) { private val log = LoggerFactory.getLogger(NativeCall::class.java) @@ -69,19 +71,19 @@ open class NativeCall( open fun nativeCall(requestMono: Mono): Flux { return nativeCallResult(requestMono) - .map(this::buildResponse) - .onErrorResume(this::processException) + .map(this::buildResponse) + .onErrorResume(this::processException) } open fun nativeCallResult(requestMono: Mono): Flux { return requestMono.flatMapMany(this::prepareCall) - .map(this::parseParams) - .parallel() - .flatMap { - this.fetch(it) - .doOnError { e -> log.warn("Error during native call: ${e.message}") } - } - .sequential() + .map(this::parseParams) + .parallel() + .flatMap { + this.fetch(it) + .doOnError { e -> log.warn("Error during native call: ${e.message}") } + } + .sequential() } fun parseParams(it: CallContext): CallContext { @@ -91,14 +93,14 @@ open class NativeCall( fun buildResponse(it: CallResult): BlockchainOuterClass.NativeCallReplyItem { val result = BlockchainOuterClass.NativeCallReplyItem.newBuilder() - .setSucceed(!it.isError()) - .setId(it.id) + .setSucceed(!it.isError()) + .setId(it.id) if (it.isError()) { it.error?.let { error -> result.setErrorMessage(error.message) } } else { - result.setPayload(ByteString.copyFrom(it.result)) + result.payload = ByteString.copyFrom(it.result) } return result.build() @@ -112,11 +114,11 @@ open class NativeCall( 0 } return BlockchainOuterClass.NativeCallReplyItem.newBuilder() - .setSucceed(false) - .setErrorMessage(it?.message ?: "Internal error") - .setId(id) - .build() - .toMono() + .setSucceed(false) + .setErrorMessage(it?.message ?: "Internal error") + .setId(id) + .build() + .toMono() } fun prepareCall(request: BlockchainOuterClass.NativeCallRequest): Flux> { @@ -130,31 +132,35 @@ open class NativeCall( } val upstream = multistreamHolder.getUpstream(chain) - ?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain))) + ?: return Flux.error(CallFailure(0, SilentException.UnsupportedBlockchain(chain))) return prepareCall(request, upstream) } - fun prepareCall(request: BlockchainOuterClass.NativeCallRequest, upstream: Multistream): Flux> { + fun prepareCall( + request: BlockchainOuterClass.NativeCallRequest, + upstream: Multistream + ): Flux> { val chain = Chain.byId(request.chainValue) return Flux.fromIterable(request.itemsList).flatMap { val method = it.method val params = it.payload.toStringUtf8() // for ethereum the actual block needed for the call may be specified in the call parameters - val callSpecificMatcher: Mono = if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) { - ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead()) - } else { - null - } ?: Mono.empty() + val callSpecificMatcher: Mono = + if (BlockchainType.from(upstream.chain) == BlockchainType.ETHEREUM) { + ethereumCallSelectors[chain]?.getMatcher(method, params, upstream.getHead()) + } else { + null + } ?: Mono.empty() callSpecificMatcher.defaultIfEmpty(Selector.empty).map { csm -> val matcher = Selector.Builder() - .withMatcher(csm) - .forMethod(method) - .forLabels(Selector.convertToMatcher(request.selector)) + .withMatcher(csm) + .forMethod(method) + .forLabels(Selector.convertToMatcher(request.selector)) - val callQuorum = upstream.getMethods().getQuorumFor(method) ?: AlwaysQuorum() // can be null in tests + val callQuorum = upstream.getMethods().getQuorumFor(method) // can be null in tests callQuorum.init(upstream.getHead()) // for NotLaggingQuorum it makes sense to select compatible upstreams before the call @@ -172,22 +178,22 @@ open class NativeCall( fun fetch(ctx: CallContext): Mono { return ctx.upstream.getRoutedApi(ctx.matcher) - .flatMap { api -> - api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params)) - .flatMap(JsonRpcResponse::requireResult) - .map { - CallResult.ok(ctx.id, it) - } - }.switchIfEmpty( - Mono.just(ctx).flatMap(this::executeOnRemote) - ) - .onErrorResume { - if (it is CallFailure) { - Mono.just(CallResult.fail(it.id, it.reason)) - } else { - Mono.just(CallResult.fail(ctx.id, it)) + .flatMap { api -> + api.read(JsonRpcRequest(ctx.payload.method, ctx.payload.params)) + .flatMap(JsonRpcResponse::requireResult) + .map { + CallResult.ok(ctx.id, it) } + }.switchIfEmpty( + Mono.just(ctx).flatMap(this::executeOnRemote) + ) + .onErrorResume { + if (it is CallFailure) { + Mono.just(CallResult.fail(it.id, it.reason)) + } else { + Mono.just(CallResult.fail(ctx.id, it)) } + } } fun executeOnRemote(ctx: CallContext): Mono { @@ -196,21 +202,21 @@ open class NativeCall( } val reader = quorumReaderFactory.create(ctx.getApis(), ctx.callQuorum) return reader - .read(JsonRpcRequest(ctx.payload.method, ctx.payload.params)) - .map { - CallResult(ctx.id, it.value, null) + .read(JsonRpcRequest(ctx.payload.method, ctx.payload.params)) + .map { + CallResult(ctx.id, it.value, null) + } + .onErrorResume { t -> + val failure = if (t is CallFailure) { + CallResult.fail(t.id, t.reason) + } else { + CallResult.fail(ctx.id, t) } - .onErrorResume { t -> - val failure = if (t is CallFailure) { - CallResult.fail(t.id, t.reason) - } else { - CallResult.fail(ctx.id, t) - } - Mono.just(failure) - } - .switchIfEmpty( - Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}")) - ) + Mono.just(failure) + } + .switchIfEmpty( + Mono.just(CallResult.fail(ctx.id, 1, "No response or no available upstream for ${ctx.payload.method}")) + ) } @Suppress("UNCHECKED_CAST") @@ -222,11 +228,13 @@ open class NativeCall( return req as List } - open class CallContext(val id: Int, - val upstream: Multistream, - val matcher: Selector.Matcher, - val callQuorum: CallQuorum, - val payload: T) { + open class CallContext( + val id: Int, + val upstream: Multistream, + val matcher: Selector.Matcher, + val callQuorum: CallQuorum, + val payload: T + ) { fun withPayload(payload: X): CallContext { return CallContext(id, upstream, matcher, callQuorum, payload) } @@ -273,4 +281,4 @@ open class NativeCall( class RawCallDetails(val method: String, val params: String) class ParsedCallDetails(val method: String, val params: List) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt index 5f2536d8..fd8ffbe2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt @@ -34,7 +34,7 @@ import reactor.core.publisher.Mono @Service class NativeSubscribe( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) { companion object { @@ -45,9 +45,9 @@ class NativeSubscribe( fun nativeSubscribe(request: Mono): Flux { return request - .flatMapMany(this@NativeSubscribe::start) - .map(this@NativeSubscribe::convertToProto) - .onErrorMap(this@NativeSubscribe::convertToStatus) + .flatMapMany(this@NativeSubscribe::start) + .map(this@NativeSubscribe::convertToProto) + .onErrorMap(this@NativeSubscribe::convertToStatus) } fun start(it: BlockchainOuterClass.NativeSubscribeRequest): Publisher { @@ -68,15 +68,15 @@ class NativeSubscribe( fun convertToStatus(t: Throwable) = when (t) { is SilentException.UnsupportedBlockchain -> StatusException( - Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}") + Status.UNAVAILABLE.withDescription("BLOCKCHAIN UNAVAILABLE: ${t.blockchainId}") ) is UnsupportedOperationException -> StatusException( - Status.UNIMPLEMENTED.withDescription(t.message) + Status.UNIMPLEMENTED.withDescription(t.message) ) else -> { log.warn("Unhandled error", t) StatusException( - Status.INTERNAL.withDescription(t.message) + Status.INTERNAL.withDescription(t.message) ) } } @@ -84,15 +84,14 @@ class NativeSubscribe( fun subscribe(chain: Chain, method: String, params: Any?): Flux { val up = multistreamHolder.getUpstream(chain) ?: return Flux.error(SilentException.UnsupportedBlockchain(chain)) return (up as EthereumMultistream) - .getSubscribe() - .subscribe(method, params) + .getSubscribe() + .subscribe(method, params) } fun convertToProto(value: Any): BlockchainOuterClass.NativeSubscribeReplyItem { val result = objectMapper.writeValueAsBytes(value) return BlockchainOuterClass.NativeSubscribeReplyItem.newBuilder() - .setPayload(ByteString.copyFrom(result)) - .build() + .setPayload(ByteString.copyFrom(result)) + .build() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt index f60cb81a..9aad6002 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt @@ -30,7 +30,7 @@ import reactor.core.publisher.Mono @Service class StreamHead( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) { private val log = LoggerFactory.getLogger(StreamHead::class.java) @@ -40,24 +40,23 @@ class StreamHead( Chain.byId(request.type.number) }.flatMapMany { chain -> val up = multistreamHolder.getUpstream(chain) - ?: return@flatMapMany Flux.error(Exception("Unavailable chain: $chain")) + ?: return@flatMapMany Flux.error(Exception("Unavailable chain: $chain")) up.getHead() - .getFlux() - .map { asProto(chain, it!!) } - .onErrorContinue { t, _ -> - log.warn("Head subscription error: ${t.message}") - } + .getFlux() + .map { asProto(chain, it!!) } + .onErrorContinue { t, _ -> + log.warn("Head subscription error: ${t.message}") + } } } fun asProto(chain: Chain, block: BlockContainer): BlockchainOuterClass.ChainHead { return BlockchainOuterClass.ChainHead.newBuilder() - .setChainValue(chain.id) - .setHeight(block.height) - .setTimestamp(block.timestamp.toEpochMilli()) - .setWeight(ByteString.copyFrom(block.difficulty.toByteArray())) - .setBlockId(block.hash.toHex()) - .build() + .setChainValue(chain.id) + .setHeight(block.height) + .setTimestamp(block.timestamp.toEpochMilli()) + .setWeight(ByteString.copyFrom(block.difficulty.toByteArray())) + .setBlockId(block.hash.toHex()) + .build() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt index dbe9aca1..74ed1b27 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeStatus.kt @@ -18,7 +18,9 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.MultistreamHolder +import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.grpc.Chain import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -27,7 +29,7 @@ import reactor.core.publisher.Mono @Service class SubscribeStatus( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) { fun subscribeStatus(requestMono: Mono): Flux { @@ -52,10 +54,10 @@ class SubscribeStatus( fun chainUnavailable(chain: Chain): BlockchainOuterClass.ChainStatus { return BlockchainOuterClass.ChainStatus.newBuilder() - .setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE) - .setChain(Common.ChainRef.forNumber(chain.id)) - .setQuorum(0) - .build() + .setAvailability(BlockchainOuterClass.AvailabilityEnum.AVAIL_UNAVAILABLE) + .setChain(Common.ChainRef.forNumber(chain.id)) + .setQuorum(0) + .build() } fun chainStatus(chain: Chain, available: UpstreamAvailability, ups: Multistream): BlockchainOuterClass.ChainStatus { @@ -67,12 +69,11 @@ class SubscribeStatus( 0 } return BlockchainOuterClass.ChainStatus.newBuilder() - .setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId)) - .setChain(Common.ChainRef.forNumber(chain.id)) - .setQuorum(quorum) - .build() + .setAvailability(BlockchainOuterClass.AvailabilityEnum.forNumber(available.grpcId)) + .setChain(Common.ChainRef.forNumber(chain.id)) + .setQuorum(quorum) + .build() } class ChainSubscription(val chain: Chain, val up: Multistream, val avail: UpstreamAvailability) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackAddress.kt index a841626c..df7d0210 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackAddress.kt @@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.grpc.Chain import reactor.core.publisher.Flux -import reactor.core.publisher.Mono /** * Base interface to tracking balance on a single blockchain @@ -28,5 +27,4 @@ interface TrackAddress { fun isSupported(chain: Chain, asset: String): Boolean fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt index 13aaea5d..31490ee9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinAddress.kt @@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.ReactorBlockchainGrpc -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.upstream.Capability @@ -27,6 +26,7 @@ import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.dshackle.upstream.grpc.BitcoinGrpcUpstream +import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain import org.apache.commons.lang3.StringUtils import org.bitcoinj.params.MainNetParams @@ -37,14 +37,12 @@ import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.math.BigInteger -import java.time.Duration import java.util.concurrent.ConcurrentHashMap import javax.annotation.PostConstruct -import kotlin.collections.HashMap @Service class TrackBitcoinAddress( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) : TrackAddress { companion object { @@ -52,8 +50,8 @@ class TrackBitcoinAddress( } override fun isSupported(chain: Chain, asset: String): Boolean { - return (asset == "bitcoin" || asset == "btc" || asset == "satoshi") - && BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain) + return (asset == "bitcoin" || asset == "btc" || asset == "satoshi") && + BlockchainType.from(chain) == BlockchainType.BITCOIN && multistreamHolder.isAvailable(chain) } /** @@ -65,8 +63,8 @@ class TrackBitcoinAddress( * Criteria for a remote grpc upstream that can provide a balance */ private val balanceUpstreamMatcher = Selector.LocalAndMatcher( - Selector.GrpcMatcher(), - Selector.CapabilityMatcher(Capability.BALANCE) + Selector.GrpcMatcher(), + Selector.CapabilityMatcher(Capability.BALANCE) ) @PostConstruct @@ -99,7 +97,7 @@ class TrackBitcoinAddress( return when { request.address.hasAddressXpub() -> { val xpubAddresses = api.getXpubAddresses() - ?: return Flux.error(IllegalStateException("Xpub verification is not available")) + ?: return Flux.error(IllegalStateException("Xpub verification is not available")) val addressXpub = request.address.addressXpub if (StringUtils.isEmpty(addressXpub.xpub)) { @@ -109,45 +107,52 @@ class TrackBitcoinAddress( val start = Math.max(0, addressXpub.start).toInt() val limit = Math.min(100, Math.max(1, addressXpub.limit)).toInt() xpubAddresses.activeAddresses(xpub, start, limit) - .map { it.toString() } - .doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") } + .map { it.toString() } + .doOnError { t -> log.error("Failed to process xpub. ${t.javaClass}:${t.message}") } } request.address.hasAddressSingle() -> { Flux.just(request.address.addressSingle.address) } request.address.hasAddressMulti() -> { Flux.fromIterable( - request.address.addressMulti.addressesList - .map { addr -> addr.address } - //TODO why sorted? - .sorted() + request.address.addressMulti.addressesList + .map { addr -> addr.address } + // TODO why sorted? + .sorted() ) } else -> Flux.error(IllegalArgumentException("Unsupported address type")) } } - fun requestBalances(chain: Chain, api: BitcoinMultistream, addresses: Flux, includeUtxo: Boolean): Flux { + fun requestBalances( + chain: Chain, + api: BitcoinMultistream, + addresses: Flux, + includeUtxo: Boolean + ): Flux { return addresses - .map { Address(chain, it) } - .flatMap { address -> - balanceForAddress(api, address, includeUtxo) - } + .map { Address(chain, it) } + .flatMap { address -> + balanceForAddress(api, address, includeUtxo) + } } fun balanceForAddress(api: BitcoinMultistream, address: Address, includeUtxo: Boolean): Mono { return api.getReader() - .listUnspent(address.bitcoinAddress) - .map { unspent -> - totalUnspent(address, includeUtxo, unspent) - } - .switchIfEmpty(Mono.just(0).map { + .listUnspent(address.bitcoinAddress) + .map { unspent -> + totalUnspent(address, includeUtxo, unspent) + } + .switchIfEmpty( + Mono.just(0).map { AddressBalance(address, BigInteger.ZERO) - }) - .onErrorResume { t -> - log.error("Failed to get unspent", t) - Mono.empty() } + ) + .onErrorResume { t -> + log.error("Failed to get unspent", t) + Mono.empty() + } } fun totalUnspent(address: Address, includeUtxo: Boolean, unspent: List): AddressBalance { @@ -156,10 +161,10 @@ class TrackBitcoinAddress( } else { unspent.map { AddressBalance( - address, - BigInteger.valueOf(it.value), - if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value)) - else emptyList() + address, + BigInteger.valueOf(it.value), + if (includeUtxo) listOf(BalanceUtxo(it.txid, it.vout, it.value)) + else emptyList() ) }.reduce { a, b -> a.plus(b) } } @@ -169,26 +174,32 @@ class TrackBitcoinAddress( val ups = api.getApiSource(balanceUpstreamMatcher) ups.request(1) return Mono.from(ups) - .map { up -> - up.cast(BitcoinGrpcUpstream::class.java).remote - } - .timeout(Defaults.timeoutInternal, Mono.empty()) - .switchIfEmpty( - Mono.just(0) - .doOnNext { - log.warn("No upstream providing balance for ${api.chain}") - } - .then(Mono.error(SilentException.DataUnavailable("BALANCE"))) - ) + .map { up -> + up.cast(BitcoinGrpcUpstream::class.java).remote + } + .timeout(Defaults.timeoutInternal, Mono.empty()) + .switchIfEmpty( + Mono.just(0) + .doOnNext { + log.warn("No upstream providing balance for ${api.chain}") + } + .then(Mono.error(SilentException.DataUnavailable("BALANCE"))) + ) } - fun getRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux { + fun getRemoteBalance( + api: BitcoinMultistream, + request: BlockchainOuterClass.BalanceRequest + ): Flux { return getBalanceGrpc(api).flatMapMany { remote -> remote.getBalance(request) } } - fun subscribeRemoteBalance(api: BitcoinMultistream, request: BlockchainOuterClass.BalanceRequest): Flux { + fun subscribeRemoteBalance( + api: BitcoinMultistream, + request: BlockchainOuterClass.BalanceRequest + ): Flux { return getBalanceGrpc(api).flatMapMany { remote -> remote.subscribeBalance(request) } @@ -197,43 +208,43 @@ class TrackBitcoinAddress( override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux { val chain = Chain.byId(request.asset.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) - ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) + ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) return if (isBalanceAvailable(chain)) { val addresses = allAddresses(upstream, request) requestBalances(chain, upstream, addresses, request.includeUtxo) - .map(this@TrackBitcoinAddress::buildResponse) - .doOnError { t -> - log.error("Failed to get balance", t) - } + .map(this@TrackBitcoinAddress::buildResponse) + .doOnError { t -> + log.error("Failed to get balance", t) + } } else { getRemoteBalance(upstream, request) - .doOnError { t -> - log.error("Failed to get balance from remote", t) - } + .doOnError { t -> + log.error("Failed to get balance from remote", t) + } } } override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux { val chain = Chain.byId(request.asset.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) - ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) + ?: return Flux.error(SilentException.UnsupportedBlockchain(request.asset.chainValue)) if (isBalanceAvailable(chain)) { val addresses = allAddresses(upstream, request).cache() val following = upstream.getHead().getFlux() - .flatMap { - requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo) - } + .flatMap { + requestBalances(chain, upstream, Flux.from(addresses), request.includeUtxo) + } val last = HashMap() val result = following - .filter { curr -> - val prev = last[curr.address.address] - //TODO utxo can change without changing balance - val changed = prev == null || curr.balance != prev - if (changed) { - last[curr.address.address] = curr.balance - } - changed + .filter { curr -> + val prev = last[curr.address.address] + // TODO utxo can change without changing balance + val changed = prev == null || curr.balance != prev + if (changed) { + last[curr.address.address] = curr.balance } + changed + } return result.map(this@TrackBitcoinAddress::buildResponse) } else { @@ -243,24 +254,30 @@ class TrackBitcoinAddress( private fun buildResponse(address: AddressBalance): BlockchainOuterClass.AddressBalance { return BlockchainOuterClass.AddressBalance.newBuilder() - .setBalance(address.balance.toString(10)) - .setAsset(Common.Asset.newBuilder() - .setChainValue(address.address.chain.id) - .setCode("BTC")) - .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address)) - .addAllUtxo( - address.utxo.map { utxo -> - BlockchainOuterClass.Utxo.newBuilder() - .setBalance(utxo.value.toString()) - .setIndex(utxo.vout.toLong()) - .setTxId(utxo.txid) - .build() - } - ) - .build() + .setBalance(address.balance.toString(10)) + .setAsset( + Common.Asset.newBuilder() + .setChainValue(address.address.chain.id) + .setCode("BTC") + ) + .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.address)) + .addAllUtxo( + address.utxo.map { utxo -> + BlockchainOuterClass.Utxo.newBuilder() + .setBalance(utxo.value.toString()) + .setIndex(utxo.vout.toLong()) + .setTxId(utxo.txid) + .build() + } + ) + .build() } - open class AddressBalance(val address: Address, var balance: BigInteger = BigInteger.ZERO, var utxo: List = emptyList()) { + open class AddressBalance( + val address: Address, + var balance: BigInteger = BigInteger.ZERO, + var utxo: List = emptyList() + ) { constructor(chain: Chain, address: String, balance: BigInteger) : this(Address(chain, address), balance) fun plus(other: AddressBalance) = AddressBalance(address, balance + other.balance, utxo.plus(other.utxo)) @@ -268,7 +285,7 @@ class TrackBitcoinAddress( open class BalanceUtxo(val txid: String, val vout: Int, val value: Long) - //TODO use bitcoin class for address + // TODO use bitcoin class for address class Address(val chain: Chain, val address: String) { val network = if (chain == Chain.BITCOIN) { MainNetParams() @@ -276,7 +293,7 @@ class TrackBitcoinAddress( TestNet3Params() } val bitcoinAddress = org.bitcoinj.core.Address.fromString( - network, address + network, address ) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinTx.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinTx.kt index ca142b50..ca15e72e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinTx.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackBitcoinTx.kt @@ -18,11 +18,11 @@ package io.emeraldpay.dshackle.rpc import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock +import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -37,7 +37,7 @@ import kotlin.math.min @Service class TrackBitcoinTx( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) : TrackTx { companion object { @@ -51,61 +51,69 @@ class TrackBitcoinTx( override fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux { val chain = Chain.byId(request.chainValue) val upstream = multistreamHolder.getUpstream(chain)?.cast(BitcoinMultistream::class.java) - ?: return Flux.error(SilentException.UnsupportedBlockchain(chain)) + ?: return Flux.error(SilentException.UnsupportedBlockchain(chain)) val txid = request.txId val confirmations = max(min(1, request.confirmationLimit), 12) return subscribe(chain, upstream, txid) - .takeUntil { tx -> - tx.confirmations >= confirmations - }.map(this::asProto) + .takeUntil { tx -> + tx.confirmations >= confirmations + }.map(this::asProto) } fun subscribe(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux { return loadExisting(upstream, txid) - .flatMapMany { status -> - if (status.mined) { - //Head almost always knows the current height, so it can continue with calculating confirmations - //without publishing an empty TxStatus first - continueWithMined(upstream, status) - } else { - loadMempool(upstream, txid) - .flatMapMany { tx -> - val next = if (tx.found) { - untilMined(upstream, tx) - } else { - untilFound(chain, upstream, txid) - } - //fist provide the current status, then updates - Flux.concat(Mono.just(tx), next) - } - } + .flatMapMany { status -> + if (status.mined) { + // Head almost always knows the current height, so it can continue with calculating confirmations + // without publishing an empty TxStatus first + continueWithMined(upstream, status) + } else { + loadMempool(upstream, txid) + .flatMapMany { tx -> + val next = if (tx.found) { + untilMined(upstream, tx) + } else { + untilFound(chain, upstream, txid) + } + // fist provide the current status, then updates + Flux.concat(Mono.just(tx), next) + } } + } } fun continueWithMined(upstream: BitcoinMultistream, status: TxStatus): Flux { return upstream.getReader().getBlock(status.blockHash!!) - .map { block -> - TxStatus(status.txid, true, ExtractBlock.getHeight(block), true, status.blockHash, ExtractBlock.getTime(block), ExtractBlock.getDifficulty(block)) - }.flatMapMany { tx -> - withConfirmations(upstream, tx) - } + .map { block -> + TxStatus( + status.txid, + true, + ExtractBlock.getHeight(block), + true, + status.blockHash, + ExtractBlock.getTime(block), + ExtractBlock.getDifficulty(block) + ) + }.flatMapMany { tx -> + withConfirmations(upstream, tx) + } } fun untilFound(chain: Chain, upstream: BitcoinMultistream, txid: String): Flux { return Flux.interval(Duration.ofSeconds(1)) - .take(Duration.ofMinutes(10)) - .flatMap { loadMempool(upstream, txid) } - .skipUntil { it.found } - .flatMap { subscribe(chain, upstream, txid) } - .doOnError { t -> - log.error("Failed to wait until found", t) - } + .take(Duration.ofMinutes(10)) + .flatMap { loadMempool(upstream, txid) } + .skipUntil { it.found } + .flatMap { subscribe(chain, upstream, txid) } + .doOnError { t -> + log.error("Failed to wait until found", t) + } } fun untilMined(upstream: BitcoinMultistream, tx: TxStatus): Mono { return upstream.getHead().getFlux().flatMap { loadExisting(upstream, tx.txid) - .filter { it.mined } + .filter { it.mined } }.single() } @@ -136,34 +144,36 @@ class TrackBitcoinTx( private fun asProto(tx: TxStatus): BlockchainOuterClass.TxStatus { val data = BlockchainOuterClass.TxStatus.newBuilder() - .setTxId(tx.txid) - .setConfirmations(tx.confirmations.toInt()) + .setTxId(tx.txid) + .setConfirmations(tx.confirmations.toInt()) data.broadcasted = tx.found val isMined = tx.mined data.mined = isMined if (isMined) { data.setBlock( - Common.BlockInfo.newBuilder() - .setBlockId(tx.blockHash!!.substring(2)) - .setTimestamp(tx.blockTime!!.toEpochMilli()) - .setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray())) - .setHeight(tx.height!!) + Common.BlockInfo.newBuilder() + .setBlockId(tx.blockHash!!.substring(2)) + .setTimestamp(tx.blockTime!!.toEpochMilli()) + .setWeight(ByteString.copyFrom(tx.blockTotalDifficulty!!.toByteArray())) + .setHeight(tx.height!!) ) } return data.build() } class TxStatus( - val txid: String, - val found: Boolean = false, - val height: Long? = null, - val mined: Boolean = false, - val blockHash: String? = null, - val blockTime: Instant? = null, - val blockTotalDifficulty: BigInteger? = null, - val confirmations: Long = 0) { + val txid: String, + val found: Boolean = false, + val height: Long? = null, + val mined: Boolean = false, + val blockHash: String? = null, + val blockTime: Instant? = null, + val blockTotalDifficulty: BigInteger? = null, + val confirmations: Long = 0 + ) { - fun withHead(headHeight: Long) = TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1) + fun withHead(headHeight: Long) = + TxStatus(txid, found, height, mined, blockHash, blockTime, blockTotalDifficulty, headHeight - height!! + 1) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackERC20Address.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackERC20Address.kt index 4cf53f4f..2a8d0430 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackERC20Address.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackERC20Address.kt @@ -2,7 +2,6 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.config.TokensConfig import io.emeraldpay.dshackle.upstream.Head @@ -11,25 +10,25 @@ import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.erc20.ERC20Token import io.emeraldpay.etherjar.hex.Hex32 import io.emeraldpay.etherjar.hex.HexQuantity +import io.emeraldpay.grpc.BlockchainType +import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.math.BigInteger -import java.util.* +import java.util.Locale import javax.annotation.PostConstruct -import kotlin.collections.HashMap @Service class TrackERC20Address( - @Autowired private val multistreamHolder: MultistreamHolder, - @Autowired private val tokensConfig: TokensConfig + @Autowired private val multistreamHolder: MultistreamHolder, + @Autowired private val tokensConfig: TokensConfig ) : TrackAddress { companion object { @@ -46,8 +45,8 @@ class TrackERC20Address( val asset = token.name!!.lowercase(Locale.getDefault()) val id = TokenId(chain, asset) val definition = TokenDefinition( - chain, asset, - ERC20Token(Address.from(token.address)) + chain, asset, + ERC20Token(Address.from(token.address)) ) tokens[id] = definition log.info("Enable ERC20 balance for $chain:$asset") @@ -56,7 +55,7 @@ class TrackERC20Address( override fun isSupported(chain: Chain, asset: String): Boolean { return tokens.containsKey(TokenId(chain, asset.lowercase(Locale.getDefault()))) && - BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain) + BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain) } override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux { @@ -64,9 +63,9 @@ class TrackERC20Address( val asset = request.asset.code.lowercase(Locale.getDefault()) val tokenDefinition = tokens[TokenId(chain, asset)] ?: return Flux.empty() return ethereumAddresses.extract(request.address) - .map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) } - .flatMap { addr -> getBalance(addr).map(addr::withBalance) } - .map { buildResponse(it) } + .map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) } + .flatMap { addr -> getBalance(addr).map(addr::withBalance) } + .map { buildResponse(it) } } override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux { @@ -76,63 +75,65 @@ class TrackERC20Address( val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty() return ethereumAddresses.extract(request.address) - .map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) } - .flatMap { addr -> - val current = getBalance(addr) - val updates = head.flatMap { getBalance(addr) } - Flux.concat(current, updates) - .distinctUntilChanged() - .map { addr.withBalance(it) } - } - .map { buildResponse(it) } + .map { TrackedAddress(chain, it, tokenDefinition.token, tokenDefinition.name) } + .flatMap { addr -> + val current = getBalance(addr) + val updates = head.flatMap { getBalance(addr) } + Flux.concat(current, updates) + .distinctUntilChanged() + .map { addr.withBalance(it) } + } + .map { buildResponse(it) } } fun getBalance(addr: TrackedAddress): Mono { val upstream = getUpstream(addr.chain) return upstream - .getDirectApi(Selector.empty) - .flatMap { api -> - api.read(prepareEthCall(addr.token, addr.address, upstream.getHead())) - .flatMap(JsonRpcResponse::requireStringResult) - .map { - Hex32.from(it).asQuantity().value - } - } + .getDirectApi(Selector.empty) + .flatMap { api -> + api.read(prepareEthCall(addr.token, addr.address, upstream.getHead())) + .flatMap(JsonRpcResponse::requireStringResult) + .map { + Hex32.from(it).asQuantity().value + } + } } fun prepareEthCall(token: ERC20Token, target: Address, head: Head): JsonRpcRequest { val call = token - .readBalanceOf(target) - .toJson() + .readBalanceOf(target) + .toJson() val height = head.getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest" return JsonRpcRequest("eth_call", listOf(call, height)) } fun getUpstream(chain: Chain): EthereumMultistream { return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java) - ?: throw SilentException.UnsupportedBlockchain(chain) + ?: throw SilentException.UnsupportedBlockchain(chain) } private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance { return BlockchainOuterClass.AddressBalance.newBuilder() - .setBalance(address.balance!!.toString(10)) - .setAsset(Common.Asset.newBuilder() - .setChainValue(address.chain.id) - .setCode(address.tokenName.uppercase(Locale.getDefault()))) - .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex())) - .build() + .setBalance(address.balance!!.toString(10)) + .setAsset( + Common.Asset.newBuilder() + .setChainValue(address.chain.id) + .setCode(address.tokenName.uppercase(Locale.getDefault())) + ) + .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex())) + .build() } - class TrackedAddress(val chain: Chain, - val address: Address, - val token: ERC20Token, - val tokenName: String, - val balance: BigInteger? = null + class TrackedAddress( + val chain: Chain, + val address: Address, + val token: ERC20Token, + val tokenName: String, + val balance: BigInteger? = null ) { fun withBalance(balance: BigInteger) = TrackedAddress(chain, address, token, tokenName, balance) } data class TokenId(val chain: Chain, val name: String) data class TokenDefinition(val chain: Chain, val name: String, val token: ERC20Token) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt index 5d641f46..c9256b3c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumAddress.kt @@ -18,24 +18,24 @@ package io.emeraldpay.dshackle.rpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.domain.Wei +import io.emeraldpay.grpc.BlockchainType +import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import java.util.* +import java.util.Locale @Service class TrackEthereumAddress( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) : TrackAddress { private val log = LoggerFactory.getLogger(TrackEthereumAddress::class.java) @@ -43,46 +43,46 @@ class TrackEthereumAddress( override fun isSupported(chain: Chain, asset: String): Boolean { return asset == "ether" && - BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain) + BlockchainType.from(chain) == BlockchainType.ETHEREUM && multistreamHolder.isAvailable(chain) } override fun getBalance(request: BlockchainOuterClass.BalanceRequest): Flux { return initAddress(request) - .flatMap { a -> getBalance(a).map { a.withBalance(it) } } - .map { buildResponse(it) } + .flatMap { a -> getBalance(a).map { a.withBalance(it) } } + .map { buildResponse(it) } } override fun subscribe(request: BlockchainOuterClass.BalanceRequest): Flux { val chain = Chain.byId(request.asset.chainValue) val head = multistreamHolder.getUpstream(chain)?.getHead()?.getFlux() ?: Flux.empty() val balances = initAddress(request) - .flatMap { tracked -> - val current = getBalance(tracked) - .map { - tracked.withBalance(it) - } - val updates = head - .flatMap { - getBalance(tracked) - }.map { - tracked.withBalance(it) - } - - Flux.concat(current, updates) - .distinctUntilChanged { - it.balance ?: Wei.ZERO - } - } - .doOnError { t -> - if (t is SilentException) { - if (t is SilentException.UnsupportedBlockchain) { - log.warn("Unsupported blockchain: ${t.blockchainId}") - } - log.debug("Failed to process subscription", t) - } else { - log.warn("Failed to process subscription", t) + .flatMap { tracked -> + val current = getBalance(tracked) + .map { + tracked.withBalance(it) } + val updates = head + .flatMap { + getBalance(tracked) + }.map { + tracked.withBalance(it) + } + + Flux.concat(current, updates) + .distinctUntilChanged { + it.balance ?: Wei.ZERO + } + } + .doOnError { t -> + if (t is SilentException) { + if (t is SilentException.UnsupportedBlockchain) { + log.warn("Unsupported blockchain: ${t.blockchainId}") + } + log.debug("Failed to process subscription", t) + } else { + log.warn("Failed to process subscription", t) } + } return balances.map { buildResponse(it) @@ -91,7 +91,7 @@ class TrackEthereumAddress( fun getUpstream(chain: Chain): EthereumMultistream { return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java) - ?: throw SilentException.UnsupportedBlockchain(chain) + ?: throw SilentException.UnsupportedBlockchain(chain) } private fun initAddress(request: BlockchainOuterClass.BalanceRequest): Flux { @@ -110,33 +110,36 @@ class TrackEthereumAddress( private fun createAddress(address: Common.SingleAddress, chain: Chain): TrackedAddress { val addressParsed = Address.from(address.address) return TrackedAddress( - chain, - addressParsed + chain, + addressParsed ) } fun getBalance(addr: TrackedAddress): Mono { return getUpstream(addr.chain) - .getReader() - .balance() - .read(addr.address) - .timeout(Defaults.timeout) + .getReader() + .balance() + .read(addr.address) + .timeout(Defaults.timeout) } private fun buildResponse(address: TrackedAddress): BlockchainOuterClass.AddressBalance { return BlockchainOuterClass.AddressBalance.newBuilder() - .setBalance(address.balance!!.amount!!.toString(10)) - .setAsset(Common.Asset.newBuilder() - .setChainValue(address.chain.id) - .setCode("ETHER")) - .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex())) - .build() + .setBalance(address.balance!!.amount!!.toString(10)) + .setAsset( + Common.Asset.newBuilder() + .setChainValue(address.chain.id) + .setCode("ETHER") + ) + .setAddress(Common.SingleAddress.newBuilder().setAddress(address.address.toHex())) + .build() } - class TrackedAddress(val chain: Chain, - val address: Address, - val balance: Wei? = null + class TrackedAddress( + val chain: Chain, + val address: Address, + val balance: Wei? = null ) { fun withBalance(balance: Wei) = TrackedAddress(chain, address, balance) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt index f0371efe..61a0656d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackEthereumTx.kt @@ -19,19 +19,19 @@ package io.emeraldpay.dshackle.rpc import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.SilentException import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.domain.TransactionId import io.emeraldpay.etherjar.rpc.RpcException import io.emeraldpay.etherjar.rpc.json.BlockJson import io.emeraldpay.etherjar.rpc.json.TransactionJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson +import io.emeraldpay.grpc.BlockchainType +import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -48,7 +48,7 @@ import kotlin.math.min @Service class TrackEthereumTx( - @Autowired private val multistreamHolder: MultistreamHolder + @Autowired private val multistreamHolder: MultistreamHolder ) : TrackTx { companion object { @@ -70,63 +70,62 @@ class TrackEthereumTx( val base = prepareTracking(request) val up = getUpstream(base.chain) return update(base) - .defaultIfEmpty(base) - .flatMapMany { - Flux.concat(Mono.just(it), subscribe(it, up)) - .distinctUntilChanged(TxDetails::status) - .map(this@TrackEthereumTx::asProto) - .subscribeOn(scheduler) - } - .doOnError { t -> - log.error("Subscription error", t) - } + .defaultIfEmpty(base) + .flatMapMany { + Flux.concat(Mono.just(it), subscribe(it, up)) + .distinctUntilChanged(TxDetails::status) + .map(this@TrackEthereumTx::asProto) + .subscribeOn(scheduler) + } + .doOnError { t -> + log.error("Subscription error", t) + } } - fun getUpstream(chain: Chain): EthereumMultistream { return multistreamHolder.getUpstream(chain)?.cast(EthereumMultistream::class.java) - ?: throw SilentException.UnsupportedBlockchain(chain) + ?: throw SilentException.UnsupportedBlockchain(chain) } fun subscribe(base: TxDetails, up: EthereumMultistream): Flux { var latestTx = base val untilFound = Mono.just(latestTx) - .subscribeOn(scheduler) - .map { - //replace with the latest value, it may be already found - latestTx + .subscribeOn(scheduler) + .map { + // replace with the latest value, it may be already found + latestTx + } + .flatMap { latest -> + if (!latest.status.found) { + update(latest).defaultIfEmpty(latestTx) + } else { + Mono.just(latest) } - .flatMap { latest -> - if (!latest.status.found) { - update(latest).defaultIfEmpty(latestTx) - } else { - Mono.just(latest) - } + } + .flatMap { received -> + if (!received.status.found) { + Mono.error(SilentException("Retry not found")) + } else { + Mono.just(received) } - .flatMap { received -> - if (!received.status.found) { - Mono.error(SilentException("Retry not found")) - } else { - Mono.just(received) - } - } - .retryWhen( - Retry.fixedDelay(10, Duration.ofSeconds(2)) - ) - .onErrorResume { Mono.empty() } + } + .retryWhen( + Retry.fixedDelay(10, Duration.ofSeconds(2)) + ) + .onErrorResume { Mono.empty() } val inBlocks = up.getHead().getFlux() - .subscribeOn(scheduler) - .flatMap { block -> - onNewBlock(latestTx, block) - } + .subscribeOn(scheduler) + .flatMap { block -> + onNewBlock(latestTx, block) + } return Flux.merge(untilFound, inBlocks) - .takeUntil(TxDetails::shouldClose) - .doOnNext { newTx -> - latestTx = newTx - } + .takeUntil(TxDetails::shouldClose) + .doOnNext { newTx -> + latestTx = newTx + } } fun onNewBlock(tx: TxDetails, block: BlockContainer): Mono { @@ -134,7 +133,8 @@ class TrackEthereumTx( if (!tx.status.mined) { val justMined = block.transactions.contains(txid) return if (justMined) { - Mono.just(tx.withStatus( + Mono.just( + tx.withStatus( mined = true, found = true, confirmations = 1, @@ -142,13 +142,14 @@ class TrackEthereumTx( blockTime = block.timestamp, blockTotalDifficulty = block.difficulty, blockHash = BlockHash(block.hash.value) - )) + ) + ) } else { update(tx) } } else { - //verify if it's still on chain - //TODO head is supposed to erase block when it was replaced, so can safely recalc here + // verify if it's still on chain + // TODO head is supposed to erase block when it was replaced, so can safely recalc here return update(tx) } } @@ -157,19 +158,19 @@ class TrackEthereumTx( val initialStatus = tx.status val upstream = getUpstream(tx.chain) return upstream.getReader() - .txByHash().read(tx.txid) - .onErrorResume(RpcException::class.java) { t -> - log.warn("Upstream error, ignoring. {}", t.rpcMessage) - Mono.empty() - } - .flatMap { updateFromBlock(upstream, tx, it) } - .doOnError { t -> - log.error("Failed to load tx block", t) - } - .switchIfEmpty(Mono.just(tx.withStatus(found = false))) - .filter { current -> - initialStatus != current.status || current.shouldClose() - } + .txByHash().read(tx.txid) + .onErrorResume(RpcException::class.java) { t -> + log.warn("Upstream error, ignoring. {}", t.rpcMessage) + Mono.empty() + } + .flatMap { updateFromBlock(upstream, tx, it) } + .doOnError { t -> + log.error("Failed to load tx block", t) + } + .switchIfEmpty(Mono.just(tx.withStatus(found = false))) + .filter { current -> + initialStatus != current.status || current.shouldClose() + } } fun prepareTracking(request: BlockchainOuterClass.TxStatusRequest): TxDetails { @@ -178,10 +179,10 @@ class TrackEthereumTx( throw SilentException.UnsupportedBlockchain(request.chainValue) } val details = TxDetails( - chain, - Instant.now(), - TransactionId.from(request.txId), - min(max(1, request.confirmationLimit), 100) + chain, + Instant.now(), + TransactionId.from(request.txId), + min(max(1, request.confirmationLimit), 100) ) return details } @@ -189,12 +190,12 @@ class TrackEthereumTx( fun setBlockDetails(tx: TxDetails, block: BlockJson): TxDetails { return if (block.number != null && block.totalDifficulty != null) { tx.withStatus( - blockTotalDifficulty = block.totalDifficulty, - blockTime = block.timestamp + blockTotalDifficulty = block.totalDifficulty, + blockTime = block.timestamp ) } else { tx.withStatus( - mined = false + mined = false ) } } @@ -205,22 +206,22 @@ class TrackEthereumTx( return Mono.empty() } return upstream.getReader() - .blocksByHashParsed().read(tx.status.blockHash) - .map { block -> - setBlockDetails(tx, block) - }.doOnError { t -> - log.warn("Failed to update weight", t) - } + .blocksByHashParsed().read(tx.status.blockHash) + .map { block -> + setBlockDetails(tx, block) + }.doOnError { t -> + log.warn("Failed to update weight", t) + } } fun updateFromBlock(upstream: EthereumMultistream, tx: TxDetails, blockTx: TransactionJson): Mono { return if (blockTx.blockNumber != null && blockTx.blockHash != null && blockTx.blockHash != ZERO_BLOCK) { val updated = tx.withStatus( - blockHash = blockTx.blockHash, - height = blockTx.blockNumber, - found = true, - mined = true, - confirmations = 1 + blockHash = blockTx.blockHash, + height = blockTx.blockNumber, + found = true, + mined = true, + confirmations = 1 ) upstream.getHead().getFlux().next().map { head -> val height = updated.status.height @@ -228,72 +229,89 @@ class TrackEthereumTx( updated } else { updated.withStatus( - confirmations = head.height - height + 1 + confirmations = head.height - height + 1 ) } }.doOnError { t -> log.error("Unable to load head details", t) }.flatMap(this::loadWeight) } else { - Mono.just(tx.withStatus( + Mono.just( + tx.withStatus( found = true, mined = false - )) + ) + ) } } private fun asProto(tx: TxDetails): BlockchainOuterClass.TxStatus { val data = BlockchainOuterClass.TxStatus.newBuilder() - .setTxId(tx.txid.toHex()) - .setConfirmations(tx.status.confirmations.toInt()) + .setTxId(tx.txid.toHex()) + .setConfirmations(tx.status.confirmations.toInt()) data.broadcasted = tx.status.found val isMined = tx.status.mined data.mined = isMined if (isMined) { data.setBlock( - Common.BlockInfo.newBuilder() - .setBlockId(tx.status.blockHash!!.toHex().substring(2)) - .setTimestamp(tx.status.blockTime!!.toEpochMilli()) - .setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray())) - .setHeight(tx.status.height!!) + Common.BlockInfo.newBuilder() + .setBlockId(tx.status.blockHash!!.toHex().substring(2)) + .setTimestamp(tx.status.blockTime!!.toEpochMilli()) + .setWeight(ByteString.copyFrom(tx.status.blockTotalDifficulty!!.toByteArray())) + .setHeight(tx.status.height!!) ) } return data.build() } - class TxDetails(val chain: Chain, - val since: Instant, - val txid: TransactionId, - val maxConfirmations: Int, - val status: TxStatus + class TxDetails( + val chain: Chain, + val since: Instant, + val txid: TransactionId, + val maxConfirmations: Int, + val status: TxStatus ) { - constructor(chain: Chain, - since: Instant, - txid: TransactionId, - maxConfirmations: Int) : this(chain, since, txid, maxConfirmations, TxStatus()) + constructor( + chain: Chain, + since: Instant, + txid: TransactionId, + maxConfirmations: Int + ) : this(chain, since, txid, maxConfirmations, TxStatus()) fun copy( - since: Instant = this.since, - status: TxStatus = this.status + since: Instant = this.since, + status: TxStatus = this.status ) = TxDetails(chain, since, txid, maxConfirmations, status) - fun withStatus(found: Boolean = this.status.found, - height: Long? = this.status.height, - mined: Boolean = this.status.mined, - blockHash: BlockHash? = this.status.blockHash, - blockTime: Instant? = this.status.blockTime, - blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty, - confirmations: Long = this.status.confirmations): TxDetails { - return copy(status = this.status.copy(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmations)) + fun withStatus( + found: Boolean = this.status.found, + height: Long? = this.status.height, + mined: Boolean = this.status.mined, + blockHash: BlockHash? = this.status.blockHash, + blockTime: Instant? = this.status.blockTime, + blockTotalDifficulty: BigInteger? = this.status.blockTotalDifficulty, + confirmations: Long = this.status.confirmations + ): TxDetails { + return copy( + status = this.status.copy( + found, + height, + mined, + blockHash, + blockTime, + blockTotalDifficulty, + confirmations + ) + ) } fun shouldClose(): Boolean { - return maxConfirmations <= this.status.confirmations - || since.isBefore(Instant.now().minus(TRACK_TTL)) - || (!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL))) - || (!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL))) + return maxConfirmations <= this.status.confirmations || + since.isBefore(Instant.now().minus(TRACK_TTL)) || + (!status.found && since.isBefore(Instant.now().minus(NOT_FOUND_TRACK_TTL))) || + (!status.mined && since.isBefore(Instant.now().minus(NOT_MINED_TRACK_TTL))) } override fun toString(): String { @@ -320,25 +338,27 @@ class TrackEthereumTx( result = 31 * result + status.hashCode() return result } - } - class TxStatus(val found: Boolean = false, - val height: Long? = null, - val mined: Boolean = false, - val blockHash: BlockHash? = null, - val blockTime: Instant? = null, - val blockTotalDifficulty: BigInteger? = null, - val confirmations: Long = 0) { + class TxStatus( + val found: Boolean = false, + val height: Long? = null, + val mined: Boolean = false, + val blockHash: BlockHash? = null, + val blockTime: Instant? = null, + val blockTotalDifficulty: BigInteger? = null, + val confirmations: Long = 0 + ) { - fun copy(found: Boolean = this.found, - height: Long? = this.height, - mined: Boolean = this.mined, - blockHash: BlockHash? = this.blockHash, - blockTime: Instant? = this.blockTime, - blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty, - confirmation: Long = this.confirmations) - = TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation) + fun copy( + found: Boolean = this.found, + height: Long? = this.height, + mined: Boolean = this.mined, + blockHash: BlockHash? = this.blockHash, + blockTime: Instant? = this.blockTime, + blockTotalDifficulty: BigInteger? = this.blockTotalDifficulty, + confirmation: Long = this.confirmations + ) = TxStatus(found, height, mined, blockHash, blockTime, blockTotalDifficulty, confirmation) fun clean() = TxStatus(false, null, false, null, null, null, 0) @@ -369,7 +389,5 @@ class TrackEthereumTx( override fun toString(): String { return "TxStatus(found=$found, height=$height, mined=$mined, blockHash=$blockHash, blockTime=$blockTime, blockTotalDifficulty=$blockTotalDifficulty, confirmations=$confirmations)" } - - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackTx.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackTx.kt index 757ed0a0..4fe8b3a6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackTx.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/TrackTx.kt @@ -22,4 +22,4 @@ import reactor.core.publisher.Flux interface TrackTx { fun isSupported(chain: Chain): Boolean fun subscribe(request: BlockchainOuterClass.TxStatusRequest): Flux -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt index f2042964..fdf1dadf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/ConfiguredUpstreams.kt @@ -16,7 +16,6 @@ */ package io.emeraldpay.dshackle.startup -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.cache.CachesFactory import io.emeraldpay.dshackle.config.UpstreamsConfig @@ -34,49 +33,48 @@ import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcHttpClient import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics +import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain import io.micrometer.core.instrument.Counter -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Repository import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.Timer +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Repository import java.net.URI -import java.util.* import java.util.concurrent.atomic.AtomicInteger import javax.annotation.PostConstruct -import kotlin.collections.HashMap @Repository open class ConfiguredUpstreams( - @Autowired private val currentUpstreams: CurrentMultistreamHolder, - @Autowired private val fileResolver: FileResolver, - @Autowired private val config: UpstreamsConfig, - @Autowired private val cachesFactory: CachesFactory + @Autowired private val currentUpstreams: CurrentMultistreamHolder, + @Autowired private val fileResolver: FileResolver, + @Autowired private val config: UpstreamsConfig, + @Autowired private val cachesFactory: CachesFactory ) { private val log = LoggerFactory.getLogger(ConfiguredUpstreams::class.java) private var seq = AtomicInteger(0) private val chainNames = mapOf( - "ethereum" to Chain.ETHEREUM, - "ethereum-classic" to Chain.ETHEREUM_CLASSIC, - "eth" to Chain.ETHEREUM, - "polygon" to Chain.MATIC, - "matic" to Chain.MATIC, - "etc" to Chain.ETHEREUM_CLASSIC, - "morden" to Chain.TESTNET_MORDEN, - "kovan" to Chain.TESTNET_KOVAN, - "kovan-testnet" to Chain.TESTNET_KOVAN, - "goerli" to Chain.TESTNET_GOERLI, - "goerli-testnet" to Chain.TESTNET_GOERLI, - "rinkeby" to Chain.TESTNET_RINKEBY, - "rinkeby-testnet" to Chain.TESTNET_RINKEBY, - "ropsten" to Chain.TESTNET_ROPSTEN, - "ropsten-testnet" to Chain.TESTNET_ROPSTEN, - "bitcoin" to Chain.BITCOIN, - "bitcoin-testnet" to Chain.TESTNET_BITCOIN + "ethereum" to Chain.ETHEREUM, + "ethereum-classic" to Chain.ETHEREUM_CLASSIC, + "eth" to Chain.ETHEREUM, + "polygon" to Chain.MATIC, + "matic" to Chain.MATIC, + "etc" to Chain.ETHEREUM_CLASSIC, + "morden" to Chain.TESTNET_MORDEN, + "kovan" to Chain.TESTNET_KOVAN, + "kovan-testnet" to Chain.TESTNET_KOVAN, + "goerli" to Chain.TESTNET_GOERLI, + "goerli-testnet" to Chain.TESTNET_GOERLI, + "rinkeby" to Chain.TESTNET_RINKEBY, + "rinkeby-testnet" to Chain.TESTNET_RINKEBY, + "ropsten" to Chain.TESTNET_ROPSTEN, + "ropsten-testnet" to Chain.TESTNET_ROPSTEN, + "bitcoin" to Chain.BITCOIN, + "bitcoin-testnet" to Chain.TESTNET_BITCOIN ) @PostConstruct @@ -95,7 +93,7 @@ open class ConfiguredUpstreams( return@forEach } val options = (up.options ?: UpstreamsConfig.Options()) - .merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults()) + .merge(defaultOptions[chain] ?: UpstreamsConfig.Options.getDefaults()) when (BlockchainType.from(chain)) { BlockchainType.ETHEREUM -> { buildEthereumUpstream(up.cast(UpstreamsConfig.EthereumConnection::class.java), chain, options) @@ -135,9 +133,10 @@ open class ConfiguredUpstreams( fun buildMethods(config: UpstreamsConfig.Upstream<*>, chain: Chain): CallMethods { return if (config.methods != null) { - ManagedCallMethods(currentUpstreams.getDefaultMethods(chain), - config.methods!!.enabled.map { it.name }.toSet(), - config.methods!!.disabled.map { it.name }.toSet() + ManagedCallMethods( + currentUpstreams.getDefaultMethods(chain), + config.methods!!.enabled.map { it.name }.toSet(), + config.methods!!.disabled.map { it.name }.toSet() ).also { config.methods!!.enabled.forEach { m -> if (m.quorum != null) { @@ -150,9 +149,11 @@ open class ConfiguredUpstreams( } } - private fun buildBitcoinUpstream(config: UpstreamsConfig.Upstream, - chain: Chain, - options: UpstreamsConfig.Options) { + private fun buildBitcoinUpstream( + config: UpstreamsConfig.Upstream, + chain: Chain, + options: UpstreamsConfig.Options + ) { val conn = config.connection!! val directApi: Reader? = buildHttpClient(config) @@ -171,19 +172,24 @@ open class ConfiguredUpstreams( } val methods = buildMethods(config, chain) - val upstream = BitcoinRpcUpstream(config.id - ?: "bitcoin-${seq.getAndIncrement()}", chain, directApi, - options, config.role, - QuorumForLabels.QuorumItem(1, config.labels), - methods, esplora) + val upstream = BitcoinRpcUpstream( + config.id + ?: "bitcoin-${seq.getAndIncrement()}", + chain, directApi, + options, config.role, + QuorumForLabels.QuorumItem(1, config.labels), + methods, esplora + ) upstream.start() currentUpstreams.update(UpstreamChange(chain, upstream, UpstreamChange.ChangeType.ADDED)) } - private fun buildEthereumUpstream(config: UpstreamsConfig.Upstream, - chain: Chain, - options: UpstreamsConfig.Options) { + private fun buildEthereumUpstream( + config: UpstreamsConfig.Upstream, + chain: Chain, + options: UpstreamsConfig.Options + ) { val conn = config.connection!! val urls = ArrayList() @@ -194,8 +200,8 @@ open class ConfiguredUpstreams( val wsFactoryApi: EthereumWsFactory? = conn.ws?.let { endpoint -> val wsApi = EthereumWsFactory( - endpoint.url, - endpoint.origin ?: URI("http://localhost"), + endpoint.url, + endpoint.origin ?: URI("http://localhost"), ) wsApi.config = endpoint endpoint.basicAuth?.let { auth -> @@ -208,11 +214,11 @@ open class ConfiguredUpstreams( log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}") val ethereumUpstream = if (wsFactoryApi != null && !conn.preferHttp) { EthereumWsUpstream( - config.id!!, - chain, wsFactoryApi, - options, config.role, - QuorumForLabels.QuorumItem(1, config.labels), - methods + config.id!!, + chain, wsFactoryApi, + options, config.role, + QuorumForLabels.QuorumItem(1, config.labels), + methods ) } else { val directApi: Reader? = buildHttpClient(config) @@ -221,11 +227,11 @@ open class ConfiguredUpstreams( return } EthereumRpcUpstream( - config.id!!, - chain, directApi, wsFactoryApi, - options, config.role, - QuorumForLabels.QuorumItem(1, config.labels), - methods + config.id!!, + chain, directApi, wsFactoryApi, + options, config.role, + QuorumForLabels.QuorumItem(1, config.labels), + methods ) } @@ -233,23 +239,26 @@ open class ConfiguredUpstreams( currentUpstreams.update(UpstreamChange(chain, ethereumUpstream, UpstreamChange.ChangeType.ADDED)) } - private fun buildGrpcUpstream(config: UpstreamsConfig.Upstream, options: UpstreamsConfig.Options) { + private fun buildGrpcUpstream( + config: UpstreamsConfig.Upstream, + options: UpstreamsConfig.Options + ) { val endpoint = config.connection!! val ds = GrpcUpstreams( - config.id!!, - endpoint.host!!, - endpoint.port ?: 2449, - endpoint.auth, - fileResolver + config.id!!, + endpoint.host!!, + endpoint.port, + endpoint.auth, + fileResolver ).apply { timeout = options.timeout } log.info("Using ALL CHAINS (gRPC) upstream, at ${endpoint.host}:${endpoint.port}") ds.start() - .doOnNext { - log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}") - } - .subscribe(currentUpstreams::update) + .doOnNext { + log.info("Chain ${it.chain} has ${it.type} through gRPC at ${endpoint.host}:${endpoint.port}") + } + .subscribe(currentUpstreams::update) } private fun buildHttpClient(config: UpstreamsConfig.Upstream): JsonRpcHttpClient? { @@ -262,29 +271,29 @@ open class ConfiguredUpstreams( } } val metricsTags = listOf( - // "unknown" is not supposed to happen - Tag.of("upstream", config.id ?: "unknown"), - // UNSPECIFIED shouldn't happen too - Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED ).chainCode) + // "unknown" is not supposed to happen + Tag.of("upstream", config.id ?: "unknown"), + // UNSPECIFIED shouldn't happen too + Tag.of("chain", (chainNames[config.chain ?: ""] ?: Chain.UNSPECIFIED).chainCode) ) val metrics = RpcMetrics( - Timer.builder("upstream.rpc.conn") - .description("Request time through a HTTP JSON RPC connection") - .tags(metricsTags) - .publishPercentileHistogram() - .register(Metrics.globalRegistry), - Counter.builder("upstream.rpc.err") - .description("Errors received on request through HTTP JSON RPC connection") - .tags(metricsTags) - .register(Metrics.globalRegistry) + Timer.builder("upstream.rpc.conn") + .description("Request time through a HTTP JSON RPC connection") + .tags(metricsTags) + .publishPercentileHistogram() + .register(Metrics.globalRegistry), + Counter.builder("upstream.rpc.err") + .description("Errors received on request through HTTP JSON RPC connection") + .tags(metricsTags) + .register(Metrics.globalRegistry) ) urls.add(endpoint.url) JsonRpcHttpClient( - endpoint.url.toString(), - metrics, - conn.rpc?.basicAuth, - tls + endpoint.url.toString(), + metrics, + conn.rpc?.basicAuth, + tls ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/QuorumForLabels.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/QuorumForLabels.kt index c1d5b2da..5b6dc661 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/QuorumForLabels.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/QuorumForLabels.kt @@ -17,9 +17,8 @@ package io.emeraldpay.dshackle.startup import io.emeraldpay.dshackle.config.UpstreamsConfig -import java.util.* +import java.util.Collections import java.util.concurrent.locks.ReentrantReadWriteLock -import kotlin.collections.ArrayList import kotlin.concurrent.read import kotlin.concurrent.write @@ -71,7 +70,6 @@ class QuorumForLabels() { return nodes.hashCode() } - /** * Details for a single element (upstream, node or aggregation) */ @@ -97,8 +95,5 @@ class QuorumForLabels() { result = 31 * result + labels.hashCode() return result } - - } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt index b1855430..213e55a1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/UpstreamChange.kt @@ -25,19 +25,19 @@ import io.emeraldpay.grpc.Chain * An update event to the list of currently available upstreams. */ class UpstreamChange( - /** - * Target blockchain - */ - val chain: Chain, - /** - * Corresponding upstream - */ - val upstream: Upstream, - /** - * Type of the change - */ - val type: ChangeType -): CachesEnabled { + /** + * Target blockchain + */ + val chain: Chain, + /** + * Corresponding upstream + */ + val upstream: Upstream, + /** + * Type of the change + */ + val type: ChangeType +) : CachesEnabled { enum class ChangeType { /** @@ -61,4 +61,4 @@ class UpstreamChange( upstream.setCaches(caches) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt index fc56ff13..9f85e7f6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/AbstractHead.kt @@ -42,37 +42,37 @@ abstract class AbstractHead : Head { completed = false } return source - .distinctUntilChanged { - it.hash - }.filter { block -> - val curr = head.get() - curr == null || curr.difficulty < block.difficulty - } - .doFinally { - // close internal stream if upstream is finished, otherwise it gets stuck, - // but technically it should never happen during normal work, only when the Head - // is stopping - completed = true - stream.tryEmitComplete() - } - .subscribeOn(Schedulers.boundedElastic()) - .subscribe { block -> - notifyBeforeBlock() - val prev = head.getAndUpdate { curr -> - if (curr == null || curr.difficulty < block.difficulty) { - block - } else { - curr - } - } - if (prev == null || prev.hash != block.hash) { - log.debug("New block ${block.height} ${block.hash}") - val result = stream.tryEmitNext(block) - if (result.isFailure && result != Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { - log.warn("Failed to dispatch block: $result as ${this.javaClass}") - } + .distinctUntilChanged { + it.hash + }.filter { block -> + val curr = head.get() + curr == null || curr.difficulty < block.difficulty + } + .doFinally { + // close internal stream if upstream is finished, otherwise it gets stuck, + // but technically it should never happen during normal work, only when the Head + // is stopping + completed = true + stream.tryEmitComplete() + } + .subscribeOn(Schedulers.boundedElastic()) + .subscribe { block -> + notifyBeforeBlock() + val prev = head.getAndUpdate { curr -> + if (curr == null || curr.difficulty < block.difficulty) { + block + } else { + curr } } + if (prev == null || prev.hash != block.hash) { + log.debug("New block ${block.height} ${block.hash}") + val result = stream.tryEmitNext(block) + if (result.isFailure && result != Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { + log.warn("Failed to dispatch block: $result as ${this.javaClass}") + } + } + } } fun notifyBeforeBlock() { @@ -91,8 +91,8 @@ abstract class AbstractHead : Head { override fun getFlux(): Flux { return Flux.concat( - Mono.justOrEmpty(head.get()), - stream.asFlux() + Mono.justOrEmpty(head.get()), + stream.asFlux() ).onBackpressureLatest() } @@ -103,4 +103,4 @@ abstract class AbstractHead : Head { override fun getCurrentHeight(): Long? { return getCurrent()?.height } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ApiSource.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ApiSource.kt index bc014cdf..5f8c0443 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ApiSource.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ApiSource.kt @@ -22,5 +22,4 @@ interface ApiSource : Publisher { fun resolve() fun request(tries: Int) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt index 9805b4b2..3e73a3c2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt @@ -3,4 +3,4 @@ package io.emeraldpay.dshackle.upstream enum class Capability { RPC, BALANCE -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt index 18bc9e3b..f52da7ce 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt @@ -16,26 +16,24 @@ */ package io.emeraldpay.dshackle.upstream -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.cache.CachesFactory import io.emeraldpay.dshackle.startup.UpstreamChange import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream -import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinRpcUpstream import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream -import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods import io.emeraldpay.dshackle.upstream.calls.CallMethods +import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream +import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired -import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Repository import reactor.core.publisher.Flux import reactor.core.publisher.Sinks -import java.util.* +import java.util.Collections import java.util.concurrent.Callable import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantLock @@ -43,15 +41,15 @@ import kotlin.concurrent.withLock @Repository open class CurrentMultistreamHolder( - @Autowired private val cachesFactory: CachesFactory + @Autowired private val cachesFactory: CachesFactory ) : MultistreamHolder { private val log = LoggerFactory.getLogger(CurrentMultistreamHolder::class.java) private val chainMapping = ConcurrentHashMap() private val chainsBus = Sinks.many() - .multicast() - .directBestEffort() + .multicast() + .directBestEffort() private val callTargets = HashMap() private val updateLock = ReentrantLock() @@ -63,17 +61,17 @@ open class CurrentMultistreamHolder( when (BlockchainType.from(chain)) { BlockchainType.ETHEREUM -> { val up = change.upstream.cast(EthereumUpstream::class.java) - val current = chainMapping[chain] as Multistream? - val factory = Callable { - EthereumMultistream(chain, ArrayList(), cachesFactory.getCaches(chain)) as Multistream + val current = chainMapping[chain] + val factory = Callable { + EthereumMultistream(chain, ArrayList(), cachesFactory.getCaches(chain)) } processUpdate(change, up, current, factory) } BlockchainType.BITCOIN -> { val up = change.upstream.cast(BitcoinUpstream::class.java) - val current = chainMapping[chain] as Multistream? - val factory = Callable { - BitcoinMultistream(chain, ArrayList(), cachesFactory.getCaches(chain)) as Multistream + val current = chainMapping[chain] + val factory = Callable { + BitcoinMultistream(chain, ArrayList(), cachesFactory.getCaches(chain)) } processUpdate(change, up, current, factory) } @@ -125,8 +123,8 @@ open class CurrentMultistreamHolder( override fun observeChains(): Flux { return Flux.concat( - Flux.fromIterable(getAvailable()), - chainsBus.asFlux() + Flux.fromIterable(getAvailable()), + chainsBus.asFlux() ) } @@ -147,4 +145,4 @@ open class CurrentMultistreamHolder( override fun isAvailable(chain: Chain): Boolean { return chainMapping.containsKey(chain) && callTargets.containsKey(chain) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt index 6646d85b..b6c1ffdd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt @@ -25,25 +25,44 @@ import reactor.core.publisher.Sinks import java.util.concurrent.atomic.AtomicReference abstract class DefaultUpstream( - private val id: String, - defaultLag: Long, - defaultAvail: UpstreamAvailability, - private val options: UpstreamsConfig.Options, - private val role: UpstreamsConfig.UpstreamRole, - private val targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? + private val id: String, + defaultLag: Long, + defaultAvail: UpstreamAvailability, + private val options: UpstreamsConfig.Options, + private val role: UpstreamsConfig.UpstreamRole, + private val targets: CallMethods?, + private val node: QuorumForLabels.QuorumItem? ) : Upstream { - constructor(id: String, options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?) : - this(id, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, QuorumForLabels.QuorumItem.empty()) + constructor( + id: String, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + targets: CallMethods? + ) : + this( + id, + Long.MAX_VALUE, + UpstreamAvailability.UNAVAILABLE, + options, + role, + targets, + QuorumForLabels.QuorumItem.empty() + ) - constructor(id: String, options: UpstreamsConfig.Options, role: UpstreamsConfig.UpstreamRole, targets: CallMethods?, node: QuorumForLabels.QuorumItem?) : - this(id, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node) + constructor( + id: String, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + targets: CallMethods?, + node: QuorumForLabels.QuorumItem? + ) : + this(id, Long.MAX_VALUE, UpstreamAvailability.UNAVAILABLE, options, role, targets, node) private val status = AtomicReference(Status(defaultLag, defaultAvail, statusByLag(defaultLag, defaultAvail))) private val statusStream = Sinks.many() - .multicast() - .directBestEffort() + .multicast() + .directBestEffort() override fun isAvailable(): Boolean { return getStatus() == UpstreamAvailability.OK @@ -53,7 +72,7 @@ abstract class DefaultUpstream( val available = value.availability val quorum = value.quorum setStatus( - if (available != null) UpstreamAvailability.fromGrpc(available.number) else UpstreamAvailability.UNAVAILABLE + if (available != null) UpstreamAvailability.fromGrpc(available.number) else UpstreamAvailability.UNAVAILABLE ) } @@ -80,7 +99,7 @@ abstract class DefaultUpstream( override fun observeStatus(): Flux { return statusStream.asFlux() - .distinctUntilChanged() + .distinctUntilChanged() } override fun setLag(lag: Long) { @@ -107,7 +126,7 @@ abstract class DefaultUpstream( } override fun getRole(): UpstreamsConfig.UpstreamRole { - return role; + return role } override fun getMethods(): CallMethods { @@ -115,11 +134,11 @@ abstract class DefaultUpstream( } private val quorumByLabel = node?.let { QuorumForLabels(it) } - ?: QuorumForLabels(QuorumForLabels.QuorumItem.empty()) + ?: QuorumForLabels(QuorumForLabels.QuorumItem.empty()) open fun getQuorumByLabel(): QuorumForLabels { return quorumByLabel } class Status(val lag: Long, val avail: UpstreamAvailability, val status: UpstreamAvailability) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt index 14910d83..75d66b27 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/EmptyHead.kt @@ -17,7 +17,6 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.dshackle.data.BlockContainer -import io.emeraldpay.dshackle.upstream.Head import reactor.core.publisher.Flux class EmptyHead : Head { @@ -32,4 +31,4 @@ class EmptyHead : Head { override fun getCurrentHeight(): Long? { return null } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt index c8c47a97..ce015728 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/FilteredApis.kt @@ -26,7 +26,7 @@ import org.reactivestreams.Subscriber import reactor.core.publisher.Flux import reactor.core.publisher.Sinks import java.time.Duration -import java.util.* +import java.util.EnumMap import java.util.concurrent.locks.Lock import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock @@ -37,15 +37,15 @@ import kotlin.math.roundToLong import kotlin.random.Random class FilteredApis( - val chain: Chain, - private val allUpstreams: List, - private val matcher: Selector.Matcher, - private val pos: Int, - /** - * Limit of retries - */ - private val retryLimit: Long, - jitter: Int + val chain: Chain, + private val allUpstreams: List, + private val matcher: Selector.Matcher, + private val pos: Int, + /** + * Limit of retries + */ + private val retryLimit: Long, + jitter: Int ) : ApiSource { companion object { @@ -68,14 +68,18 @@ class FilteredApis( private val metricsSetup: Lock = ReentrantLock() } - constructor(chain: Chain, - allUpstreams: List, - matcher: Selector.Matcher, - pos: Int) : this(chain, allUpstreams, matcher, pos, 10, 7) + constructor( + chain: Chain, + allUpstreams: List, + matcher: Selector.Matcher, + pos: Int + ) : this(chain, allUpstreams, matcher, pos, 10, 7) - constructor(chain: Chain, - allUpstreams: List, - matcher: Selector.Matcher) : this(chain, allUpstreams, matcher, 0, 10, 10) + constructor( + chain: Chain, + allUpstreams: List, + matcher: Selector.Matcher + ) : this(chain, allUpstreams, matcher, 0, 10, 10) private val delay: Int private val standardUpstreams: List @@ -101,8 +105,8 @@ class FilteredApis( startFrom(it, pos) } standardWithFallback = emptyList() - .plus(standardUpstreams) - .plus(fallbackUpstreams) + .plus(standardUpstreams) + .plus(fallbackUpstreams) if (Global.metricsExtended) { getMetrics(chain).let { monitoring -> @@ -133,8 +137,8 @@ class FilteredApis( fun waitDuration(rawn: Long): Duration { val n = max(rawn, 1) val time = min( - (n.toDouble().pow(2.0) * delay).roundToLong(), - MAX_WAIT_MILLIS + (n.toDouble().pow(2.0) * delay).roundToLong(), + MAX_WAIT_MILLIS ) return Duration.ofMillis(time) } @@ -145,10 +149,10 @@ class FilteredApis( // if all failed, try both standard and fallback upstreams, repeating in cycle val retries = (0 until (retryLimit - 1)).map { r -> Flux.fromIterable(standardWithFallback) - // add delay to let upstream to restore if it's a temp failure - // but delay only start of the check, not between upstreams - // i.e. if all upstreams failed -> wait -> check all without waiting in between - .delaySubscription(waitDuration(r + 1)) + // add delay to let upstream to restore if it's a temp failure + // but delay only start of the check, not between upstreams + // i.e. if all upstreams failed -> wait -> check all without waiting in between + .delaySubscription(waitDuration(r + 1)) }.let { Flux.concat(it) } var result = Flux.concat(first, retries) @@ -156,14 +160,14 @@ class FilteredApis( if (Global.metricsExtended) { var count = 0 result = result - .doOnNext { count++ } - .doFinally { metrics[chain]?.tried?.record(count.toDouble()) } + .doOnNext { count++ } + .doFinally { metrics[chain]?.tried?.record(count.toDouble()) } } result.filter { up -> up.isAvailable() && matcher.matches(up) } - .zipWith(control.asFlux()) - .map { it.t1 } - .subscribe(subscriber) + .zipWith(control.asFlux()) + .map { it.t1 } + .subscribe(subscriber) } override fun resolve() { @@ -171,7 +175,7 @@ class FilteredApis( } override fun request(tries: Int) { - //TODO check the buffer size before submitting + // TODO check the buffer size before submitting repeat(tries) { control.tryEmitNext(true) } @@ -183,16 +187,16 @@ class FilteredApis( class Monitoring(chain: Chain) { val countStd: DistributionSummary = DistributionSummary.builder("$metricsCode.exist") - .description("Count of available upstreams to select") - .tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "std"))) - .register(Metrics.globalRegistry) + .description("Count of available upstreams to select") + .tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "std"))) + .register(Metrics.globalRegistry) val countFallback: DistributionSummary = DistributionSummary.builder("$metricsCode.exist") - .description("Count of available fallback upstreams to select") - .tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "fallback"))) - .register(Metrics.globalRegistry) + .description("Count of available fallback upstreams to select") + .tags(listOf(Tag.of("chain", chain.chainCode), Tag.of("role", "fallback"))) + .register(Metrics.globalRegistry) val tried: DistributionSummary = DistributionSummary.builder("$metricsCode.tried") - .description("How many upstreams were checked") - .tags(listOf(Tag.of("chain", chain.chainCode))) - .register(Metrics.globalRegistry) + .description("How many upstreams were checked") + .tags(listOf(Tag.of("chain", chain.chainCode))) + .register(Metrics.globalRegistry) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt index 7bc06854..ec366dff 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Head.kt @@ -18,7 +18,6 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.dshackle.data.BlockContainer import reactor.core.publisher.Flux -import reactor.core.publisher.Mono /** * Subscription to listen to updates to the head of a blockchain. @@ -38,4 +37,4 @@ interface Head { fun onBeforeBlock(handler: Runnable) fun getCurrentHeight(): Long? -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt index b9f93fce..748e729d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt @@ -31,8 +31,8 @@ import java.time.Duration * other upstreams. */ abstract class HeadLagObserver( - private val master: Head, - private val followers: Collection + private val master: Head, + private val followers: Collection ) : Lifecycle { private val log = LoggerFactory.getLogger(HeadLagObserver::class.java) @@ -55,18 +55,18 @@ abstract class HeadLagObserver( fun subscription(): Flux { return master.getFlux() - .flatMap(this::probeFollowers) - .map { item -> - item.t2.setLag(item.t1) - } + .flatMap(this::probeFollowers) + .map { item -> + item.t2.setLag(item.t1) + } } fun probeFollowers(top: BlockContainer): Flux> { return Flux.fromIterable(followers) - .parallel(followers.size) - .flatMap { up -> mapLagging(top, up, getCurrentBlocks(up)).subscribeOn(Schedulers.boundedElastic()) } - .sequential() - .onErrorContinue { t, _ -> log.warn("Failed to update lagging distance", t) } + .parallel(followers.size) + .flatMap { up -> mapLagging(top, up, getCurrentBlocks(up)).subscribeOn(Schedulers.boundedElastic()) } + .sequential() + .onErrorContinue { t, _ -> log.warn("Failed to update lagging distance", t) } } open fun getCurrentBlocks(up: Upstream): Flux { @@ -76,12 +76,12 @@ abstract class HeadLagObserver( fun mapLagging(top: BlockContainer, up: Upstream, blocks: Flux): Flux> { return blocks - .map { extractDistance(top, it) } - .takeUntil { lag -> lag <= 0L } - .map { Tuples.of(it, up) } - .doOnError { t -> - log.warn("Failed to find distance for $up", t) - } + .map { extractDistance(top, it) } + .takeUntil { lag -> lag <= 0L } + .map { Tuples.of(it, up) } + .doOnError { t -> + log.warn("Failed to find distance for $up", t) + } } open fun extractDistance(top: BlockContainer, curr: BlockContainer): Long { @@ -93,5 +93,4 @@ abstract class HeadLagObserver( } abstract fun forkDistance(top: BlockContainer, curr: BlockContainer): Long - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt index 9ce8f5ee..57a65ca1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MergedHead.kt @@ -23,7 +23,7 @@ import reactor.core.Disposable import reactor.core.publisher.Flux class MergedHead( - private val sources: Iterable + private val sources: Iterable ) : AbstractHead(), Lifecycle, CachesEnabled { private var subscription: Disposable? = null @@ -59,5 +59,4 @@ class MergedHead( } } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt index 462a2e35..f64c21bc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt @@ -16,7 +16,7 @@ */ package io.emeraldpay.dshackle.upstream -import io.emeraldpay.dshackle.cache.* +import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.calls.AggregatedCallMethods @@ -33,10 +33,9 @@ import org.springframework.context.Lifecycle import reactor.core.Disposable import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import reactor.core.scheduler.Schedulers import java.time.Duration import java.time.Instant -import java.util.* +import java.util.Locale import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.locks.ReentrantLock import java.util.function.Predicate @@ -46,10 +45,10 @@ import kotlin.concurrent.withLock * Aggregation of multiple upstreams responding to a single blockchain */ abstract class Multistream( - val chain: Chain, - private val upstreams: MutableList, - val caches: Caches, - val postprocessor: RequestPostprocessor + val chain: Chain, + private val upstreams: MutableList, + val caches: Caches, + val postprocessor: RequestPostprocessor ) : Upstream, Lifecycle { companion object { @@ -70,14 +69,19 @@ abstract class Multistream( init { UpstreamAvailability.values().forEach { status -> - Metrics.gauge("$metrics.availability", - listOf(Tag.of("chain", chain.chainCode), Tag.of("status", status.name.lowercase(Locale.getDefault()))), this) { + Metrics.gauge( + "$metrics.availability", + listOf(Tag.of("chain", chain.chainCode), Tag.of("status", status.name.lowercase(Locale.getDefault()))), + this + ) { upstreams.count { it.getStatus() == status }.toDouble() } } - Metrics.gauge("$metrics.connected", - listOf(Tag.of("chain", chain.chainCode)), this) { + Metrics.gauge( + "$metrics.connected", + listOf(Tag.of("chain", chain.chainCode)), this + ) { upstreams.size.toDouble() } @@ -87,8 +91,10 @@ abstract class Multistream( } private fun monitorUpstream(upstream: Upstream) { - Metrics.gauge("$metrics.lag", - listOf(Tag.of("chain", chain.chainCode), Tag.of("upstream", upstream.getId())), upstream) { + Metrics.gauge( + "$metrics.lag", + listOf(Tag.of("chain", chain.chainCode), Tag.of("upstream", upstream.getId())), upstream + ) { it.getLag().toDouble() } } @@ -139,9 +145,14 @@ abstract class Multistream( val apis = getApiSource(matcher) apis.request(1) return Mono.from(apis) - .map(Upstream::getApi) - .map { RequestPostprocessor.wrap(it, postprocessor) } //TODO do it on upstream init, not each time it's called - .switchIfEmpty(Mono.error(Exception("No API available for $chain"))) + .map(Upstream::getApi) + .map { + RequestPostprocessor.wrap( + it, + postprocessor + ) + } // TODO do it on upstream init, not each time it's called + .switchIfEmpty(Mono.error(Exception("No API available for $chain"))) } /** @@ -157,7 +168,7 @@ abstract class Multistream( reconfigLock.withLock { val upstreams = getAll() upstreams.map { it.getMethods() }.let { - //TODO made list of uniq instances, and then if only one, just use it directly + // TODO made list of uniq instances, and then if only one, just use it directly callMethods = AggregatedCallMethods(it) } capabilities = if (upstreams.isEmpty()) { @@ -173,13 +184,13 @@ abstract class Multistream( override fun observeStatus(): Flux { val upstreamsFluxes = getAll().map { up -> Flux.concat( - Mono.just(up.getStatus()), - up.observeStatus() + Mono.just(up.getStatus()), + up.observeStatus() ).map { UpstreamStatus(up, it) } } return Flux.merge(upstreamsFluxes) - .filter(FilterBestAvailability()) - .map { it.status } + .filter(FilterBestAvailability()) + .map { it.status } } override fun isAvailable(): Boolean { @@ -192,12 +203,12 @@ abstract class Multistream( else upstreams.minOf { it.getStatus() } } - //TODO options for multistream are useless + // TODO options for multistream are useless override fun getOptions(): UpstreamsConfig.Options { return UpstreamsConfig.Options() } - //TODO roles for multistream are useless + // TODO roles for multistream are useless override fun getRole(): UpstreamsConfig.UpstreamRole { return UpstreamsConfig.UpstreamRole.STANDARD } @@ -213,11 +224,11 @@ abstract class Multistream( override fun start() { val repeated = Flux.interval(Duration.ofSeconds(30)) val whenChanged = observeStatus() - .distinctUntilChanged() + .distinctUntilChanged() subscription = Flux.merge(repeated, whenChanged) - // print status _change_ every 15 seconds, at most; otherwise prints it on interval of 30 seconds - .sample(Duration.ofSeconds(15)) - .subscribe { printStatus() } + // print status _change_ every 15 seconds, at most; otherwise prints it on interval of 30 seconds + .sample(Duration.ofSeconds(15)) + .subscribe { printStatus() } } override fun stop() { @@ -274,16 +285,16 @@ abstract class Multistream( try { height = getHead().getCurrentHeight() } catch (e: java.lang.IllegalStateException) { - //timout + // timout } catch (e: Exception) { log.warn("Head processing error: ${e.javaClass} ${e.message}") } val statuses = upstreams.map { it.getStatus() } - .groupBy { it } - .map { "${it.key.name}/${it.value.size}" } - .joinToString(",") + .groupBy { it } + .map { "${it.key.name}/${it.value.size}" } + .joinToString(",") val lag = upstreams.map { it.getLag() } - .joinToString(", ") + .joinToString(", ") log.info("State of ${chain.chainCode}: height=${height ?: '?'}, status=$statuses, lag=[$lag]") } @@ -292,15 +303,15 @@ abstract class Multistream( class UpstreamStatus(val upstream: Upstream, val status: UpstreamAvailability, val ts: Instant = Instant.now()) - class FilterBestAvailability() : Predicate { + class FilterBestAvailability : Predicate { private val lastRef = AtomicReference() override fun test(t: UpstreamStatus): Boolean { val curr = lastRef.updateAndGet { last -> - val changed = last == null - || t.status < last.status - || (last.upstream == t.upstream && t.status != last.status) - || last.ts.isBefore(t.ts - Duration.ofSeconds(60)) + val changed = last == null || + t.status < last.status || + (last.upstream == t.upstream && t.status != last.status) || + last.ts.isBefore(t.ts - Duration.ofSeconds(60)) if (changed) { t } else { @@ -310,5 +321,4 @@ abstract class Multistream( return curr == t } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MultistreamHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MultistreamHolder.kt index e9f85f68..852ffdee 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MultistreamHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MultistreamHolder.kt @@ -29,4 +29,4 @@ interface MultistreamHolder { fun observeChains(): Flux fun getDefaultMethods(chain: Chain): CallMethods fun isAvailable(chain: Chain): Boolean -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestPostprocessor.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestPostprocessor.kt index 5f21123e..69ec2693 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestPostprocessor.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestPostprocessor.kt @@ -14,25 +14,27 @@ interface RequestPostprocessor { } companion object { - fun wrap(reader: Reader, processor: RequestPostprocessor): Reader { + fun wrap( + reader: Reader, + processor: RequestPostprocessor + ): Reader { return Wrapper(reader, processor) } } class Wrapper( - private val reader: Reader, - private val processor: RequestPostprocessor + private val reader: Reader, + private val processor: RequestPostprocessor ) : Reader { override fun read(key: JsonRpcRequest): Mono { return reader.read(key) - .doOnNext { - if (it.hasResult()) { - val result = it.getResult() - processor.onReceive(key.method, key.params, result) - } + .doOnNext { + if (it.hasResult()) { + val result = it.getResult() + processor.onReceive(key.method, key.params, result) } + } } - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt index 43ef6b40..50a1365f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt @@ -19,8 +19,7 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.config.UpstreamsConfig import org.apache.commons.lang3.StringUtils -import java.util.* -import kotlin.collections.ArrayList +import java.util.Collections class Selector { @@ -36,8 +35,8 @@ class Selector { req.hasLabelSelector() -> req.labelSelector.let { selector -> if (StringUtils.isNotEmpty(selector.name)) { val values = selector.valueList - .map { it?.trim() ?: "" } - .filter { StringUtils.isNotEmpty(it) } + .map { it?.trim() ?: "" } + .filter { StringUtils.isNotEmpty(it) } if (values.isEmpty()) { ExistsMatcher(selector.name) } else { @@ -47,8 +46,24 @@ class Selector { AnyLabelMatcher() } } - req.hasAndSelector() -> AndMatcher(Collections.unmodifiableCollection(req.andSelector.selectorsList.map { convertToMatcher(it) })) - req.hasOrSelector() -> OrMatcher(Collections.unmodifiableCollection(req.orSelector.selectorsList.map { convertToMatcher(it) })) + req.hasAndSelector() -> AndMatcher( + Collections.unmodifiableCollection( + req.andSelector.selectorsList.map { + convertToMatcher( + it + ) + } + ) + ) + req.hasOrSelector() -> OrMatcher( + Collections.unmodifiableCollection( + req.orSelector.selectorsList.map { + convertToMatcher( + it + ) + } + ) + ) req.hasNotSelector() -> NotMatcher(convertToMatcher(req.notSelector.selector)) req.hasExistsSelector() -> ExistsMatcher(req.existsSelector.name) else -> AnyLabelMatcher() @@ -110,8 +125,8 @@ class Selector { } class MultiMatcher( - private val matchers: Collection - ): Matcher { + private val matchers: Collection + ) : Matcher { override fun matches(up: Upstream): Boolean { return matchers.all { it.matches(up) } } @@ -139,8 +154,8 @@ class Selector { } class MethodMatcher( - val method: String - ): Matcher { + val method: String + ) : Matcher { override fun matches(up: Upstream): Boolean { return up.getMethods().isAllowed(method) } @@ -154,7 +169,7 @@ class Selector { } } - abstract class LabelSelectorMatcher: Matcher { + abstract class LabelSelectorMatcher : Matcher { override fun matches(up: Upstream): Boolean { return up.getLabels().any(this::matches) } @@ -163,7 +178,7 @@ class Selector { abstract fun asProto(): BlockchainOuterClass.Selector? } - class EmptyMatcher: Matcher { + class EmptyMatcher : Matcher { override fun matches(up: Upstream): Boolean { return true } @@ -177,7 +192,7 @@ class Selector { } } - class AnyLabelMatcher: LabelSelectorMatcher() { + class AnyLabelMatcher : LabelSelectorMatcher() { override fun matches(labels: UpstreamsConfig.Labels): Boolean { return true @@ -224,9 +239,9 @@ class Selector { override fun asProto(): BlockchainOuterClass.Selector { return BlockchainOuterClass.Selector.newBuilder().setLabelSelector( - BlockchainOuterClass.LabelSelector.newBuilder() - .setName(name) - .addAllValue(values) + BlockchainOuterClass.LabelSelector.newBuilder() + .setName(name) + .addAllValue(values) ).build() } @@ -239,16 +254,16 @@ class Selector { } } - class OrMatcher(val matchers: Collection): LabelSelectorMatcher() { + class OrMatcher(val matchers: Collection) : LabelSelectorMatcher() { override fun matches(labels: UpstreamsConfig.Labels): Boolean { return matchers.any { matcher -> matcher.matches(labels) } } override fun asProto(): BlockchainOuterClass.Selector { return BlockchainOuterClass.Selector.newBuilder().setOrSelector( - BlockchainOuterClass.OrSelector.newBuilder() - .addAllSelectors(matchers.map { it.asProto() }) - .build() + BlockchainOuterClass.OrSelector.newBuilder() + .addAllSelectors(matchers.map { it.asProto() }) + .build() ).build() } @@ -261,16 +276,16 @@ class Selector { } } - class AndMatcher(val matchers: Collection): LabelSelectorMatcher() { + class AndMatcher(val matchers: Collection) : LabelSelectorMatcher() { override fun matches(labels: UpstreamsConfig.Labels): Boolean { return matchers.all { matcher -> matcher.matches(labels) } } override fun asProto(): BlockchainOuterClass.Selector { return BlockchainOuterClass.Selector.newBuilder().setAndSelector( - BlockchainOuterClass.AndSelector.newBuilder() - .addAllSelectors(matchers.map { it.asProto() }) - .build() + BlockchainOuterClass.AndSelector.newBuilder() + .addAllSelectors(matchers.map { it.asProto() }) + .build() ).build() } @@ -283,16 +298,16 @@ class Selector { } } - class NotMatcher(val matcher: LabelSelectorMatcher): LabelSelectorMatcher() { + class NotMatcher(val matcher: LabelSelectorMatcher) : LabelSelectorMatcher() { override fun matches(labels: UpstreamsConfig.Labels): Boolean { return !matcher.matches(labels) } override fun asProto(): BlockchainOuterClass.Selector { return BlockchainOuterClass.Selector.newBuilder().setNotSelector( - BlockchainOuterClass.NotSelector.newBuilder() - .setSelector(matcher.asProto()) - .build() + BlockchainOuterClass.NotSelector.newBuilder() + .setSelector(matcher.asProto()) + .build() ).build() } @@ -305,21 +320,21 @@ class Selector { } } - class ExistsMatcher(val name: String): LabelSelectorMatcher() { + class ExistsMatcher(val name: String) : LabelSelectorMatcher() { override fun matches(labels: UpstreamsConfig.Labels): Boolean { return labels.containsKey(name) } override fun asProto(): BlockchainOuterClass.Selector { return BlockchainOuterClass.Selector.newBuilder().setExistsSelector( - BlockchainOuterClass.ExistsSelector.newBuilder() - .setName(name) - .build() + BlockchainOuterClass.ExistsSelector.newBuilder() + .setName(name) + .build() ).build() } override fun describeInternal(): String { - return "label '${name}' exists" + return "label '$name' exists" } override fun toString(): String { @@ -341,7 +356,7 @@ class Selector { } } - class GrpcMatcher() : Matcher { + class GrpcMatcher : Matcher { override fun matches(up: Upstream): Boolean { return up.isGrpc() } @@ -355,7 +370,7 @@ class Selector { } } - class HeightMatcher(val height: Long): Matcher { + class HeightMatcher(val height: Long) : Matcher { override fun matches(up: Upstream): Boolean { return (up.getHead().getCurrentHeight() ?: 0) >= height } @@ -381,4 +396,4 @@ class Selector { return "Matcher: ${describeInternal()}" } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt index 5ea8fcbe..d170acbc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt @@ -22,7 +22,6 @@ import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import reactor.core.publisher.Flux -import reactor.core.publisher.Mono interface Upstream { fun isAvailable(): Boolean @@ -41,4 +40,4 @@ interface Upstream { fun isGrpc(): Boolean fun cast(selfType: Class): T -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamAvailability.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamAvailability.kt index 4bed644c..70788173 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamAvailability.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamAvailability.kt @@ -22,18 +22,22 @@ enum class UpstreamAvailability(val grpcId: Int) { * Active fully synchronized node */ OK(1), + /** * Good node, but is still synchronizing to a latest block */ LAGGING(2), + /** * May be good, but node doesn't have enough peers to be sure it's on corrected chain */ IMMATURE(3), + /** * Node is doing it's initial synchronization, is behind by at least several blocks */ SYNCING(4), + /** * Unavailable node */ @@ -49,4 +53,4 @@ enum class UpstreamAvailability(val grpcId: Int) { } ?: UNAVAILABLE } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt index 66426ca8..a46d7c71 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/AddressActiveCheck.kt @@ -20,7 +20,7 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Mono open class AddressActiveCheck( - private val esploraClient: EsploraClient + private val esploraClient: EsploraClient ) { companion object { @@ -28,8 +28,7 @@ open class AddressActiveCheck( } open fun isActive(address: Address): Mono { - //TODO cache with bloom filter + // TODO cache with bloom filter return esploraClient.getTransactions(address).map { it.isNotEmpty() } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinHeadLagObserver.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinHeadLagObserver.kt index 28b3ad94..ffc3740b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinHeadLagObserver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinHeadLagObserver.kt @@ -22,8 +22,8 @@ import io.emeraldpay.dshackle.upstream.Upstream import org.slf4j.LoggerFactory class BitcoinHeadLagObserver( - master: Head, - followers: Collection + master: Head, + followers: Collection ) : HeadLagObserver(master, followers) { companion object { @@ -31,8 +31,7 @@ class BitcoinHeadLagObserver( } override fun forkDistance(top: BlockContainer, curr: BlockContainer): Long { - //TODO fetch actual blocks + // TODO fetch actual blocks return 3 } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt index fc7e44d4..568562da 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinMultistream.kt @@ -19,7 +19,13 @@ import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.EmptyReader import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.EmptyHead +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.MergedHead +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.RequestPostprocessor +import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.grpc.Chain @@ -29,9 +35,9 @@ import reactor.core.publisher.Mono @Suppress("UNCHECKED_CAST") open class BitcoinMultistream( - chain: Chain, - val upstreams: MutableList, - caches: Caches + chain: Chain, + val upstreams: MutableList, + caches: Caches ) : Multistream(chain, upstreams as MutableList, caches, RequestPostprocessor.Empty()), Lifecycle { companion object { @@ -80,7 +86,7 @@ open class BitcoinMultistream( } override fun getRoutedApi(matcher: Selector.Matcher): Mono> { - //TODO + // TODO return Mono.just(EmptyReader()) } @@ -130,4 +136,4 @@ open class BitcoinMultistream( super.stop() reader.stop() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt index c272fe9b..e1c43c2c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinReader.kt @@ -29,9 +29,9 @@ import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.cast open class BitcoinReader( - private val upstreams: BitcoinMultistream, - head: Head, - esploraClient: EsploraClient? + private val upstreams: BitcoinMultistream, + head: Head, + esploraClient: EsploraClient? ) : Lifecycle { companion object { @@ -78,10 +78,10 @@ open class BitcoinReader( fun castedRead(req: JsonRpcRequest, clazz: Class): Mono { return upstreams.getDirectApi(Selector.empty).flatMap { api -> api.read(req) - .flatMap(JsonRpcResponse::requireResult) - .map { - objectMapper.readValue(it, clazz) as T - } + .flatMap(JsonRpcResponse::requireResult) + .map { + objectMapper.readValue(it, clazz) as T + } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt index bf7830df..f0353f44 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt @@ -32,14 +32,15 @@ import java.time.Duration import java.util.concurrent.Executors class BitcoinRpcHead( - private val api: Reader, - private val extractBlock: ExtractBlock, - private val interval: Duration = Duration.ofSeconds(15) + private val api: Reader, + private val extractBlock: ExtractBlock, + private val interval: Duration = Duration.ofSeconds(15) ) : Head, AbstractHead(), Lifecycle { companion object { private val log = LoggerFactory.getLogger(BitcoinRpcHead::class.java) - val scheduler = Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("bitcoin-rpc-head"))) + val scheduler = + Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("bitcoin-rpc-head"))) } private var refreshSubscription: Disposable? = null @@ -54,22 +55,22 @@ class BitcoinRpcHead( return } val base = Flux.interval(interval) - .publishOn(scheduler) - .flatMap { - api.read(JsonRpcRequest("getbestblockhash", emptyList())) - .flatMap(JsonRpcResponse::requireStringResult) - .timeout(Defaults.timeout, Mono.error(Exception("Best block hash is not received"))) - } - .distinctUntilChanged() - .flatMap { hash -> - api.read(JsonRpcRequest("getblock", listOf(hash))) - .flatMap(JsonRpcResponse::requireResult) - .map(extractBlock::extract) - .timeout(Defaults.timeout, Mono.error(Exception("Block data is not received"))) - } - .onErrorContinue { err, _ -> - log.debug("RPC error ${err.message}") - } + .publishOn(scheduler) + .flatMap { + api.read(JsonRpcRequest("getbestblockhash", emptyList())) + .flatMap(JsonRpcResponse::requireStringResult) + .timeout(Defaults.timeout, Mono.error(Exception("Best block hash is not received"))) + } + .distinctUntilChanged() + .flatMap { hash -> + api.read(JsonRpcRequest("getblock", listOf(hash))) + .flatMap(JsonRpcResponse::requireResult) + .map(extractBlock::extract) + .timeout(Defaults.timeout, Mono.error(Exception("Block data is not received"))) + } + .onErrorContinue { err, _ -> + log.debug("RPC error ${err.message}") + } refreshSubscription = super.follow(base) } @@ -78,5 +79,4 @@ class BitcoinRpcHead( refreshSubscription = null copy?.dispose() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt index 2f5d8c23..18afd4f1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcUpstream.kt @@ -18,7 +18,10 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse @@ -28,14 +31,14 @@ import org.springframework.context.Lifecycle import reactor.core.Disposable open class BitcoinRpcUpstream( - id: String, - chain: Chain, - private val directApi: Reader, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - node: QuorumForLabels.QuorumItem, - callMethods: CallMethods, - esploraClient: EsploraClient? = null + id: String, + chain: Chain, + private val directApi: Reader, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + node: QuorumForLabels.QuorumItem, + callMethods: CallMethods, + esploraClient: EsploraClient? = null ) : BitcoinUpstream(id, chain, options, role, callMethods, node, esploraClient), Lifecycle { companion object { @@ -53,8 +56,8 @@ open class BitcoinRpcUpstream( private fun createHead(): Head { return BitcoinRpcHead( - directApi, - ExtractBlock() + directApi, + ExtractBlock() ) } @@ -71,7 +74,7 @@ open class BitcoinRpcUpstream( } override fun getCapabilities(): Set { - return capabilities; + return capabilities } override fun isGrpc(): Boolean { @@ -110,7 +113,7 @@ open class BitcoinRpcUpstream( } else { val validator = BitcoinUpstreamValidator(directApi, getOptions()) validatorSubscription = validator.start() - .subscribe(this::setStatus) + .subscribe(this::setStatus) } } @@ -120,5 +123,4 @@ open class BitcoinRpcUpstream( } validatorSubscription?.dispose() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt index 9325b37f..6e02efff 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstream.kt @@ -24,22 +24,23 @@ import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory abstract class BitcoinUpstream( - id: String, - val chain: Chain, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - callMethods: CallMethods, - node: QuorumForLabels.QuorumItem, - val esploraClient: EsploraClient? = null + id: String, + val chain: Chain, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + callMethods: CallMethods, + node: QuorumForLabels.QuorumItem, + val esploraClient: EsploraClient? = null ) : DefaultUpstream(id, options, role, callMethods, node) { - constructor(id: String, - chain: Chain, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty()) + constructor( + id: String, + chain: Chain, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole + ) : this(id, chain, options, role, DefaultBitcoinMethods(), QuorumForLabels.QuorumItem.empty()) companion object { private val log = LoggerFactory.getLogger(BitcoinUpstream::class.java) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstreamValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstreamValidator.kt index e800afe4..6fd2a32e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstreamValidator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinUpstreamValidator.kt @@ -29,36 +29,36 @@ import java.time.Duration import java.util.concurrent.Executors class BitcoinUpstreamValidator( - private val api: Reader, - private val options: UpstreamsConfig.Options + private val api: Reader, + private val options: UpstreamsConfig.Options ) { companion object { private val log = LoggerFactory.getLogger(BitcoinUpstreamValidator::class.java) - val scheduler = Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("bitcoin-validator"))) + val scheduler = + Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("bitcoin-validator"))) } fun validate(): Mono { return api.read(JsonRpcRequest("getconnectioncount", emptyList())) - .flatMap(JsonRpcResponse::requireResult) - .map { Integer.parseInt(String(it)) } - .map { count -> - val minPeers = options.minPeers ?: 1 - if (count < minPeers) { - UpstreamAvailability.IMMATURE - } else { - UpstreamAvailability.OK - } + .flatMap(JsonRpcResponse::requireResult) + .map { Integer.parseInt(String(it)) } + .map { count -> + val minPeers = options.minPeers ?: 1 + if (count < minPeers) { + UpstreamAvailability.IMMATURE + } else { + UpstreamAvailability.OK } - .onErrorReturn(UpstreamAvailability.UNAVAILABLE) + } + .onErrorReturn(UpstreamAvailability.UNAVAILABLE) } fun start(): Flux { return Flux.interval(Duration.ofSeconds(15)) - .subscribeOn(scheduler) - .flatMap { - validate() - } + .subscribeOn(scheduler) + .flatMap { + validate() + } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/CachingMempoolData.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/CachingMempoolData.kt index ec18ac72..3e26e5a5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/CachingMempoolData.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/CachingMempoolData.kt @@ -31,8 +31,8 @@ import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.locks.ReentrantLock open class CachingMempoolData( - private val upstreams: BitcoinMultistream, - private val head: Head + private val upstreams: BitcoinMultistream, + private val head: Head ) : Lifecycle { companion object { @@ -51,12 +51,12 @@ open class CachingMempoolData( return if (value.since < Instant.now().minus(TTL)) { updateLock.lock() fetchFromUpstream() - .timeout(Duration.ofSeconds(3), Mono.empty()) - .doOnNext { - current.set(Container(Instant.now(), it)) - }.doFinally { - updateLock.unlock() - } + .timeout(Duration.ofSeconds(3), Mono.empty()) + .doOnNext { + current.set(Container(Instant.now(), it)) + }.doFinally { + updateLock.unlock() + } } else { Mono.just(value.value) } @@ -66,8 +66,8 @@ open class CachingMempoolData( fun fetchFromUpstream(): Mono> { return upstreams.getDirectApi(Selector.empty).flatMap { api -> api.read(JsonRpcRequest("getrawmempool", emptyList())) - .flatMap(JsonRpcResponse::requireResult) - .map { objectMapper.readValue(it, List::class.java) as List } + .flatMap(JsonRpcResponse::requireResult) + .map { objectMapper.readValue(it, List::class.java) as List } } } @@ -94,5 +94,4 @@ open class CachingMempoolData( headListener?.dispose() headListener = null } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt index ef0c6df0..aa2b623d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt @@ -30,13 +30,13 @@ import java.net.URI import java.security.KeyStore import java.security.cert.CertificateFactory import java.security.cert.X509Certificate -import java.util.* +import java.util.Base64 import java.util.function.Consumer class EsploraClient( - private val url: URI, - basicAuth: AuthConfig.ClientBasicAuth? = null, - tlsCAAuth: ByteArray? = null + private val url: URI, + basicAuth: AuthConfig.ClientBasicAuth? = null, + tlsCAAuth: ByteArray? = null ) { companion object { @@ -78,43 +78,48 @@ class EsploraClient( fun getUtxo(address: Address): Mono> { val response = httpClient - .get() - .uri("$url/address/$address/utxo") + .get() + .uri("$url/address/$address/utxo") val json = parseResponse(response) return json.map { val parsed = Global.objectMapper.readerFor(EsploraUnspent::class.java) - .readValues(it); + .readValues(it) parsed.readAll() } } fun getTransactions(address: Address): Mono>> { val response = httpClient - .get() - .uri("$url/address/$address/txs") + .get() + .uri("$url/address/$address/txs") val json = parseResponse(response) return json.map { val parsed = Global.objectMapper.readerFor(Map::class.java) - .readValues>(it); + .readValues>(it) parsed.readAll() } } private fun parseResponse(response: HttpClient.ResponseReceiver<*>): Mono { return response - .response { header, bytes -> - if (header.status().code() != 200) { - Mono.error(EsploraException("HTTP Code: ${header.status().code()} for ${url.scheme}://${url.host}${header.fullPath()}")) - } else { - bytes.aggregate().asByteArray() - } + .response { header, bytes -> + if (header.status().code() != 200) { + Mono.error( + EsploraException( + "HTTP Code: ${ + header.status().code() + } for ${url.scheme}://${url.host}${header.fullPath()}" + ) + ) + } else { + bytes.aggregate().asByteArray() } - .single() - .map { bytes -> String(bytes) } + } + .single() + .map { bytes -> String(bytes) } } class EsploraException(msg: String) : Exception(msg) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt index ba5e54f6..2b2e2e11 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt @@ -24,8 +24,8 @@ import reactor.core.publisher.Mono import java.util.function.Function class EsploraUnspentReader( - private val esploraClient: EsploraClient, - private val head: Head + private val esploraClient: EsploraClient, + private val head: Head ) : UnspentReader { companion object { @@ -34,10 +34,10 @@ class EsploraUnspentReader( private val convert: (T: EsploraUnspent) -> SimpleUnspent = { base -> SimpleUnspent( - base.txid, - base.vout, - base.value, - head.getCurrentHeight()?.let { base.height - it } ?: 0 + base.txid, + base.vout, + base.value, + head.getCurrentHeight()?.let { base.height - it } ?: 0 ) } @@ -47,7 +47,6 @@ class EsploraUnspentReader( override fun read(key: Address): Mono> { return esploraClient.getUtxo(key) - .map(convertAll) + .map(convertAll) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/ExtractBlock.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/ExtractBlock.kt index cb87fea0..8049d886 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/ExtractBlock.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/ExtractBlock.kt @@ -25,7 +25,7 @@ import org.slf4j.LoggerFactory import java.math.BigInteger import java.time.Instant -class ExtractBlock() { +class ExtractBlock { companion object { private val log = LoggerFactory.getLogger(ExtractBlock::class.java) @@ -59,16 +59,14 @@ class ExtractBlock() { val transactions = (data["tx"] as List?)?.map(TxId.Companion::from) ?: emptyList() return BlockContainer( - getHeight(data) ?: throw IllegalArgumentException("Block JSON has no height"), - BlockId.from(hash), - getDifficulty(data) ?: throw IllegalArgumentException("Block JSON has no chainwork"), - getTime(data) ?: throw IllegalArgumentException("Block JSON has no time"), - false, - json, - data, - transactions + getHeight(data) ?: throw IllegalArgumentException("Block JSON has no height"), + BlockId.from(hash), + getDifficulty(data) ?: throw IllegalArgumentException("Block JSON has no chainwork"), + getTime(data) ?: throw IllegalArgumentException("Block JSON has no time"), + false, + json, + data, + transactions ) } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt index 92db7f86..773109bc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Mono class RpcUnspentReader( - private val upstreams: BitcoinMultistream + private val upstreams: BitcoinMultistream ) : UnspentReader { companion object { @@ -35,10 +35,10 @@ class RpcUnspentReader( private val convert: (T: RpcUnspent) -> SimpleUnspent = { base -> SimpleUnspent( - base.txid, - base.vout, - base.amount, - base.confirmations + base.txid, + base.vout, + base.amount, + base.confirmations ) } @@ -46,16 +46,15 @@ class RpcUnspentReader( val address = key.toString() return upstreams.getDirectApi(Selector.empty).flatMap { api -> api.read(JsonRpcRequest("listunspent", emptyList())) - .flatMap(JsonRpcResponse::requireResult) - .map { - Global.objectMapper.readerFor(RpcUnspent::class.java).readValues(it).readAll() - } - .map { - it.filter { - it.address == address - }.map(convert) - } + .flatMap(JsonRpcResponse::requireResult) + .map { + Global.objectMapper.readerFor(RpcUnspent::class.java).readValues(it).readAll() + } + .map { + it.filter { + it.address == address + }.map(convert) + } } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt index 42f4fbde..d922c587 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt @@ -19,5 +19,4 @@ import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import org.bitcoinj.core.Address -interface UnspentReader : Reader> { -} \ No newline at end of file +interface UnspentReader : Reader> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt index e91012c6..6ff4a37a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt @@ -30,7 +30,7 @@ import reactor.util.function.Tuples import java.util.concurrent.atomic.AtomicInteger open class XpubAddresses( - private val addressActiveCheck: AddressActiveCheck + private val addressActiveCheck: AddressActiveCheck ) { companion object { @@ -76,28 +76,27 @@ open class XpubAddresses( } return Flux.range(start, limit) - .map { HDKeyDerivation.deriveChildKey(key, ChildNumber(it, false)) } - .map { Address.fromKey(network, ECKey.fromPublicOnly(it.pubKeyPoint), type) } + .map { HDKeyDerivation.deriveChildKey(key, ChildNumber(it, false)) } + .map { Address.fromKey(network, ECKey.fromPublicOnly(it.pubKeyPoint), type) } } open fun activeAddresses(xpub: String, start: Int, limit: Int): Flux
{ val lastActive = AtomicInteger(0) return this.allAddresses(xpub, start, limit) - .zipWith(Flux.range(0, limit)) - .takeUntil { - it.t2 - lastActive.get() >= INACTIVE_LIMIT - } - .concatMap { toCheck -> - addressActiveCheck.isActive(toCheck.t1) - .doOnNext { active -> if (active) lastActive.set(toCheck.t2) } - .map { Tuples.of(toCheck.t1, it) } - } - .filter { - it.t2 - } - .map { - it.t1 - } + .zipWith(Flux.range(0, limit)) + .takeUntil { + it.t2 - lastActive.get() >= INACTIVE_LIMIT + } + .concatMap { toCheck -> + addressActiveCheck.isActive(toCheck.t1) + .doOnNext { active -> if (active) lastActive.set(toCheck.t2) } + .map { Tuples.of(toCheck.t1, it) } + } + .filter { + it.t2 + } + .map { + it.t1 + } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt index 7e91cb39..840386be 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspent.kt @@ -18,9 +18,9 @@ package io.emeraldpay.dshackle.upstream.bitcoin.data import java.time.Instant data class EsploraUnspent( - val txid: String, - val vout: Int, - val value: Long, - val timestamp: Instant, - val height: Long -) \ No newline at end of file + val txid: String, + val vout: Int, + val value: Long, + val timestamp: Instant, + val height: Long +) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt index 5e3abe5c..e6581d9c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/EsploraUnspentDeserializer.kt @@ -26,11 +26,11 @@ class EsploraUnspentDeserializer : JsonDeserializer() { val node: JsonNode = jp.readValueAsTree() val status = node.get("status") return EsploraUnspent( - node.get("txid").asText(), - node.get("vout").asInt(), - node.get("value").asLong(), - Instant.ofEpochSecond(status.get("block_time").asLong()), - status.get("block_height").asLong() + node.get("txid").asText(), + node.get("vout").asInt(), + node.get("value").asLong(), + Instant.ofEpochSecond(status.get("block_time").asLong()), + status.get("block_height").asLong() ) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt index 63700ef6..ee3be9c2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspent.kt @@ -16,9 +16,9 @@ package io.emeraldpay.dshackle.upstream.bitcoin.data data class RpcUnspent( - val txid: String, - val vout: Int, - val address: String, - val amount: Long, - val confirmations: Long -) \ No newline at end of file + val txid: String, + val vout: Int, + val address: String, + val amount: Long, + val confirmations: Long +) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt index cd63b6ed..85d1d308 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/RpcUnspentDeserializer.kt @@ -26,12 +26,11 @@ class RpcUnspentDeserializer : JsonDeserializer() { override fun deserialize(jp: JsonParser, ctxt: DeserializationContext): RpcUnspent { val node: JsonNode = jp.readValueAsTree() return RpcUnspent( - node.get("txid").asText(), - node.get("vout").asInt(), - node.get("address").asText(), - BigDecimal(node.get("amount").asText()).multiply(BigDecimal.TEN.pow(8)).longValueExact(), - node.get("confirmations").asLong() + node.get("txid").asText(), + node.get("vout").asInt(), + node.get("address").asText(), + BigDecimal(node.get("amount").asText()).multiply(BigDecimal.TEN.pow(8)).longValueExact(), + node.get("confirmations").asLong() ) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt index afc9e9db..2c6bb471 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/data/SimpleUnspent.kt @@ -16,8 +16,8 @@ package io.emeraldpay.dshackle.upstream.bitcoin.data data class SimpleUnspent( - val txid: String, - val vout: Int, - val value: Long, - val confirmations: Long -) \ No newline at end of file + val txid: String, + val vout: Int, + val value: Long, + val confirmations: Long +) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/AggregatedCallMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/AggregatedCallMethods.kt index dafeb3e7..9df4084b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/AggregatedCallMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/AggregatedCallMethods.kt @@ -17,15 +17,14 @@ package io.emeraldpay.dshackle.upstream.calls import io.emeraldpay.dshackle.quorum.CallQuorum -import java.util.* -import kotlin.collections.HashSet +import java.util.Collections /** * Aggregation over several parent configuration. It dispatches call to a first delegate that supports it. */ class AggregatedCallMethods( - private val delegates: Collection -): CallMethods { + private val delegates: Collection +) : CallMethods { private val allMethods: Set @@ -73,4 +72,4 @@ class AggregatedCallMethods( it.isHardcoded(method) }?.executeHardcoded(method) ?: throw IllegalStateException("No hardcoded for $method") } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/CallMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/CallMethods.kt index 368979bb..1b2b80f9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/CallMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/CallMethods.kt @@ -47,4 +47,4 @@ interface CallMethods { * Read [supposed to be predefined] method from this config */ fun executeHardcoded(method: String): ByteArray -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt index 8ae62914..e7c27276 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt @@ -15,36 +15,41 @@ */ package io.emeraldpay.dshackle.upstream.calls -import io.emeraldpay.dshackle.quorum.* +import io.emeraldpay.dshackle.quorum.AlwaysQuorum +import io.emeraldpay.dshackle.quorum.BroadcastQuorum +import io.emeraldpay.dshackle.quorum.CallQuorum +import io.emeraldpay.dshackle.quorum.NonEmptyQuorum +import io.emeraldpay.dshackle.quorum.NotLaggingQuorum import io.emeraldpay.etherjar.rpc.RpcException -import java.util.* +import java.util.Collections -class DefaultBitcoinMethods() : CallMethods { +class DefaultBitcoinMethods : CallMethods { private val freshMethods = listOf( - "getblock", - "gettransaction", "gettxout", - "getmemorypool" + "getblock", + "gettransaction", "gettxout", + "getmemorypool" ).sorted() private val anyResponseMethods = listOf( - "getblockhash", "getrawtransaction" + "getblockhash", "getrawtransaction" ).sorted() private val headVerifiedMethods = listOf( - "getbestblockhash", "getblocknumber", "getblockcount", - "listunspent", "getreceivedbyaddress" + "getbestblockhash", "getblocknumber", "getblockcount", + "listunspent", "getreceivedbyaddress" ).sorted() private val hardcodedMethods = listOf( - "getconnectioncount", "getnetworkinfo" + "getconnectioncount", "getnetworkinfo" ).sorted() private val broadcastMethods = listOf( - "sendrawtransaction" + "sendrawtransaction" ).sorted() - private val allowedMethods = (freshMethods + anyResponseMethods + headVerifiedMethods + hardcodedMethods + broadcastMethods).sorted() + private val allowedMethods = + (freshMethods + anyResponseMethods + headVerifiedMethods + hardcodedMethods + broadcastMethods).sorted() override fun getQuorumFor(method: String): CallQuorum { return when { @@ -66,7 +71,7 @@ class DefaultBitcoinMethods() : CallMethods { } override fun isHardcoded(method: String): Boolean { - return Collections.binarySearch(hardcodedMethods, method) >= 0; + return Collections.binarySearch(hardcodedMethods, method) >= 0 } override fun executeHardcoded(method: String): ByteArray { @@ -76,5 +81,4 @@ class DefaultBitcoinMethods() : CallMethods { else -> throw RpcException(-32601, "Method not found") } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt index f4188aea..b00b3c57 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt @@ -16,69 +16,70 @@ */ package io.emeraldpay.dshackle.upstream.calls -import com.fasterxml.jackson.databind.ObjectMapper -import io.emeraldpay.dshackle.quorum.* -import io.emeraldpay.grpc.Chain -import io.emeraldpay.etherjar.rpc.JacksonRpcConverter +import io.emeraldpay.dshackle.quorum.AlwaysQuorum +import io.emeraldpay.dshackle.quorum.BroadcastQuorum +import io.emeraldpay.dshackle.quorum.CallQuorum +import io.emeraldpay.dshackle.quorum.NonceQuorum +import io.emeraldpay.dshackle.quorum.NotLaggingQuorum import io.emeraldpay.etherjar.rpc.RpcException -import java.util.* +import io.emeraldpay.grpc.Chain /** * Default configuration for Ethereum based RPC. Defines optimal Quorum strategies for different methods, and provides * hardcoded results for base methods, such as `net_version`, `web3_clientVersion` and similar */ class DefaultEthereumMethods( - private val chain: Chain + private val chain: Chain ) : CallMethods { private val anyResponseMethods = listOf( - "eth_gasPrice", - "eth_call", - "eth_estimateGas" + "eth_gasPrice", + "eth_call", + "eth_estimateGas" ) private val firstValueMethods = listOf( - "eth_getBlockTransactionCountByHash", - "eth_getUncleCountByBlockHash", - "eth_getBlockByHash", - "eth_getTransactionByHash", - "eth_getTransactionByBlockHashAndIndex", - "eth_getStorageAt", - "eth_getCode", - "eth_getUncleByBlockHashAndIndex" + "eth_getBlockTransactionCountByHash", + "eth_getUncleCountByBlockHash", + "eth_getBlockByHash", + "eth_getTransactionByHash", + "eth_getTransactionByBlockHashAndIndex", + "eth_getStorageAt", + "eth_getCode", + "eth_getUncleByBlockHashAndIndex" ) private val specialMethods = listOf( - "eth_getTransactionCount", - "eth_blockNumber", - "eth_getBalance", - "eth_sendRawTransaction" + "eth_getTransactionCount", + "eth_blockNumber", + "eth_getBalance", + "eth_sendRawTransaction" ) private val headVerifiedMethods = listOf( - "eth_getBlockTransactionCountByNumber", - "eth_getUncleCountByBlockNumber", - "eth_getBlockByNumber", - "eth_getTransactionByBlockNumberAndIndex", - "eth_getTransactionReceipt", - "eth_getUncleByBlockNumberAndIndex", - "eth_feeHistory" + "eth_getBlockTransactionCountByNumber", + "eth_getUncleCountByBlockNumber", + "eth_getBlockByNumber", + "eth_getTransactionByBlockNumberAndIndex", + "eth_getTransactionReceipt", + "eth_getUncleByBlockNumberAndIndex", + "eth_feeHistory" ) private val allowedMethods = anyResponseMethods + firstValueMethods + specialMethods + headVerifiedMethods private val hardcodedMethods = listOf( - "net_version", - "net_peerCount", - "net_listening", - "web3_clientVersion", - "eth_protocolVersion", - "eth_syncing", - "eth_coinbase", - "eth_mining", - "eth_hashrate", - "eth_accounts", - "eth_chainId" + "net_version", + "net_peerCount", + "net_listening", + "web3_clientVersion", + "eth_protocolVersion", + "eth_syncing", + "eth_coinbase", + "eth_mining", + "eth_hashrate", + "eth_accounts", + "eth_chainId" ) override fun getQuorumFor(method: String): CallQuorum { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DirectCallMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DirectCallMethods.kt index 7bc619a6..a087e5f4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DirectCallMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DirectCallMethods.kt @@ -47,4 +47,4 @@ open class DirectCallMethods(private val methods: Set) : CallMethods { override fun executeHardcoded(method: String): ByteArray { return "unsupported".toByteArray() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt index 23df0e1a..32b00dcb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/EthereumCallSelector.kt @@ -23,14 +23,14 @@ import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.etherjar.hex.HexQuantity import org.slf4j.LoggerFactory import reactor.core.publisher.Mono -import java.util.* +import java.util.Collections /** * Get a matcher based on a criteria provided with a RPC request. I.e. when the client requests data for "latest", or "0x19f816" block. * The implementation is specific for Ethereum. */ class EthereumCallSelector( - private val heightReader: Reader + private val heightReader: Reader ) { companion object { @@ -38,11 +38,11 @@ class EthereumCallSelector( // ref https://eth.wiki/json-rpc/API#the-default-block-parameter private val TAG_METHODS = listOf( - "eth_getBalance", - "eth_getCode", - "eth_getTransactionCount", - // no "eth_getStorageAt" because it's has different structure, and therefore separate logic - "eth_call" + "eth_getBalance", + "eth_getCode", + "eth_getTransactionCount", + // no "eth_getStorageAt" because it's has different structure, and therefore separate logic + "eth_call" ).sorted() } @@ -73,7 +73,7 @@ class EthereumCallSelector( // for earliest it doesn't nothing, we expect to have 0 block "earliest" -> 0L else -> if (tag.startsWith("0x")) { - return if (tag.length == 66) { //32-byte hash is represented as 0x + 64 characters + return if (tag.length == 66) { // 32-byte hash is represented as 0x + 64 characters blockByHash(tag, head) } else { blockByHeight(tag) @@ -115,12 +115,11 @@ class EthereumCallSelector( return try { val blockId = BlockId.from(blockHash) heightReader.read(blockId) - .switchIfEmpty(Mono.justOrEmpty(head.getCurrentHeight())) - .map { Selector.HeightMatcher(it) } + .switchIfEmpty(Mono.justOrEmpty(head.getCurrentHeight())) + .map { Selector.HeightMatcher(it) } } catch (t: Throwable) { log.warn("Invalid blockHash: $blockHash") Mono.empty() } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/ManagedCallMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/ManagedCallMethods.kt index be9f1981..ca2299c1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/ManagedCallMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/ManagedCallMethods.kt @@ -21,8 +21,7 @@ import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.quorum.NonEmptyQuorum import io.emeraldpay.dshackle.quorum.NotLaggingQuorum import org.slf4j.LoggerFactory -import java.util.* -import kotlin.collections.HashMap +import java.util.Collections /** * Wrapper on top of another configuration, that may disable or enable additional methods on top of it. @@ -30,10 +29,10 @@ import kotlin.collections.HashMap * parent config. */ class ManagedCallMethods( - private val delegate: CallMethods, - private val enabled: Set, - private val disabled: Set -): CallMethods { + private val delegate: CallMethods, + private val enabled: Set, + private val disabled: Set +) : CallMethods { companion object { private val log = LoggerFactory.getLogger(ManagedCallMethods::class.java) @@ -42,7 +41,7 @@ class ManagedCallMethods( private val delegated = delegate.getSupportedMethods().sorted() private val allAllowed: Set = Collections.unmodifiableSet( - enabled + delegated - disabled + enabled + delegated - disabled ) private val quorum: MutableMap = HashMap() @@ -91,4 +90,4 @@ class ManagedCallMethods( override fun executeHardcoded(method: String): ByteArray { return delegate.executeHardcoded(method) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/CacheRequested.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/CacheRequested.kt index 4b864916..83e5d142 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/CacheRequested.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/CacheRequested.kt @@ -20,13 +20,12 @@ import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.DefaultContainer import io.emeraldpay.dshackle.data.TxId -import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.RequestPostprocessor import io.emeraldpay.etherjar.rpc.json.TransactionReceiptJson import org.slf4j.LoggerFactory class CacheRequested( - private val caches: Caches + private val caches: Caches ) : RequestPostprocessor { companion object { @@ -50,13 +49,12 @@ class CacheRequested( // note: json could be a `null` value val parsed = Global.objectMapper.readValue(json, TransactionReceiptJson::class.java) ?: return val value = DefaultContainer( - TxId.from(parsed.transactionHash), - BlockId.from(parsed.blockHash), - parsed.blockNumber, - json, - parsed + TxId.from(parsed.transactionHash), + BlockId.from(parsed.blockHash), + parsed.blockNumber, + json, + parsed ) caches.cacheReceipt(Caches.Tag.REQUESTED, value) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt index 3ccd4f8d..8fbe804d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/DefaultEthereumHead.kt @@ -34,29 +34,29 @@ open class DefaultEthereumHead : Head, AbstractHead() { fun getLatestBlock(api: Reader): Mono { return api.read(JsonRpcRequest("eth_blockNumber", emptyList())) - .subscribeOn(EthereumRpcHead.scheduler) - .timeout(Defaults.timeout, Mono.error(Exception("Block number not received"))) - .flatMap { - if (it.error != null) { - Mono.error(it.error.asException(null)) - } else { - val value = it.getResultAsProcessedString() - Mono.just(HexQuantity.from(value)) - } - } - .flatMap { - //fetching by Block Height here, critical to use the same upstream as in previous call, - //b/c different upstreams may have different blocks on the same height - api.read(JsonRpcRequest("eth_getBlockByNumber", listOf(it.toHex(), false))) - .subscribeOn(EthereumRpcHead.scheduler) - .timeout(Defaults.timeout, Mono.error(Exception("Block data not received"))) - } - .map { - BlockContainer.fromEthereumJson(it.getResult()) - } - .onErrorResume { err -> - log.debug("Failed to fetch latest block: ${err.message}") - Mono.empty() + .subscribeOn(EthereumRpcHead.scheduler) + .timeout(Defaults.timeout, Mono.error(Exception("Block number not received"))) + .flatMap { + if (it.error != null) { + Mono.error(it.error.asException(null)) + } else { + val value = it.getResultAsProcessedString() + Mono.just(HexQuantity.from(value)) } + } + .flatMap { + // fetching by Block Height here, critical to use the same upstream as in previous call, + // b/c different upstreams may have different blocks on the same height + api.read(JsonRpcRequest("eth_getBlockByNumber", listOf(it.toHex(), false))) + .subscribeOn(EthereumRpcHead.scheduler) + .timeout(Defaults.timeout, Mono.error(Exception("Block data not received"))) + } + .map { + BlockContainer.fromEthereumJson(it.getResult()) + } + .onErrorResume { err -> + log.debug("Failed to fetch latest block: ${err.message}") + Mono.empty() + } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt index fe2daa70..8aea6350 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt @@ -34,10 +34,10 @@ import java.util.concurrent.TimeoutException * Common reads from upstream, makes actual calls with applying quorum and retries */ class EthereumDirectReader( - private val up: Multistream, - private val caches: Caches, - private val balanceCache: CurrentBlockCache, - private val callMethodsFactory: Factory + private val up: Multistream, + private val caches: Caches, + private val balanceCache: CurrentBlockCache, + private val callMethodsFactory: Factory ) { companion object { @@ -69,22 +69,21 @@ class EthereumDirectReader( override fun read(key: TransactionId): Mono { val request = JsonRpcRequest("eth_getTransactionByHash", listOf(key.toHex())) return readWithQuorum(request) - .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Tx not read $key"))) - .retryWhen(Retry.backoff(3, Duration.ofSeconds(1))) - .flatMap { txbytes -> - val tx = objectMapper.readValue(txbytes, TransactionJson::class.java) - if (tx == null) { - Mono.empty() - } else { - Mono.just(TxContainer.from(tx, txbytes)) - } + .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Tx not read $key"))) + .retryWhen(Retry.backoff(3, Duration.ofSeconds(1))) + .flatMap { txbytes -> + val tx = objectMapper.readValue(txbytes, TransactionJson::class.java) + if (tx == null) { + Mono.empty() + } else { + Mono.just(TxContainer.from(tx, txbytes)) } - .doOnNext { tx -> - if (tx.blockId != null) { - caches.cache(Caches.Tag.REQUESTED, tx) - } + } + .doOnNext { tx -> + if (tx.blockId != null) { + caches.cache(Caches.Tag.REQUESTED, tx) } - + } } } balanceReader = object : Reader { @@ -92,20 +91,20 @@ class EthereumDirectReader( val height = up.getHead().getCurrentHeight()?.let { HexQuantity.from(it).toHex() } ?: "latest" val request = JsonRpcRequest("eth_getBalance", listOf(key.toHex(), height)) return readWithQuorum(request) - .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Balance not read $key"))) - .map { - val str = String(it) - // it's a json string, i.e. wrapped with quotes, ex. _"0x1234"_ - if (str.startsWith("\"") && str.endsWith("\"")) { - Wei.from(str.substring(1, str.length - 1)) - } else { - throw RpcException(RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, "Not Wei value") - } - } - .retryWhen(Retry.backoff(3, Duration.ofSeconds(1))) - .doOnNext { value -> - balanceCache.put(key, value) + .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Balance not read $key"))) + .map { + val str = String(it) + // it's a json string, i.e. wrapped with quotes, ex. _"0x1234"_ + if (str.startsWith("\"") && str.endsWith("\"")) { + Wei.from(str.substring(1, str.length - 1)) + } else { + throw RpcException(RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, "Not Wei value") } + } + .retryWhen(Retry.backoff(3, Duration.ofSeconds(1))) + .doOnNext { value -> + balanceCache.put(key, value) + } } } } @@ -113,19 +112,19 @@ class EthereumDirectReader( @Suppress("UNCHECKED_CAST") private fun readBlock(request: JsonRpcRequest, id: String): Mono { return readWithQuorum(request) - .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Block not read $id"))) - .retryWhen(Retry.backoff(3, Duration.ofSeconds(1))) - .flatMap { blockbytes -> - val block = objectMapper.readValue(blockbytes, BlockJson::class.java) as BlockJson? - if (block == null) { - Mono.empty() - } else { - Mono.just(BlockContainer.from(block, blockbytes)) - } - } - .doOnNext { block -> - caches.cache(Caches.Tag.REQUESTED, block) + .timeout(Defaults.timeoutInternal, Mono.error(TimeoutException("Block not read $id"))) + .retryWhen(Retry.backoff(3, Duration.ofSeconds(1))) + .flatMap { blockbytes -> + val block = objectMapper.readValue(blockbytes, BlockJson::class.java) as BlockJson? + if (block == null) { + Mono.empty() + } else { + Mono.just(BlockContainer.from(block, blockbytes)) } + } + .doOnNext { block -> + caches.cache(Caches.Tag.REQUESTED, block) + } } /** @@ -133,8 +132,8 @@ class EthereumDirectReader( */ private fun readWithQuorum(request: JsonRpcRequest): Mono { return quorumReaderFactory - .create(up.getApiSource(Selector.empty), callMethodsFactory.create().getQuorumFor(request.method)) - .read(request) - .map { it.value } + .create(up.getApiSource(Selector.empty), callMethodsFactory.create().getQuorumFor(request.method)) + .read(request) + .map { it.value } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt index 97843fb4..e7eab2c7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt @@ -36,8 +36,8 @@ import java.util.function.BiFunction * If any of the expected block transactions is not available it returns empty */ class EthereumFullBlocksReader( - private val blocks: Reader, - private val txes: Reader + private val blocks: Reader, + private val txes: Reader ) : Reader { companion object { @@ -58,11 +58,11 @@ class EthereumFullBlocksReader( if (block.transactions.isEmpty()) { // in fact it's not necessary to create a copy, made just for code clarity but it may be a performance loss val fullBlock = BlockContainer( - block.height, block.hash, block.difficulty, block.timestamp, - true, - block.json, - block.parsed, - block.transactions + block.height, block.hash, block.difficulty, block.timestamp, + true, + block.json, + block.parsed, + block.transactions ) return@flatMap Mono.just(fullBlock) } @@ -70,8 +70,8 @@ class EthereumFullBlocksReader( val blockSplit = splitByTransactions(block.json!!) val transactions = Flux.fromIterable(block.transactions) - .flatMapSequential { txes.read(it) } - .collectList() + .flatMapSequential { txes.read(it) } + .collectList() return@flatMap transactions.flatMap { transactionsData -> // make sure that all transaction are loaded, otherwise just return empty because cannot make full block data @@ -79,17 +79,22 @@ class EthereumFullBlocksReader( log.warn("No data to fill the block") Mono.empty() } else { - joinWithTransactions(blockSplit.t1, blockSplit.t2, Flux.fromIterable(transactionsData).map { it.json!! }) - .reduce(ByteBuffer.allocate(block.json.size * 4), accumulate) - .map(this@EthereumFullBlocksReader::extractContent) - .map { json -> - BlockContainer(block.height, block.hash, block.difficulty, block.timestamp, - true, - json, - null, - block.transactions - ) - } + joinWithTransactions( + blockSplit.t1, + blockSplit.t2, + Flux.fromIterable(transactionsData).map { it.json!! } + ) + .reduce(ByteBuffer.allocate(block.json.size * 4), accumulate) + .map(this@EthereumFullBlocksReader::extractContent) + .map { json -> + BlockContainer( + block.height, block.hash, block.difficulty, block.timestamp, + true, + json, + null, + block.transactions + ) + } } } } @@ -103,7 +108,7 @@ class EthereumFullBlocksReader( } fun splitByTransactions(json: ByteArray): Tuple2 { - //TODO find a lib that implements Knuth-Morris-Pratt Pattern Matching Algorithm for byte arrays + // TODO find a lib that implements Knuth-Morris-Pratt Pattern Matching Algorithm for byte arrays // and reimplement without making a string copy from bytes val s = String(json) @@ -119,23 +124,22 @@ class EthereumFullBlocksReader( fun joinWithTransactions(head: ByteArray, tail: ByteArray, transactions: Flux): Flux { val separator = Flux.range(0, Integer.MAX_VALUE) - .map { it != 0 } + .map { it != 0 } val transactionsWithSeparator = transactions.zipWith(separator) - .flatMap { - val tx = Flux.just(it.t1) - if (it.t2) { - Flux.concat(Flux.just(",".toByteArray()), tx) - } else { - tx - } + .flatMap { + val tx = Flux.just(it.t1) + if (it.t2) { + Flux.concat(Flux.just(",".toByteArray()), tx) + } else { + tx } + } return Flux.concat( - Flux.just(head), - transactionsWithSeparator, - Flux.just(tail) + Flux.just(head), + transactionsWithSeparator, + Flux.just(tail) ) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt index 59ac966b..9d9d3b87 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumHeadLagObserver.kt @@ -16,17 +16,15 @@ */ package io.emeraldpay.dshackle.upstream.ethereum -import io.emeraldpay.dshackle.upstream.HeadLagObserver -import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.HeadLagObserver +import io.emeraldpay.dshackle.upstream.Upstream import org.slf4j.LoggerFactory -import reactor.core.publisher.Flux -import java.time.Duration class EthereumHeadLagObserver( - master: Head, - followers: Collection + master: Head, + followers: Collection ) : HeadLagObserver(master, followers) { companion object { @@ -34,8 +32,7 @@ class EthereumHeadLagObserver( } override fun forkDistance(top: BlockContainer, curr: BlockContainer): Long { - //TODO look for common ancestor? though it may be a corruption + // TODO look for common ancestor? though it may be a corruption return 6 } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt index 21cc3cf6..484e3881 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumMultistream.kt @@ -19,7 +19,11 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.MergedHead +import io.emeraldpay.dshackle.upstream.Multistream +import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse import io.emeraldpay.grpc.Chain @@ -29,9 +33,9 @@ import reactor.core.publisher.Mono @Suppress("UNCHECKED_CAST") open class EthereumMultistream( - chain: Chain, - val upstreams: MutableList, - caches: Caches + chain: Chain, + val upstreams: MutableList, + caches: Caches ) : Multistream(chain, upstreams as MutableList, caches, CacheRequested(caches)) { companion object { @@ -126,4 +130,4 @@ open class EthereumMultistream( open fun getSubscribe(): EthereumSubscribe { return subscribe } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt index df19db35..607582f2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumReader.kt @@ -20,12 +20,18 @@ import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CurrentBlockCache import io.emeraldpay.dshackle.cache.HeightByHashAdding -import io.emeraldpay.dshackle.data.* -import io.emeraldpay.dshackle.reader.* +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.data.SourceContainer +import io.emeraldpay.dshackle.data.TxContainer +import io.emeraldpay.dshackle.data.TxId +import io.emeraldpay.dshackle.reader.CompoundReader +import io.emeraldpay.dshackle.reader.Reader +import io.emeraldpay.dshackle.reader.RekeyingReader +import io.emeraldpay.dshackle.reader.RpcReader +import io.emeraldpay.dshackle.reader.TransformingReader import io.emeraldpay.dshackle.upstream.Multistream -import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.calls.CallMethods -import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.domain.BlockHash import io.emeraldpay.etherjar.domain.TransactionId @@ -42,9 +48,9 @@ import java.util.function.Function * Reader for the common operations, that wraps caches + native call with quorum verification */ open class EthereumReader( - private val up: Multistream, - private val caches: Caches, - private val callMethodsFactory: Factory + private val up: Multistream, + private val caches: Caches, + private val callMethodsFactory: Factory ) : Lifecycle { companion object { @@ -61,8 +67,8 @@ open class EthereumReader( existing.withoutTransactionDetails() } else { objectMapper - .readValue(block.json, BlockJson::class.java) - .withoutTransactionDetails() + .readValue(block.json, BlockJson::class.java) + .withoutTransactionDetails() } } @@ -81,30 +87,30 @@ open class EthereumReader( private val idToTxHash = Function { id -> TransactionId.from(id.value) } private val blocksByIdAsCont = CompoundReader( - caches.getBlocksByHash(), - RekeyingReader(idToBlockHash, directReader.blockReader) + caches.getBlocksByHash(), + RekeyingReader(idToBlockHash, directReader.blockReader) ) private val heightByHash = HeightByHashAdding(caches, blocksByIdAsCont) fun blocksByHashAsCont(): Reader { return CompoundReader( - RekeyingReader(blockHashToId, caches.getBlocksByHash()), - directReader.blockReader + RekeyingReader(blockHashToId, caches.getBlocksByHash()), + directReader.blockReader ) } fun blocksByHashParsed(): Reader> { return TransformingReader( - blocksByHashAsCont(), - extractBlock + blocksByHashAsCont(), + extractBlock ) } fun blocksByIdParsed(): Reader> { return TransformingReader( - blocksByIdAsCont(), - extractBlock + blocksByIdAsCont(), + extractBlock ) } @@ -114,43 +120,44 @@ open class EthereumReader( open fun blocksByHeightAsCont(): Reader { return CompoundReader( - caches.getBlocksByHeight(), - directReader.blockByHeightReader + caches.getBlocksByHeight(), + directReader.blockByHeightReader ) } fun txByHash(): Reader { return TransformingReader( - CompoundReader( - RekeyingReader(txHashToId, caches.getTxByHash()), - directReader.txReader - ), - extractTx + CompoundReader( + RekeyingReader(txHashToId, caches.getTxByHash()), + directReader.txReader + ), + extractTx ) } open fun txByHashAsCont(): Reader { return CompoundReader( - caches.getTxByHash(), - RekeyingReader(idToTxHash, directReader.txReader) + caches.getTxByHash(), + RekeyingReader(idToTxHash, directReader.txReader) ) } fun balance(): Reader { - //TODO include height as part of cache? + // TODO include height as part of cache? return CompoundReader( - balanceCache, directReader.balanceReader + balanceCache, directReader.balanceReader ) } fun receipts(): Reader { - //TODO put into cache + // TODO put into cache val requested = RekeyingReader( - { txid: TxId -> txid.toHexWithPrefix() }, - RpcReader.basicRequest(up, "eth_getTransactionReceipt")) + { txid: TxId -> txid.toHexWithPrefix() }, + RpcReader.basicRequest(up, "eth_getTransactionReceipt") + ) return CompoundReader( - caches.getReceipts(), - requested + caches.getReceipts(), + requested ) } @@ -159,7 +166,7 @@ open class EthereumReader( } override fun isRunning(): Boolean { - //TODO should be always running? + // TODO should be always running? return up.isRunning } @@ -172,4 +179,4 @@ open class EthereumReader( override fun stop() { } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt index 960e2266..bb5099d9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcHead.kt @@ -16,29 +16,26 @@ */ package io.emeraldpay.dshackle.upstream.ethereum -import io.emeraldpay.dshackle.Defaults -import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.etherjar.hex.HexQuantity import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle import org.springframework.scheduling.concurrent.CustomizableThreadFactory import reactor.core.Disposable import reactor.core.publisher.Flux -import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers import java.time.Duration import java.util.concurrent.Executors class EthereumRpcHead( - private val api: Reader, - private val interval: Duration = Duration.ofSeconds(10) -): DefaultEthereumHead(), Lifecycle { + private val api: Reader, + private val interval: Duration = Duration.ofSeconds(10) +) : DefaultEthereumHead(), Lifecycle { companion object { - val scheduler = Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("ethereum-rpc-head"))) + val scheduler = + Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("ethereum-rpc-head"))) } private val log = LoggerFactory.getLogger(EthereumRpcHead::class.java) @@ -47,10 +44,10 @@ class EthereumRpcHead( override fun start() { val base = Flux.interval(interval) - .publishOn(scheduler) - .flatMap { - getLatestBlock(api) - } + .publishOn(scheduler) + .flatMap { + getLatestBlock(api) + } refreshSubscription = super.follow(base) } @@ -62,5 +59,4 @@ class EthereumRpcHead( refreshSubscription?.dispose() refreshSubscription = null } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt index b354abc9..6846da0b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumRpcUpstream.kt @@ -5,7 +5,10 @@ import io.emeraldpay.dshackle.cache.CachesEnabled import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.MergedHead +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest @@ -17,22 +20,23 @@ import reactor.core.Disposable import java.time.Duration open class EthereumRpcUpstream( - id: String, - val chain: Chain, - private val directReader: Reader, - private val ethereumWsFactory: EthereumWsFactory? = null, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - private val node: QuorumForLabels.QuorumItem, - targets: CallMethods + id: String, + val chain: Chain, + private val directReader: Reader, + private val ethereumWsFactory: EthereumWsFactory? = null, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + private val node: QuorumForLabels.QuorumItem, + targets: CallMethods ) : EthereumUpstream(id, options, role, targets, node), Upstream, CachesEnabled, Lifecycle { constructor(id: String, chain: Chain, api: Reader) : - this(id, chain, api, null, - UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.UpstreamRole.STANDARD, - QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()), - DirectCallMethods()) - + this( + id, chain, api, null, + UpstreamsConfig.Options.getDefaults(), UpstreamsConfig.UpstreamRole.STANDARD, + QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels()), + DirectCallMethods() + ) private val log = LoggerFactory.getLogger(EthereumRpcUpstream::class.java) @@ -56,7 +60,7 @@ open class EthereumRpcUpstream( log.debug("Start validation for upstream ${this.getId()}") val validator = EthereumUpstreamValidator(this, getOptions()) validatorSubscription = validator.start() - .subscribe(this::setStatus) + .subscribe(this::setStatus) } } @@ -115,4 +119,4 @@ open class EthereumRpcUpstream( } return this as T } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt index bb00f6a1..346a4f1c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumSubscribe.kt @@ -3,14 +3,13 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectLogs import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectNewHeads import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectSyncing -import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ProduceLogs import io.emeraldpay.etherjar.domain.Address import io.emeraldpay.etherjar.hex.Hex32 import org.slf4j.LoggerFactory import reactor.core.publisher.Flux open class EthereumSubscribe( - val upstream: EthereumMultistream + val upstream: EthereumMultistream ) { companion object { @@ -45,8 +44,8 @@ open class EthereumSubscribe( } data class LogsRequest( - val address: List
, - val topics: List + val address: List
, + val topics: List ) fun readLogsRequest(params: Map): LogsRequest { @@ -94,4 +93,4 @@ open class EthereumSubscribe( } return LogsRequest(addresses, topics) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt index f629373e..c14bd120 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstream.kt @@ -18,15 +18,16 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.calls.CallMethods abstract class EthereumUpstream( - id: String, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - targets: CallMethods?, - private val node: QuorumForLabels.QuorumItem? + id: String, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + targets: CallMethods?, + private val node: QuorumForLabels.QuorumItem? ) : DefaultUpstream(id, options, role, targets, node) { private val capabilities = if (options.providesBalance != false) { @@ -42,4 +43,4 @@ abstract class EthereumUpstream( override fun getLabels(): Collection { return node?.let { listOf(it.labels) } ?: emptyList() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt index a968753a..f7f6bfcc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt @@ -33,51 +33,52 @@ import java.time.Duration import java.util.concurrent.Executors open class EthereumUpstreamValidator( - private val upstream: EthereumUpstream, - private val options: UpstreamsConfig.Options + private val upstream: EthereumUpstream, + private val options: UpstreamsConfig.Options ) { companion object { private val log = LoggerFactory.getLogger(EthereumUpstreamValidator::class.java) - val scheduler = Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("ethereum-validator"))) + val scheduler = + Schedulers.fromExecutor(Executors.newCachedThreadPool(CustomizableThreadFactory("ethereum-validator"))) } private val objectMapper: ObjectMapper = Global.objectMapper open fun validate(): Mono { return upstream - .getApi() - .read(JsonRpcRequest("eth_syncing", listOf())) - .flatMap(JsonRpcResponse::requireResult) - .map { objectMapper.readValue(it, SyncingJson::class.java) } - .timeout(Defaults.timeoutInternal, Mono.error(Exception("Validation timeout for Syncing"))) - .flatMap { value -> - if (value.isSyncing) { - Mono.just(UpstreamAvailability.SYNCING) - } else { - upstream - .getApi() - .read(JsonRpcRequest("net_peerCount", listOf())) - .flatMap(JsonRpcResponse::requireStringResult) - .map(Integer::decode) - .timeout(Defaults.timeoutInternal, Mono.error(Exception("Validation timeout for Peers"))) - .map { count -> - val minPeers = options.minPeers ?: 1 - if (count < minPeers) { - UpstreamAvailability.IMMATURE - } else { - UpstreamAvailability.OK - } - } - } + .getApi() + .read(JsonRpcRequest("eth_syncing", listOf())) + .flatMap(JsonRpcResponse::requireResult) + .map { objectMapper.readValue(it, SyncingJson::class.java) } + .timeout(Defaults.timeoutInternal, Mono.error(Exception("Validation timeout for Syncing"))) + .flatMap { value -> + if (value.isSyncing) { + Mono.just(UpstreamAvailability.SYNCING) + } else { + upstream + .getApi() + .read(JsonRpcRequest("net_peerCount", listOf())) + .flatMap(JsonRpcResponse::requireStringResult) + .map(Integer::decode) + .timeout(Defaults.timeoutInternal, Mono.error(Exception("Validation timeout for Peers"))) + .map { count -> + val minPeers = options.minPeers ?: 1 + if (count < minPeers) { + UpstreamAvailability.IMMATURE + } else { + UpstreamAvailability.OK + } + } } - .onErrorReturn(UpstreamAvailability.UNAVAILABLE) + } + .onErrorReturn(UpstreamAvailability.UNAVAILABLE) } fun start(): Flux { return Flux.interval(Duration.ofSeconds(15)) - .subscribeOn(scheduler) - .flatMap { - validate() - } + .subscribeOn(scheduler) + .flatMap { + validate() + } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt index 96752883..09ffd1be 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsFactory.kt @@ -24,7 +24,11 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.UpstreamAvailability -import io.emeraldpay.dshackle.upstream.rpcclient.* +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcError +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest +import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse +import io.emeraldpay.dshackle.upstream.rpcclient.ResponseWSParser +import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics import io.emeraldpay.etherjar.rpc.RpcResponseError import io.emeraldpay.etherjar.rpc.json.BlockJson import io.emeraldpay.etherjar.rpc.json.TransactionRefJson @@ -51,16 +55,15 @@ import reactor.retry.Repeat import reactor.util.function.Tuples import java.net.URI import java.time.Duration -import java.util.* +import java.util.Base64 import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger - class EthereumWsFactory( - private val uri: URI, - private val origin: URI + private val uri: URI, + private val origin: URI ) { var basicAuth: AuthConfig.ClientBasicAuth? = null @@ -78,19 +81,20 @@ class EthereumWsFactory( } class EthereumWs( - private val uri: URI, - private val origin: URI, - private val basicAuth: AuthConfig.ClientBasicAuth?, - private val rpcMetrics: RpcMetrics?, - private val upstream: DefaultUpstream?, - private val validator: EthereumUpstreamValidator? + private val uri: URI, + private val origin: URI, + private val basicAuth: AuthConfig.ClientBasicAuth?, + private val rpcMetrics: RpcMetrics?, + private val upstream: DefaultUpstream?, + private val validator: EthereumUpstreamValidator? ) : AutoCloseable { companion object { private val log = LoggerFactory.getLogger(EthereumWs::class.java) private const val IDS_START = 100 - private const val START_REQUEST = "{\"jsonrpc\":\"2.0\", \"method\":\"eth_subscribe\", \"id\":\"blocks\", \"params\":[\"newHeads\"]}" + private const val START_REQUEST = + "{\"jsonrpc\":\"2.0\", \"method\":\"eth_subscribe\", \"id\":\"blocks\", \"params\":[\"newHeads\"]}" // WebSocket Frame limit. // Default is 65_536, but Geth responds with larger frames, @@ -116,17 +120,17 @@ class EthereumWsFactory( private val parser = ResponseWSParser() private val blocks = Sinks - .many() - .multicast() - .directBestEffort() + .many() + .multicast() + .directBestEffort() private var rpcSend = Sinks - .many() - .unicast() - .onBackpressureBuffer() + .many() + .unicast() + .onBackpressureBuffer() private val rpcReceive = Sinks - .many() - .multicast() - .directBestEffort() + .many() + .multicast() + .directBestEffort() private val sendIdSeq = AtomicInteger(IDS_START) private val sendExecutor = Executors.newSingleThreadExecutor() private var keepConnection = true @@ -155,9 +159,9 @@ class EthereumWsFactory( // so we need to create a new Sink. Emit Complete is probably useless, and just in case rpcSend.tryEmitComplete() rpcSend = Sinks - .many() - .unicast() - .onBackpressureBuffer() + .many() + .unicast() + .onBackpressureBuffer() val retryInterval = currentBackOff.nextBackOff() if (retryInterval == BackOffExecution.STOP) { log.warn("Reconnect backoff exhausted. Permanently closing the connection") @@ -165,116 +169,117 @@ class EthereumWsFactory( } log.info("Reconnect to $uri in ${retryInterval}ms...") Global.control.schedule( - { - reconnecting.set(false) - connectInternal() - }, - retryInterval, TimeUnit.MILLISECONDS) + { + reconnecting.set(false) + connectInternal() + }, + retryInterval, TimeUnit.MILLISECONDS + ) } private fun connectInternal() { log.info("Connecting to WebSocket: $uri") connection?.dispose() connection = HttpClient.create() - .doOnDisconnected { - log.info("Disconnected from $uri") - // mark upstream as UNAVAIL - upstream?.setStatus(UpstreamAvailability.UNAVAILABLE) - if (keepConnection) { - tryReconnectLater() - } + .doOnDisconnected { + log.info("Disconnected from $uri") + // mark upstream as UNAVAIL + upstream?.setStatus(UpstreamAvailability.UNAVAILABLE) + if (keepConnection) { + tryReconnectLater() } - .doOnError( - { _, t -> - log.warn("Failed to connect to $uri. Error: ${t.message}") - // going to try to reconnect later - tryReconnectLater() - }, - { _, _ -> } - ) - .headers { headers -> - headers.add(HttpHeaderNames.ORIGIN, origin) - basicAuth?.let { auth -> - val tmp: String = auth.username + ":" + auth.password - val base64password = Base64.getEncoder().encodeToString(tmp.toByteArray()) - headers.add(HttpHeaderNames.AUTHORIZATION, "Basic $base64password") - } + } + .doOnError( + { _, t -> + log.warn("Failed to connect to $uri. Error: ${t.message}") + // going to try to reconnect later + tryReconnectLater() + }, + { _, _ -> } + ) + .headers { headers -> + headers.add(HttpHeaderNames.ORIGIN, origin) + basicAuth?.let { auth -> + val tmp: String = auth.username + ":" + auth.password + val base64password = Base64.getEncoder().encodeToString(tmp.toByteArray()) + headers.add(HttpHeaderNames.AUTHORIZATION, "Basic $base64password") } - .let { - if (uri.scheme == "wss") it.secure() else it - } - .websocket( - WebsocketClientSpec.builder() - .handlePing(true) - .compress(false) - .maxFramePayloadLength(frameSize) - .build() - ) - .uri(uri) - .handle { inbound, outbound -> - handle(inbound, outbound) - } - .onErrorResume { t -> - log.debug("Dropping WS connection to $uri. Error: ${t.message}") - Mono.empty() - } - .subscribe() + } + .let { + if (uri.scheme == "wss") it.secure() else it + } + .websocket( + WebsocketClientSpec.builder() + .handlePing(true) + .compress(false) + .maxFramePayloadLength(frameSize) + .build() + ) + .uri(uri) + .handle { inbound, outbound -> + handle(inbound, outbound) + } + .onErrorResume { t -> + log.debug("Dropping WS connection to $uri. Error: ${t.message}") + Mono.empty() + } + .subscribe() } fun handle(inbound: WebsocketInbound, outbound: WebsocketOutbound): Publisher { - //restart backoff after connection + // restart backoff after connection currentBackOff = reconnectBackoff.start() - //validate the connection, it can also be UNAVAIL if market as such after disconnect + // validate the connection, it can also be UNAVAIL if market as such after disconnect validator?.validate() val consumer = inbound - .aggregateFrames(msgSizeLimit) - .receiveFrames() - .map { ByteBufInputStream(it.content()).readAllBytes() } - .flatMap { - try { - val msg = parser.parse(it) - if (msg.type == ResponseWSParser.Type.SUBSCRIPTION) { - onSubscription(msg) - } else { - onRpc(msg) - } - } catch (t: Throwable) { - log.warn("Failed to process WS message. ${t.javaClass}: ${t.message}") - Mono.empty() + .aggregateFrames(msgSizeLimit) + .receiveFrames() + .map { ByteBufInputStream(it.content()).readAllBytes() } + .flatMap { + try { + val msg = parser.parse(it) + if (msg.type == ResponseWSParser.Type.SUBSCRIPTION) { + onSubscription(msg) + } else { + onRpc(msg) } - } - .onErrorResume { t -> - log.warn("Connection dropped to $uri. Error: ${t.message}", t) - // going to try to reconnect later - tryReconnectLater() - // completes current outbound flow + } catch (t: Throwable) { + log.warn("Failed to process WS message. ${t.javaClass}: ${t.message}") Mono.empty() } + } + .onErrorResume { t -> + log.warn("Connection dropped to $uri. Error: ${t.message}", t) + // going to try to reconnect later + tryReconnectLater() + // completes current outbound flow + Mono.empty() + } val start = Mono.just(START_REQUEST).map { Unpooled.wrappedBuffer(it.toByteArray()) } val calls = rpcSend - .asFlux() - .map { - Unpooled.wrappedBuffer(Global.objectMapper.writeValueAsBytes(it)) - } + .asFlux() + .map { + Unpooled.wrappedBuffer(Global.objectMapper.writeValueAsBytes(it)) + } return outbound.send( - Flux.merge( - start, - calls.subscribeOn(Schedulers.boundedElastic()), - consumer.then(Mono.empty()).subscribeOn(Schedulers.boundedElastic()) - ) + Flux.merge( + start, + calls.subscribeOn(Schedulers.boundedElastic()), + consumer.then(Mono.empty()).subscribeOn(Schedulers.boundedElastic()) + ) ) } fun onRpc(msg: ResponseWSParser.WsResponse): Mono { return if (msg.id.isNumber()) { val resp = JsonRpcResponse( - msg.value, msg.error, msg.id + msg.value, msg.error, msg.id ) Mono.fromCallable { val status = rpcReceive.tryEmitNext(resp) @@ -283,7 +288,7 @@ class EthereumWsFactory( } }.then() } else { - //it's a response to the newHeads subscription, just ignore it + // it's a response to the newHeads subscription, just ignore it Mono.empty() } } @@ -303,30 +308,30 @@ class EthereumWsFactory( // so we need to fetch the full block data return if (block.difficulty == null || block.transactions == null) { Mono.just(block.hash) - .flatMap { hash -> - call(JsonRpcRequest("eth_getBlockByHash", listOf(hash.toHex(), false))) - .flatMap { resp -> - if (resp.isNull()) { - Mono.error(SilentException("Received null for block $hash")) - } else { - Mono.just(resp) - } - } - .flatMap(JsonRpcResponse::requireResult) - .map { BlockContainer.fromEthereumJson(it) } - .subscribeOn(Schedulers.boundedElastic()) - .timeout(Defaults.timeoutInternal, Mono.empty()) - }.repeatWhenEmpty { n -> - Repeat.times(5) - .exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(500)) - .apply(n) - } - .timeout(Defaults.timeout, Mono.empty()) - .onErrorResume { Mono.empty() } - .doOnNext { - blocks.tryEmitNext(it) - } - .then() + .flatMap { hash -> + call(JsonRpcRequest("eth_getBlockByHash", listOf(hash.toHex(), false))) + .flatMap { resp -> + if (resp.isNull()) { + Mono.error(SilentException("Received null for block $hash")) + } else { + Mono.just(resp) + } + } + .flatMap(JsonRpcResponse::requireResult) + .map { BlockContainer.fromEthereumJson(it) } + .subscribeOn(Schedulers.boundedElastic()) + .timeout(Defaults.timeoutInternal, Mono.empty()) + }.repeatWhenEmpty { n -> + Repeat.times(5) + .exponentialBackoff(Duration.ofMillis(50), Duration.ofMillis(500)) + .apply(n) + } + .timeout(Defaults.timeout, Mono.empty()) + .onErrorResume { Mono.empty() } + .doOnNext { + blocks.tryEmitNext(it) + } + .then() } else { Mono.fromCallable { blocks.tryEmitNext(BlockContainer.from(block)) @@ -359,27 +364,31 @@ class EthereumWsFactory( fun waitForResponse(request: JsonRpcRequest, originalId: Int, startTime: Long): Mono { val expectedId = request.id.toLong() return Mono.just(request) - .flatMap { - Flux.from(rpcReceive.asFlux()) - .doOnSubscribe { sendRpc(request) } - .filter { resp -> resp.id.asNumber() == expectedId } - .take(Defaults.timeout) - .take(1) - .singleOrEmpty() - .doOnNext { - rpcMetrics?.timer?.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) - } - .doOnError { - rpcMetrics?.errors?.increment() - } - .map { it.copyWithId(JsonRpcResponse.Id.from(originalId)) } - .defaultIfEmpty( - JsonRpcResponse(null, - JsonRpcError(RpcResponseError.CODE_INTERNAL_ERROR, "Response not received from WebSocket"), - JsonRpcResponse.Id.from(originalId) - ) - ) - } + .flatMap { + Flux.from(rpcReceive.asFlux()) + .doOnSubscribe { sendRpc(request) } + .filter { resp -> resp.id.asNumber() == expectedId } + .take(Defaults.timeout) + .take(1) + .singleOrEmpty() + .doOnNext { + rpcMetrics?.timer?.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) + } + .doOnError { + rpcMetrics?.errors?.increment() + } + .map { it.copyWithId(JsonRpcResponse.Id.from(originalId)) } + .defaultIfEmpty( + JsonRpcResponse( + null, + JsonRpcError( + RpcResponseError.CODE_INTERNAL_ERROR, + "Response not received from WebSocket" + ), + JsonRpcResponse.Id.from(originalId) + ) + ) + } } fun getBlocksFlux(): Flux { @@ -393,6 +402,4 @@ class EthereumWsFactory( connection = null } } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt index c13a7cd3..fdd4850b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsHead.kt @@ -23,7 +23,7 @@ import reactor.core.Disposable import reactor.core.publisher.Flux class EthereumWsHead( - private val ws: EthereumWsFactory.EthereumWs + private val ws: EthereumWsFactory.EthereumWs ) : DefaultEthereumHead(), Lifecycle { private val log = LoggerFactory.getLogger(EthereumWsHead::class.java) @@ -37,9 +37,9 @@ class EthereumWsHead( override fun start() { this.subscription?.dispose() val heads = Flux.merge( - // get the current block, not just wait for the next update - getLatestBlock(JsonRpcWsClient(ws)), - ws.getBlocksFlux() + // get the current block, not just wait for the next update + getLatestBlock(JsonRpcWsClient(ws)), + ws.getBlocksFlux() ) this.subscription = super.follow(heads) } @@ -48,5 +48,4 @@ class EthereumWsHead( subscription?.dispose() subscription = null } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsUpstream.kt index 7046be7b..c09385f3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumWsUpstream.kt @@ -35,13 +35,13 @@ import org.springframework.context.Lifecycle import reactor.core.Disposable class EthereumWsUpstream( - id: String, - val chain: Chain, - ethereumWsFactory: EthereumWsFactory, - options: UpstreamsConfig.Options, - role: UpstreamsConfig.UpstreamRole, - node: QuorumForLabels.QuorumItem, - targets: CallMethods + id: String, + val chain: Chain, + ethereumWsFactory: EthereumWsFactory, + options: UpstreamsConfig.Options, + role: UpstreamsConfig.UpstreamRole, + node: QuorumForLabels.QuorumItem, + targets: CallMethods ) : EthereumUpstream(id, options, role, targets, node), Upstream, Lifecycle { companion object { @@ -57,20 +57,20 @@ class EthereumWsUpstream( init { val metricsTags = listOf( - Tag.of("upstream", id), - // UNSPECIFIED shouldn't happen too - Tag.of("chain", chain.chainCode) + Tag.of("upstream", id), + // UNSPECIFIED shouldn't happen too + Tag.of("chain", chain.chainCode) ) val metrics = RpcMetrics( - Timer.builder("upstream.ws.conn") - .description("Request time through a WebSocket JSON RPC connection") - .tags(metricsTags) - .publishPercentileHistogram() - .register(Metrics.globalRegistry), - Counter.builder("upstream.ws.err") - .description("Errors received on request through WebSocket JSON RPC connection") - .tags(metricsTags) - .register(Metrics.globalRegistry) + Timer.builder("upstream.ws.conn") + .description("Request time through a WebSocket JSON RPC connection") + .tags(metricsTags) + .publishPercentileHistogram() + .register(Metrics.globalRegistry), + Counter.builder("upstream.ws.err") + .description("Errors received on request through WebSocket JSON RPC connection") + .tags(metricsTags) + .register(Metrics.globalRegistry) ) validator = EthereumUpstreamValidator(this, getOptions()) @@ -106,7 +106,7 @@ class EthereumWsUpstream( log.debug("Start validation for upstream ${this.getId()}") validatorSubscription = validator.start() - .subscribe(this::setStatus) + .subscribe(this::setStatus) } override fun stop() { @@ -119,4 +119,4 @@ class EthereumWsUpstream( override fun isRunning(): Boolean { return head.isRunning } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NativeCallRouter.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NativeCallRouter.kt index e05813ec..3808d1d9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NativeCallRouter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NativeCallRouter.kt @@ -37,9 +37,9 @@ import java.math.BigInteger * @see EthereumReader */ class NativeCallRouter( - private val reader: EthereumReader, - private val methods: CallMethods, - private val head: Head + private val reader: EthereumReader, + private val methods: CallMethods, + private val head: Head ) : Reader { companion object { @@ -47,14 +47,14 @@ class NativeCallRouter( } private val fullBlocksReader = EthereumFullBlocksReader( - reader.blocksByIdAsCont(), - reader.txByHashAsCont() + reader.blocksByIdAsCont(), + reader.txByHashAsCont() ) override fun read(key: JsonRpcRequest): Mono { if (methods.isHardcoded(key.method)) { return Mono.just(methods.executeHardcoded(key.method)) - .map { JsonRpcResponse(it, null) } + .map { JsonRpcResponse(it, null) } } if (!methods.isAllowed(key.method)) { return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) @@ -159,7 +159,7 @@ class NativeCallRouter( } val withTx = params[1].toString().toBoolean() var block = reader.blocksByHeightAsCont() - .read(number) + .read(number) block = if (withTx) { block.flatMap { fullBlocksReader.read(it.hash) @@ -169,4 +169,4 @@ class NativeCallRouter( } return block.map { it.json!! } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt index 603d1215..3c59b7de 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectBlockUpdates.kt @@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.core.scheduler.Schedulers import java.time.Duration -import java.util.* +import java.util.LinkedList import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.concurrent.read @@ -32,7 +32,7 @@ import kotlin.concurrent.withLock import kotlin.concurrent.write class ConnectBlockUpdates( - private val upstream: EthereumMultistream + private val upstream: EthereumMultistream ) { companion object { @@ -61,13 +61,13 @@ class ConnectBlockUpdates( return currentRecheck } val created = extract(upstream.getHead()) - .publishOn(Schedulers.boundedElastic()) - .publish() - .refCount(1, Duration.ofSeconds(60)) - .doFinally { - //forget it on disconnect, so next time it's recreated - connected = null - } + .publishOn(Schedulers.boundedElastic()) + .publish() + .refCount(1, Duration.ofSeconds(60)) + .doFinally { + // forget it on disconnect, so next time it's recreated + connected = null + } connected = created return created } @@ -75,7 +75,7 @@ class ConnectBlockUpdates( fun extract(head: Head): Flux { return head.getFlux() - .flatMap(this@ConnectBlockUpdates::extract) + .flatMap(this@ConnectBlockUpdates::extract) } fun extract(block: BlockContainer): Flux { @@ -117,35 +117,35 @@ class ConnectBlockUpdates( fun whenReplaced(prev: BlockContainer): Flux { return Flux.fromIterable(prev.transactions).map { Update( - prev.hash, - prev.height, - UpdateType.DROP, - it + prev.hash, + prev.height, + UpdateType.DROP, + it ) } } fun extractUpdates(block: BlockContainer): Flux { return Flux.fromIterable(block.transactions) - .map { - Update( - block.hash, - block.height, - UpdateType.NEW, - it - ) - } + .map { + Update( + block.hash, + block.height, + UpdateType.NEW, + it + ) + } } data class Update( - val blockHash: BlockId, - val blockNumber: Long, - val type: UpdateType, - val transactionId: TxId, + val blockHash: BlockId, + val blockNumber: Long, + val type: UpdateType, + val transactionId: TxId, ) enum class UpdateType { NEW, DROP } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt index 6e1a8d5c..d1bda8e7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectLogs.kt @@ -25,8 +25,8 @@ import reactor.core.publisher.Flux import java.util.function.Function class ConnectLogs( - upstream: EthereumMultistream, - private val connectBlockUpdates: ConnectBlockUpdates, + upstream: EthereumMultistream, + private val connectBlockUpdates: ConnectBlockUpdates, ) { companion object { @@ -51,19 +51,25 @@ class ConnectLogs( } // filtered output return start() - .transform(filtered(addresses, topics)) + .transform(filtered(addresses, topics)) } fun filtered(addresses: List
, topics: List): Function, Flux> { - //sort search criteria to use binary search later + // sort search criteria to use binary search later val sortedAddresses: List
= addresses.sortedWith(ADDR_COMPARATOR) val sortedTopics: List = topics.sortedWith(TOPIC_COMPARATOR) return Function { logs -> logs.filter { - val goodAddress = sortedAddresses.isEmpty() || sortedAddresses.binarySearch(it.address, ADDR_COMPARATOR) >= 0 - val goodTopic = sortedTopics.isEmpty() || (it.topics.isNotEmpty() && sortedTopics.binarySearch(it.topics[0], TOPIC_COMPARATOR) >= 0) + val goodAddress = + sortedAddresses.isEmpty() || sortedAddresses.binarySearch(it.address, ADDR_COMPARATOR) >= 0 + val goodTopic = sortedTopics.isEmpty() || ( + it.topics.isNotEmpty() && sortedTopics.binarySearch( + it.topics[0], + TOPIC_COMPARATOR + ) >= 0 + ) goodAddress && goodTopic } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt index 64e15af2..0bcc1f71 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectNewHeads.kt @@ -28,7 +28,7 @@ import kotlin.concurrent.withLock * Connects/reconnects to the upstream to produce NewHeads messages */ class ConnectNewHeads( - private val upstream: EthereumMultistream + private val upstream: EthereumMultistream ) { companion object { @@ -49,16 +49,16 @@ class ConnectNewHeads( return currentRecheck } val created = ProduceNewHeads(upstream.getHead()) - .start() - .publishOn(Schedulers.boundedElastic()) - .publish() - .refCount(1, Duration.ofSeconds(60)) - .doFinally { - //forget it on disconnect, so next time it's recreated - connected = null - } + .start() + .publishOn(Schedulers.boundedElastic()) + .publish() + .refCount(1, Duration.ofSeconds(60)) + .doFinally { + // forget it on disconnect, so next time it's recreated + connected = null + } connected = created return created } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectSyncing.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectSyncing.kt index bc224d56..f1f2e821 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectSyncing.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ConnectSyncing.kt @@ -24,7 +24,7 @@ import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock class ConnectSyncing( - private val upstream: EthereumMultistream + private val upstream: EthereumMultistream ) { companion object { @@ -45,15 +45,15 @@ class ConnectSyncing( return currentRecheck } val created = upstream.observeStatus() - .map { it != UpstreamAvailability.OK } - .publish() - .refCount(1, Duration.ofSeconds(60)) - .doFinally { - //forget it on disconnect, so next time it's recreated - connected = null - } + .map { it != UpstreamAvailability.OK } + .publish() + .refCount(1, Duration.ofSeconds(60)) + .doFinally { + // forget it on disconnect, so next time it's recreated + connected = null + } connected = created return created } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt index 65966870..d6c1620a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceLogs.kt @@ -31,7 +31,7 @@ import reactor.kotlin.core.publisher.switchIfEmpty import java.util.concurrent.TimeUnit class ProduceLogs( - private val receipts: Reader + private val receipts: Reader ) { companion object { @@ -45,8 +45,8 @@ class ProduceLogs( // need to keep history of recent messages in case they get removed. cannot rely on // any other cache or upstream because if when it gets removed it's unavailable in any other source private val oldMessages = CacheBuilder.newBuilder() - .expireAfterWrite(5, TimeUnit.HOURS) - .build>() + .expireAfterWrite(5, TimeUnit.HOURS) + .build>() fun produce(block: Flux): Flux { return block.flatMap { update -> @@ -65,43 +65,43 @@ class ProduceLogs( return Flux.empty() } return Flux.fromIterable(old) - .map { it.copy(removed = true) } + .map { it.copy(removed = true) } } fun produceAdded(update: ConnectBlockUpdates.Update): Flux { return receipts.read(update.transactionId) - .switchIfEmpty { - log.warn("Cannot find receipt for tx ${update.transactionId}") - Mono.empty() - } - .map { objectMapper.readValue(it, TransactionReceiptJson::class.java) } - .flatMapMany { receipt -> - try { - val messages = receipt.logs - .map { txlog -> - LogMessage( - txlog.address, - txlog.blockHash, - txlog.blockNumber, - txlog.data ?: HexData.empty(), - txlog.logIndex, - txlog.topics, - txlog.transactionHash, - txlog.transactionIndex, - false - ) - } - oldMessages.put(LogReference(update.blockHash, update.transactionId), messages) - Flux.fromIterable(messages) - } catch (t: Throwable) { - log.warn("Invalid Receipt ${update.transactionId}. ${t.javaClass}: ${t.message}") - Flux.empty() - } + .switchIfEmpty { + log.warn("Cannot find receipt for tx ${update.transactionId}") + Mono.empty() + } + .map { objectMapper.readValue(it, TransactionReceiptJson::class.java) } + .flatMapMany { receipt -> + try { + val messages = receipt.logs + .map { txlog -> + LogMessage( + txlog.address, + txlog.blockHash, + txlog.blockNumber, + txlog.data ?: HexData.empty(), + txlog.logIndex, + txlog.topics, + txlog.transactionHash, + txlog.transactionIndex, + false + ) + } + oldMessages.put(LogReference(update.blockHash, update.transactionId), messages) + Flux.fromIterable(messages) + } catch (t: Throwable) { + log.warn("Invalid Receipt ${update.transactionId}. ${t.javaClass}: ${t.message}") + Flux.empty() } + } } private data class LogReference( - val block: BlockId, - val tx: TxId + val block: BlockId, + val tx: TxId ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt index d92f0f13..3b9f6162 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/ProduceNewHeads.kt @@ -29,7 +29,7 @@ import reactor.core.publisher.Flux * @see NewHeadMessage */ class ProduceNewHeads( - val head: Head + val head: Head ) { companion object { @@ -40,27 +40,26 @@ class ProduceNewHeads( fun start(): Flux { return head.getFlux() - .map { - if (it.parsed != null) { - it.parsed as BlockJson - } else { - objectMapper.readValue(it.json, BlockJson::class.java) - } - } - .map { block -> - NewHeadMessage( - block.number, - block.hash, - block.parentHash, - block.timestamp, - block.difficulty, - block.gasLimit, - block.gasUsed, - block.logsBloom, - block.miner, - block.baseFeePerGas?.amount - ) + .map { + if (it.parsed != null) { + it.parsed as BlockJson + } else { + objectMapper.readValue(it.json, BlockJson::class.java) } + } + .map { block -> + NewHeadMessage( + block.number, + block.hash, + block.parentHash, + block.timestamp, + block.difficulty, + block.gasLimit, + block.gasUsed, + block.logsBloom, + block.miner, + block.baseFeePerGas?.amount + ) + } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt index 43f6a431..f0ef7d53 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/LogMessage.kt @@ -24,21 +24,21 @@ import io.emeraldpay.etherjar.hex.HexData import io.emeraldpay.etherjar.rpc.json.HexDataSerializer data class LogMessage( - @get:JsonSerialize(using = HexDataSerializer::class) - val address: Address, - @get:JsonSerialize(using = HexDataSerializer::class) - val blockHash: BlockHash, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val blockNumber: Long, - @get:JsonSerialize(using = HexDataSerializer::class) - val data: HexData, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val logIndex: Long, - @get:JsonSerialize(contentUsing = HexDataSerializer::class) - val topics: List, - @get:JsonSerialize(using = HexDataSerializer::class) - val transactionHash: TransactionId, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val transactionIndex: Long, - val removed: Boolean -) \ No newline at end of file + @get:JsonSerialize(using = HexDataSerializer::class) + val address: Address, + @get:JsonSerialize(using = HexDataSerializer::class) + val blockHash: BlockHash, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val blockNumber: Long, + @get:JsonSerialize(using = HexDataSerializer::class) + val data: HexData, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val logIndex: Long, + @get:JsonSerialize(contentUsing = HexDataSerializer::class) + val topics: List, + @get:JsonSerialize(using = HexDataSerializer::class) + val transactionHash: TransactionId, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val transactionIndex: Long, + val removed: Boolean +) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt index ac3b631d..90eec052 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NewHeadMessage.kt @@ -30,25 +30,25 @@ import java.time.Instant * transactionRoot and some others. */ data class NewHeadMessage( - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val number: Long, - @get:JsonSerialize(using = HexDataSerializer::class) - val hash: BlockHash, - @get:JsonSerialize(using = HexDataSerializer::class) - val parentHash: BlockHash, - @get:JsonSerialize(using = TimestampSerializer::class) - val timestamp: Instant, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val difficulty: BigInteger, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val gasLimit: Long, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - val gasUsed: Long, - @get:JsonSerialize(using = HexDataSerializer::class) - val logsBloom: Bloom, - @get:JsonSerialize(using = HexDataSerializer::class) - val miner: Address, - @get:JsonSerialize(using = NumberAsHexSerializer::class) - @get:JsonInclude(JsonInclude.Include.NON_NULL) - val baseFeePerGas: BigInteger? + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val number: Long, + @get:JsonSerialize(using = HexDataSerializer::class) + val hash: BlockHash, + @get:JsonSerialize(using = HexDataSerializer::class) + val parentHash: BlockHash, + @get:JsonSerialize(using = TimestampSerializer::class) + val timestamp: Instant, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val difficulty: BigInteger, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val gasLimit: Long, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + val gasUsed: Long, + @get:JsonSerialize(using = HexDataSerializer::class) + val logsBloom: Bloom, + @get:JsonSerialize(using = HexDataSerializer::class) + val miner: Address, + @get:JsonSerialize(using = NumberAsHexSerializer::class) + @get:JsonInclude(JsonInclude.Include.NON_NULL) + val baseFeePerGas: BigInteger? ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NumberAsHexSerializer.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NumberAsHexSerializer.kt index 438f9a6b..679cb17a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NumberAsHexSerializer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/NumberAsHexSerializer.kt @@ -38,5 +38,4 @@ class NumberAsHexSerializer : JsonSerializer() { } gen.writeString(hex.toHex()) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/TimestampSerializer.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/TimestampSerializer.kt index 14eafa58..583f3b2a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/TimestampSerializer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/json/TimestampSerializer.kt @@ -35,5 +35,4 @@ class TimestampSerializer : JsonSerializer() { } numberAsHex.serialize(value.epochSecond, gen, serializers) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt index 72406b8a..0b79d84b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/BitcoinGrpcUpstream.kt @@ -22,14 +22,18 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinUpstream import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.rpc.RpcException +import io.emeraldpay.grpc.Chain import org.reactivestreams.Publisher import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle @@ -40,16 +44,18 @@ import java.util.concurrent.TimeoutException import java.util.function.Function class BitcoinGrpcUpstream( - private val parentId: String, - chain: Chain, - val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, - private val client: JsonRpcGrpcClient + private val parentId: String, + chain: Chain, + val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, + private val client: JsonRpcGrpcClient ) : BitcoinUpstream( - "$parentId/${chain.chainCode}", - chain, - UpstreamsConfig.Options.getDefaults(), - UpstreamsConfig.UpstreamRole.STANDARD -), GrpcUpstream, Lifecycle { + "$parentId/${chain.chainCode}", + chain, + UpstreamsConfig.Options.getDefaults(), + UpstreamsConfig.UpstreamRole.STANDARD +), + GrpcUpstream, + Lifecycle { companion object { private val log = LoggerFactory.getLogger(BitcoinGrpcUpstream::class.java) @@ -59,13 +65,13 @@ class BitcoinGrpcUpstream( private val defaultReader: Reader = client.forSelector(Selector.empty) private val blockConverter: Function = Function { value -> val block = BlockContainer( - value.height, - BlockId.from(value.blockId), - BigInteger(1, value.weight.toByteArray()), - Instant.ofEpochMilli(value.timestamp), - false, - null, - null + value.height, + BlockId.from(value.blockId), + BigInteger(1, value.weight.toByteArray()), + Instant.ofEpochMilli(value.timestamp), + false, + null, + null ) block } @@ -73,18 +79,18 @@ class BitcoinGrpcUpstream( // head comes without transaction data // need to download transactions for the block defaultReader.read(JsonRpcRequest("getblock", listOf(existingBlock.hash.toHex()))) - .flatMap(JsonRpcResponse::requireResult) - .map(extractBlock::extract) - .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) - .doOnError { t -> - setStatus(UpstreamAvailability.UNAVAILABLE) - val msg = "Failed to download block data for chain $chain on $parentId" - if (t is RpcException || t is TimeoutException) { - log.warn("$msg. Message: ${t.message}") - } else { - log.error(msg, t) - } + .flatMap(JsonRpcResponse::requireResult) + .map(extractBlock::extract) + .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) + .doOnError { t -> + setStatus(UpstreamAvailability.UNAVAILABLE) + val msg = "Failed to download block data for chain $chain on $parentId" + if (t is RpcException || t is TimeoutException) { + log.warn("$msg. Message: ${t.message}") + } else { + log.error(msg, t) } + } } private val upstreamStatus = GrpcUpstreamStatus() private val grpcHead = GrpcHead(chain, this, blockConverter, reloadBlock) @@ -136,5 +142,4 @@ class BitcoinGrpcUpstream( this.capabilities = RemoteCapabilities.extract(conf) conf.status?.let { status -> onStatus(status) } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt index 700d91f8..7fe9b05e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/EthereumGrpcUpstream.kt @@ -24,15 +24,19 @@ import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.startup.QuorumForLabels -import io.emeraldpay.dshackle.upstream.* +import io.emeraldpay.dshackle.upstream.Capability +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.calls.CallMethods -import io.emeraldpay.dshackle.upstream.ethereum.* +import io.emeraldpay.dshackle.upstream.ethereum.EthereumUpstream import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcRequest import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcResponse -import io.emeraldpay.grpc.Chain import io.emeraldpay.etherjar.domain.BlockHash -import io.emeraldpay.etherjar.rpc.* +import io.emeraldpay.etherjar.rpc.RpcException +import io.emeraldpay.grpc.Chain import org.reactivestreams.Publisher import org.slf4j.LoggerFactory import org.springframework.context.Lifecycle @@ -43,26 +47,28 @@ import java.util.concurrent.TimeoutException import java.util.function.Function open class EthereumGrpcUpstream( - private val parentId: String, - private val chain: Chain, - private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, - private val client: JsonRpcGrpcClient + private val parentId: String, + private val chain: Chain, + private val remote: ReactorBlockchainGrpc.ReactorBlockchainStub, + private val client: JsonRpcGrpcClient ) : EthereumUpstream( - "$parentId/${chain.chainCode}", - UpstreamsConfig.Options.getDefaults(), - UpstreamsConfig.UpstreamRole.STANDARD, - null, null -), GrpcUpstream, Lifecycle { + "$parentId/${chain.chainCode}", + UpstreamsConfig.Options.getDefaults(), + UpstreamsConfig.UpstreamRole.STANDARD, + null, null +), + GrpcUpstream, + Lifecycle { private val blockConverter: Function = Function { value -> val block = BlockContainer( - value.height, - BlockId.from(BlockHash.from("0x" + value.blockId)), - BigInteger(1, value.weight.toByteArray()), - Instant.ofEpochMilli(value.timestamp), - false, - null, - null + value.height, + BlockId.from(BlockHash.from("0x" + value.blockId)), + BigInteger(1, value.weight.toByteArray()), + Instant.ofEpochMilli(value.timestamp), + false, + null, + null ) block } @@ -71,20 +77,20 @@ open class EthereumGrpcUpstream( // head comes without transaction data // need to download transactions for the block defaultReader.read(JsonRpcRequest("eth_getBlockByHash", listOf(existingBlock.hash.toHexWithPrefix(), false))) - .flatMap(JsonRpcResponse::requireResult) - .map { - BlockContainer.fromEthereumJson(it) - } - .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) - .doOnError { t -> - setStatus(UpstreamAvailability.UNAVAILABLE) - val msg = "Failed to download block data for chain $chain on $parentId" - if (t is RpcException || t is TimeoutException) { - log.warn("$msg. Message: ${t.message}") - } else { - log.error(msg, t) - } + .flatMap(JsonRpcResponse::requireResult) + .map { + BlockContainer.fromEthereumJson(it) + } + .timeout(timeout, Mono.error(TimeoutException("Timeout from upstream"))) + .doOnError { t -> + setStatus(UpstreamAvailability.UNAVAILABLE) + val msg = "Failed to download block data for chain $chain on $parentId" + if (t is RpcException || t is TimeoutException) { + log.warn("$msg. Message: ${t.message}") + } else { + log.error(msg, t) } + } } private val log = LoggerFactory.getLogger(EthereumGrpcUpstream::class.java) @@ -103,7 +109,6 @@ open class EthereumGrpcUpstream( return grpcHead.isRunning } - override fun stop() { grpcHead.stop() } @@ -157,4 +162,4 @@ open class EthereumGrpcUpstream( override fun isGrpc(): Boolean { return true } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt index 524c357e..4b5a7368 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt @@ -35,16 +35,16 @@ import java.time.Duration import java.util.function.Function class GrpcHead( - private val chain: Chain, - private val parent: DefaultUpstream, - /** - * Converted from remote head details to the block container, which could be partial at this point - */ - private val converter: Function, - /** - * Populate block data with all missing details, of any - */ - private val enhancer: Function>? + private val chain: Chain, + private val parent: DefaultUpstream, + /** + * Converted from remote head details to the block container, which could be partial at this point + */ + private val converter: Function, + /** + * Populate block data with all missing details, of any + */ + private val enhancer: Function>? ) : AbstractHead(), Lifecycle { companion object { @@ -62,10 +62,10 @@ class GrpcHead( } val source = Flux.concat( - // first connect immediately - Flux.just(remote), - // following requests do with delay, give it a time to recover - Flux.just(remote).repeat().delayElements(Defaults.retryConnection) + // first connect immediately + Flux.just(remote), + // following requests do with delay, give it a time to recover + Flux.just(remote).repeat().delayElements(Defaults.retryConnection) ).flatMap(this::subscribeHead) start(source) @@ -73,16 +73,16 @@ class GrpcHead( fun subscribeHead(client: ReactorBlockchainGrpc.ReactorBlockchainStub): Publisher { val chainRef = Common.Chain.newBuilder() - .setTypeValue(chain.id) - .build() + .setTypeValue(chain.id) + .build() return client.subscribeHead(chainRef) - // simple retry on failure, if eventually failed then it supposed to resubscribe later from outer method - .retryWhen(Retry.backoff(4, Duration.ofSeconds(1))) - .onErrorContinue { err, _ -> - log.warn("Disconnected $chain from ${parent.getId()}: ${err.message}") - parent.setStatus(UpstreamAvailability.UNAVAILABLE) - Mono.empty() - } + // simple retry on failure, if eventually failed then it supposed to resubscribe later from outer method + .retryWhen(Retry.backoff(4, Duration.ofSeconds(1))) + .onErrorContinue { err, _ -> + log.warn("Disconnected $chain from ${parent.getId()}: ${err.message}") + parent.setStatus(UpstreamAvailability.UNAVAILABLE) + Mono.empty() + } } /** @@ -90,12 +90,12 @@ class GrpcHead( */ fun start(source: Flux) { var blocks = source.map(converter) - .distinctUntilChanged { - it.hash - }.filter { block -> - val curr = this.getCurrent() - curr == null || curr.difficulty < block.difficulty - } + .distinctUntilChanged { + it.hash + }.filter { block -> + val curr = this.getCurrent() + curr == null || curr.difficulty < block.difficulty + } if (enhancer != null) { blocks = blocks.flatMap(enhancer) } @@ -118,4 +118,4 @@ class GrpcHead( override fun stop() { headSubscription?.dispose() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt index c07778fd..166252cb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstream.kt @@ -24,5 +24,4 @@ interface GrpcUpstream { * Called on the first creation, and each time a new state received from upstream */ fun update(conf: BlockchainOuterClass.DescribeChain) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamStatus.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamStatus.kt index 5baff7ad..d64348b3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamStatus.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamStatus.kt @@ -21,9 +21,8 @@ import io.emeraldpay.dshackle.startup.QuorumForLabels import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.DirectCallMethods import org.slf4j.LoggerFactory -import java.util.* +import java.util.Collections import java.util.concurrent.atomic.AtomicReference -import kotlin.collections.ArrayList class GrpcUpstreamStatus { @@ -40,15 +39,16 @@ class GrpcUpstreamStatus { val updateNodes = QuorumForLabels() conf.nodesList.forEach { remoteNode -> - val node = QuorumForLabels.QuorumItem(remoteNode.quorum, - remoteNode.labelsList.let { provided -> - val labels = UpstreamsConfig.Labels() - provided.forEach { - labels[it.name] = it.value - } - updateLabels.add(labels) - labels + val node = QuorumForLabels.QuorumItem( + remoteNode.quorum, + remoteNode.labelsList.let { provided -> + val labels = UpstreamsConfig.Labels() + provided.forEach { + labels[it.name] = it.value } + updateLabels.add(labels) + labels + } ) updateNodes.add(node) } @@ -69,4 +69,4 @@ class GrpcUpstreamStatus { fun getCallMethods(): CallMethods { return targets ?: throw IllegalStateException("Upstream is not initialized yet") } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt index 572311cd..3d294efa 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt @@ -18,15 +18,15 @@ package io.emeraldpay.dshackle.upstream.grpc import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc -import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.dshackle.Defaults import io.emeraldpay.dshackle.FileResolver import io.emeraldpay.dshackle.config.AuthConfig -import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.startup.UpstreamChange import io.emeraldpay.dshackle.upstream.DefaultUpstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcGrpcClient import io.emeraldpay.dshackle.upstream.rpcclient.RpcMetrics +import io.emeraldpay.grpc.BlockchainType import io.emeraldpay.grpc.Chain import io.grpc.ManagedChannelBuilder import io.grpc.netty.NettyChannelBuilder @@ -34,7 +34,10 @@ import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.Timer -import io.netty.handler.ssl.* +import io.netty.handler.ssl.ApplicationProtocolConfig +import io.netty.handler.ssl.ClientAuth +import io.netty.handler.ssl.SslContext +import io.netty.handler.ssl.SslContextBuilder import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.slf4j.LoggerFactory @@ -42,17 +45,16 @@ import reactor.core.Disposable import reactor.core.publisher.Flux import java.net.ConnectException import java.time.Duration -import java.util.* import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock class GrpcUpstreams( - private val id: String, - private val host: String, - private val port: Int, - private val auth: AuthConfig.ClientTlsAuth? = null, - private val fileResolver: FileResolver + private val id: String, + private val host: String, + private val port: Int, + private val auth: AuthConfig.ClientTlsAuth? = null, + private val fileResolver: FileResolver ) { private val log = LoggerFactory.getLogger(GrpcUpstreams::class.java) @@ -65,14 +67,14 @@ class GrpcUpstreams( fun start(): Flux { val channel: ManagedChannelBuilder<*> = if (auth != null && StringUtils.isNotEmpty(auth.ca)) { NettyChannelBuilder.forAddress(host, port) - .useTransportSecurity() - .enableRetry() - .maxRetryAttempts(3) - .sslContext(withTls(auth)) + .useTransportSecurity() + .enableRetry() + .maxRetryAttempts(3) + .sslContext(withTls(auth)) } else { log.warn("Using insecure connection to $host:$port") ManagedChannelBuilder.forAddress(host, port) - .usePlaintext() + .usePlaintext() } val client = ReactorBlockchainGrpc.newReactorStub(channel.build()) @@ -81,34 +83,34 @@ class GrpcUpstreams( val statusSubscription = AtomicReference() val updates = Flux.interval(Duration.ZERO, Duration.ofMinutes(1)) - .flatMap { - client.describe(BlockchainOuterClass.DescribeRequest.newBuilder().build()) - }.onErrorContinue { t, _ -> - if (ExceptionUtils.indexOfType(t, ConnectException::class.java) >= 0) { - log.warn("gRPC upstream $host:$port is unavailable") - known.values.forEach { - it.setStatus(UpstreamAvailability.UNAVAILABLE) - } - } else { - log.error("Failed to get description from $host:$port", t) + .flatMap { + client.describe(BlockchainOuterClass.DescribeRequest.newBuilder().build()) + }.onErrorContinue { t, _ -> + if (ExceptionUtils.indexOfType(t, ConnectException::class.java) >= 0) { + log.warn("gRPC upstream $host:$port is unavailable") + known.values.forEach { + it.setStatus(UpstreamAvailability.UNAVAILABLE) } - }.flatMap { value -> - processDescription(value) - }.doOnNext { - val subscription = client.subscribeStatus(BlockchainOuterClass.StatusRequest.newBuilder().build()) - .subscribe { value -> - val chain = Chain.byId(value.chain.number) - if (chain != Chain.UNSPECIFIED) { - known[chain]?.onStatus(value) - } - } - statusSubscription.updateAndGet { prev -> - prev?.dispose() - subscription - } - }.doOnError { t -> - log.error("Failed to process update from gRPC upstream $id", t) + } else { + log.error("Failed to get description from $host:$port", t) } + }.flatMap { value -> + processDescription(value) + }.doOnNext { + val subscription = client.subscribeStatus(BlockchainOuterClass.StatusRequest.newBuilder().build()) + .subscribe { value -> + val chain = Chain.byId(value.chain.number) + if (chain != Chain.UNSPECIFIED) { + known[chain]?.onStatus(value) + } + } + statusSubscription.updateAndGet { prev -> + prev?.dispose() + subscription + } + }.doOnError { t -> + log.error("Failed to process update from gRPC upstream $id", t) + } return updates } @@ -143,40 +145,42 @@ class GrpcUpstreams( internal fun withTls(auth: AuthConfig.ClientTlsAuth): SslContext { val sslContext = SslContextBuilder.forClient() - .clientAuth(ClientAuth.REQUIRE) + .clientAuth(ClientAuth.REQUIRE) sslContext.trustManager(fileResolver.resolve(auth.ca!!).inputStream()) if (StringUtils.isNotEmpty(auth.key) && StringUtils.isNoneEmpty(auth.certificate)) { sslContext.keyManager( - fileResolver.resolve(auth.certificate!!).inputStream(), - fileResolver.resolve(auth.key!!).inputStream()) + fileResolver.resolve(auth.certificate!!).inputStream(), + fileResolver.resolve(auth.key!!).inputStream() + ) } else { log.warn("Connect to remote using only CA certificate") } val alpn = ApplicationProtocolConfig( - ApplicationProtocolConfig.Protocol.ALPN, - ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, - ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, - "grpc-exp", "h2") + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + "grpc-exp", "h2" + ) sslContext.applicationProtocolConfig(alpn) return sslContext.build() } fun getOrCreate(chain: Chain): UpstreamChange { val metricsTags = listOf( - Tag.of("upstream", id), - Tag.of("chain", chain.chainCode) + Tag.of("upstream", id), + Tag.of("chain", chain.chainCode) ) val metrics = RpcMetrics( - Timer.builder("upstream.grpc.conn") - .description("Request time through a gRPC connection") - .tags(metricsTags) - .publishPercentileHistogram() - .register(Metrics.globalRegistry), - Counter.builder("upstream.grpc.err") - .description("Errors received on request through gRPC connection") - .tags(metricsTags) - .register(Metrics.globalRegistry) + Timer.builder("upstream.grpc.conn") + .description("Request time through a gRPC connection") + .tags(metricsTags) + .publishPercentileHistogram() + .register(Metrics.globalRegistry), + Counter.builder("upstream.grpc.err") + .description("Errors received on request through gRPC connection") + .tags(metricsTags) + .register(Metrics.globalRegistry) ) val blockchainType = BlockchainType.from(chain) @@ -224,4 +228,4 @@ class GrpcUpstreams( fun get(chain: Chain): DefaultUpstream { return known[chain]!! } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/RemoteCapabilities.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/RemoteCapabilities.kt index 3f03e14b..0974c551 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/RemoteCapabilities.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/RemoteCapabilities.kt @@ -19,5 +19,4 @@ class RemoteCapabilities { } ?: emptySet() } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcError.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcError.kt index 50baa980..b5ed39bd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcError.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcError.kt @@ -25,7 +25,7 @@ class JsonRpcError(val code: Int, val message: String, val details: Any?) { @JvmStatic fun from(err: RpcException): JsonRpcError { return JsonRpcError( - err.code, err.rpcMessage, err.details + err.code, err.rpcMessage, err.details ) } } @@ -55,6 +55,4 @@ class JsonRpcError(val code: Int, val message: String, val details: Any?) { override fun toString(): String { return "JsonRpcError(code=$code, message='$message', details=$details)" } - - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcException.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcException.kt index 1ab5c9bd..5dbaa65e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcException.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcException.kt @@ -18,8 +18,8 @@ package io.emeraldpay.dshackle.upstream.rpcclient import io.emeraldpay.etherjar.rpc.RpcException class JsonRpcException( - val id: JsonRpcResponse.Id, - val error: JsonRpcError + val id: JsonRpcResponse.Id, + val error: JsonRpcError ) : Exception(error.message) { constructor(id: Int, message: String) : this(JsonRpcResponse.NumberId(id), JsonRpcError(-32005, message)) @@ -34,8 +34,8 @@ class JsonRpcException( } } ?: JsonRpcResponse.NumberId(-4) return JsonRpcException( - id, JsonRpcError.from(err) + id, JsonRpcError.from(err) ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt index 98d64479..f154b673 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt @@ -15,26 +15,23 @@ */ package io.emeraldpay.dshackle.upstream.rpcclient -import com.fasterxml.jackson.databind.ObjectMapper import com.google.protobuf.ByteString import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.Selector -import io.emeraldpay.grpc.Chain -import io.grpc.Channel import io.emeraldpay.etherjar.rpc.RpcException import io.emeraldpay.etherjar.rpc.RpcResponseError +import io.emeraldpay.grpc.Chain import org.slf4j.LoggerFactory -import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.util.concurrent.TimeUnit class JsonRpcGrpcClient( - private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub, - private val chain: Chain, - private val metrics: RpcMetrics + private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub, + private val chain: Chain, + private val metrics: RpcMetrics ) { companion object { @@ -46,16 +43,16 @@ class JsonRpcGrpcClient( } class Executor( - private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub, - private val chain: Chain, - private val matcher: Selector.Matcher, - private val metrics: RpcMetrics + private val stub: ReactorBlockchainGrpc.ReactorBlockchainStub, + private val chain: Chain, + private val matcher: Selector.Matcher, + private val metrics: RpcMetrics ) : Reader { override fun read(key: JsonRpcRequest): Mono { var startTime: Long = 0 val req = BlockchainOuterClass.NativeCallRequest.newBuilder() - .setChainValue(chain.id) + .setChainValue(chain.id) if (matcher != Selector.empty) { Selector.extractLabels(matcher)?.asProto().let { @@ -64,37 +61,40 @@ class JsonRpcGrpcClient( } BlockchainOuterClass.NativeCallItem.newBuilder() - .setId(1) - .setMethod(key.method) - .setPayload(ByteString.copyFrom(Global.objectMapper.writeValueAsBytes(key.params))) - .build().let { - req.addItems(it) - } + .setId(1) + .setMethod(key.method) + .setPayload(ByteString.copyFrom(Global.objectMapper.writeValueAsBytes(key.params))) + .build().let { + req.addItems(it) + } return Mono.just(key) - .doOnNext { - startTime = System.nanoTime() - }.flatMap { - stub.nativeCall(req.build()) - .single() - .flatMap { resp -> - if (resp.succeed) { - val bytes = resp.payload.toByteArray() - Mono.just(JsonRpcResponse(bytes, null)) - } else { - metrics.errors.increment() - Mono.error(RpcException(RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR, resp.errorMessage)) - } - } - } - .doOnNext { - if (startTime > 0) { - val now = System.nanoTime() - metrics.timer.record(now - startTime, TimeUnit.NANOSECONDS) + .doOnNext { + startTime = System.nanoTime() + }.flatMap { + stub.nativeCall(req.build()) + .single() + .flatMap { resp -> + if (resp.succeed) { + val bytes = resp.payload.toByteArray() + Mono.just(JsonRpcResponse(bytes, null)) + } else { + metrics.errors.increment() + Mono.error( + RpcException( + RpcResponseError.CODE_UPSTREAM_CONNECTION_ERROR, + resp.errorMessage + ) + ) + } } + } + .doOnNext { + if (startTime > 0) { + val now = System.nanoTime() + metrics.timer.record(now - startTime, TimeUnit.NANOSECONDS) } + } } - } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClient.kt index 5fbf6b04..07bc9601 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpClient.kt @@ -30,7 +30,7 @@ import java.io.ByteArrayInputStream import java.security.KeyStore import java.security.cert.CertificateFactory import java.security.cert.X509Certificate -import java.util.* +import java.util.Base64 import java.util.concurrent.TimeUnit import java.util.function.Consumer @@ -38,10 +38,10 @@ import java.util.function.Consumer * JSON RPC client */ class JsonRpcHttpClient( - private val target: String, - private val metrics: RpcMetrics, - basicAuth: AuthConfig.ClientBasicAuth? = null, - tlsCAAuth: ByteArray? = null + private val target: String, + private val metrics: RpcMetrics, + basicAuth: AuthConfig.ClientBasicAuth? = null, + tlsCAAuth: ByteArray? = null ) : Reader { companion object { @@ -84,14 +84,20 @@ class JsonRpcHttpClient( fun execute(request: ByteArray): Mono { val response = httpClient - .post() - .uri(target) - .send(Mono.just(request).map { Unpooled.wrappedBuffer(it) }) + .post() + .uri(target) + .send(Mono.just(request).map { Unpooled.wrappedBuffer(it) }) return response.response { header, bytes -> if (header.status().code() != 200) { - Mono.error(JsonRpcException(JsonRpcResponse.NumberId(-2), - JsonRpcError(RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, "HTTP Code: ${header.status().code()}")) + Mono.error( + JsonRpcException( + JsonRpcResponse.NumberId(-2), + JsonRpcError( + RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, + "HTTP Code: ${header.status().code()}" + ) + ) ) } else { bytes.aggregate().asByteArray() @@ -102,27 +108,26 @@ class JsonRpcHttpClient( override fun read(key: JsonRpcRequest): Mono { var startTime: Long = 0 return Mono.just(key) - .map(JsonRpcRequest::toJson) - .doOnNext { - startTime = System.nanoTime() + .map(JsonRpcRequest::toJson) + .doOnNext { + startTime = System.nanoTime() + } + .flatMap(this@JsonRpcHttpClient::execute) + .doOnNext { + if (startTime > 0) { + val now = System.nanoTime() + metrics.timer.record(now - startTime, TimeUnit.NANOSECONDS) } - .flatMap(this@JsonRpcHttpClient::execute) - .doOnNext { - if (startTime > 0) { - val now = System.nanoTime() - metrics.timer.record(now - startTime, TimeUnit.NANOSECONDS) - } - } - .map(parser::parse) - .onErrorResume { t -> - val err = when (t) { - is RpcException -> JsonRpcResponse.error(t.code, t.rpcMessage) - is JsonRpcException -> JsonRpcResponse.error(t.error, JsonRpcResponse.NumberId(1)) - else -> JsonRpcResponse.error(1, t.message ?: t.javaClass.name) - } - metrics.errors.increment() - Mono.just(err) + } + .map(parser::parse) + .onErrorResume { t -> + val err = when (t) { + is RpcException -> JsonRpcResponse.error(t.code, t.rpcMessage) + is JsonRpcException -> JsonRpcResponse.error(t.error, JsonRpcResponse.NumberId(1)) + else -> JsonRpcResponse.error(1, t.message ?: t.javaClass.name) } + metrics.errors.increment() + Mono.just(err) + } } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcRequest.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcRequest.kt index bef4df17..b0d92372 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcRequest.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcRequest.kt @@ -22,19 +22,19 @@ import com.fasterxml.jackson.databind.JsonNode import io.emeraldpay.dshackle.Global data class JsonRpcRequest( - val method: String, - val params: List, - val id: Int + val method: String, + val params: List, + val id: Int ) { constructor(method: String, params: List) : this(method, params, 1) fun toJson(): ByteArray { val json = mapOf( - "jsonrpc" to "2.0", - "id" to id, - "method" to method, - "params" to params + "jsonrpc" to "2.0", + "id" to id, + "method" to method, + "params" to params ) return Global.objectMapper.writeValueAsBytes(json) } @@ -64,6 +64,5 @@ data class JsonRpcRequest( } return JsonRpcRequest(method, params, id) } - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcResponse.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcResponse.kt index 63fb1e67..96854af4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcResponse.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcResponse.kt @@ -21,9 +21,9 @@ import com.fasterxml.jackson.databind.SerializerProvider import reactor.core.publisher.Mono class JsonRpcResponse( - private val result: ByteArray?, - val error: JsonRpcError?, - val id: Id + private val result: ByteArray?, + val error: JsonRpcError?, + val id: Id ) { constructor(result: ByteArray?, error: JsonRpcError?) : this(result, error, NumberId(0)) @@ -38,7 +38,7 @@ class JsonRpcResponse( @JvmStatic fun ok(value: ByteArray, id: Id): JsonRpcResponse { - return JsonRpcResponse(value, null, id); + return JsonRpcResponse(value, null, id) } @JvmStatic @@ -250,4 +250,4 @@ class JsonRpcResponse( gen.writeEndObject() } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcWsClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcWsClient.kt index 9bed273d..9bb50050 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcWsClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcWsClient.kt @@ -20,11 +20,10 @@ import io.emeraldpay.dshackle.upstream.ethereum.EthereumWsFactory import reactor.core.publisher.Mono class JsonRpcWsClient( - private val ws: EthereumWsFactory.EthereumWs + private val ws: EthereumWsFactory.EthereumWs ) : Reader { override fun read(key: JsonRpcRequest): Mono { return ws.call(key) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt index af75484b..56f106a0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt @@ -44,7 +44,12 @@ abstract class ResponseParser { val parser: JsonParser = jsonFactory.createParser(json) parser.nextToken() if (parser.currentToken != JsonToken.START_OBJECT) { - return Preparsed(error = JsonRpcError(RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, "Invalid JSON: not an Object")) + return Preparsed( + error = JsonRpcError( + RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, + "Invalid JSON: not an Object" + ) + ) } while (parser.nextToken() != JsonToken.END_OBJECT) { val field = parser.currentName @@ -56,13 +61,23 @@ abstract class ResponseParser { if (state.isReady) { return state } - return Preparsed(error = JsonRpcError(RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, "Invalid JSON structure: never finalized")) + return Preparsed( + error = JsonRpcError( + RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, + "Invalid JSON structure: never finalized" + ) + ) } open fun process(parser: JsonParser, json: ByteArray, field: String, state: Preparsed): Preparsed { if (field == "jsonrpc") { if (!parser.nextToken().isScalarValue) { - return state.copy(error = JsonRpcError(RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, "Invalid JSON (jsonrpc value)")) + return state.copy( + error = JsonRpcError( + RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, + "Invalid JSON (jsonrpc value)" + ) + ) } // just skip the field return state @@ -71,7 +86,7 @@ abstract class ResponseParser { } else if (field == "result") { val result = readResult(json, parser) return if (result == null) { - //if result is null we should check if an error is also present, and if it's set then return only the error + // if result is null we should check if an error is also present, and if it's set then return only the error state.copy(nullResult = true) } else { state.copy(result = result) @@ -160,23 +175,22 @@ abstract class ResponseParser { } data class Preparsed( - val id: JsonRpcResponse.Id? = null, - val result: ByteArray? = null, - val nullResult: Boolean = false, - val error: JsonRpcError? = null, - val subMethod: String? = null, - val subId: String? = null + val id: JsonRpcResponse.Id? = null, + val result: ByteArray? = null, + val nullResult: Boolean = false, + val error: JsonRpcError? = null, + val subMethod: String? = null, + val subId: String? = null ) { private val isResultSet = result != null || nullResult val isRpcReady: Boolean = id != null && - (error != null || isResultSet) + (error != null || isResultSet) val isSubReady: Boolean = subId != null && - isResultSet + isResultSet val isReady: Boolean = isRpcReady || isSubReady - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParser.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParser.kt index 7db80ace..33d58b23 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParser.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParser.kt @@ -17,7 +17,7 @@ package io.emeraldpay.dshackle.upstream.rpcclient import org.slf4j.LoggerFactory -open class ResponseRpcParser() : ResponseParser() { +open class ResponseRpcParser : ResponseParser() { companion object { private val log = LoggerFactory.getLogger(ResponseRpcParser::class.java) @@ -32,5 +32,4 @@ open class ResponseRpcParser() : ResponseParser() { } return JsonRpcResponse(state.result, null, state.id ?: JsonRpcResponse.Id.from(-1)) } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt index 07e7e654..87c76746 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonToken import org.slf4j.LoggerFactory import java.io.IOException - class ResponseWSParser : ResponseParser() { companion object { @@ -31,18 +30,18 @@ class ResponseWSParser : ResponseParser() { override fun build(state: Preparsed): WsResponse { if (state.isRpcReady) { return WsResponse( - Type.RPC, - state.id!!, - if (state.nullResult) NULL_RESULT else state.result, - state.error + Type.RPC, + state.id!!, + if (state.nullResult) NULL_RESULT else state.result, + state.error ) } if (state.isSubReady) { return WsResponse( - Type.SUBSCRIPTION, - JsonRpcResponse.Id.from(state.subId!!), - if (state.nullResult) NULL_RESULT else state.result, - state.error + Type.SUBSCRIPTION, + JsonRpcResponse.Id.from(state.subId!!), + if (state.nullResult) NULL_RESULT else state.result, + state.error ) } throw IllegalStateException("State is not ready") @@ -51,7 +50,7 @@ class ResponseWSParser : ResponseParser() { override fun process(parser: JsonParser, json: ByteArray, field: String, state: Preparsed): Preparsed { if ("method" == field) { parser.nextToken() - val method = parser.getValueAsString() + val method = parser.valueAsString return state.copy(subMethod = method) } if ("params" == field) { @@ -66,7 +65,7 @@ class ResponseWSParser : ResponseParser() { // }, // "subscription": "...." // } - //} + // } return decodeSubscription(parser, json, state) } return super.process(parser, json, field, state) @@ -101,10 +100,9 @@ class ResponseWSParser : ResponseParser() { } data class WsResponse( - val type: Type, - val id: JsonRpcResponse.Id, - val value: ByteArray?, - val error: JsonRpcError? + val type: Type, + val id: JsonRpcResponse.Id, + val value: ByteArray?, + val error: JsonRpcError? ) - -} \ No newline at end of file +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/RpcMetrics.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/RpcMetrics.kt index a21e40d1..67e8156b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/RpcMetrics.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/RpcMetrics.kt @@ -19,6 +19,6 @@ import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Timer class RpcMetrics( - val timer: Timer, - val errors: Counter -) \ No newline at end of file + val timer: Timer, + val errors: Counter +)